You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@carbondata.apache.org by praveenmeenakshi56 <gi...@git.apache.org> on 2018/03/07 15:08:00 UTC
[GitHub] carbondata pull request #2042: [CARBONDATA-2236]added sdv test cases for sta...
GitHub user praveenmeenakshi56 opened a pull request:
https://github.com/apache/carbondata/pull/2042
[CARBONDATA-2236]added sdv test cases for standard partition
added sdv test cases for standard partition
Be sure to do all of the following checklist to help us incorporate
your contribution quickly and easily:
- [ ] Any interfaces changed?
- [ ] Any backward compatibility impacted?
- [ ] Document update required?
- [ ] Testing done
Please provide details on
- Whether new unit test cases have been added or why no new tests are required?
- How it is tested? Please attach test report.
- Is it a performance related change? Please attach the performance test report.
- Any additional information to help reviewers in testing this change.
- [ ] For large changes, please consider breaking it into sub-tasks under an umbrella JIRA.
You can merge this pull request into a Git repository by running:
$ git pull https://github.com/praveenmeenakshi56/carbondata partition_test
Alternatively you can review and apply these changes as the patch at:
https://github.com/apache/carbondata/pull/2042.patch
To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:
This closes #2042
----
commit 718af59351368db4577be1cad4808b9d789abf72
Author: praveenmeenakshi56 <pr...@...>
Date: 2018-03-07T15:02:39Z
added sdv test cases for standard partition
----
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2042
Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder1/4371/
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2042
Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder1/4269/
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2042
Build Success with Spark 2.2.1, Please check CI http://88.99.58.216:8080/job/ApacheCarbonPRBuilder/3132/
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2042
Build Failed with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder1/4158/
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2042
Build Success with Spark 2.2.1, Please check CI http://88.99.58.216:8080/job/ApacheCarbonPRBuilder/2881/
---
[GitHub] carbondata pull request #2042: [CARBONDATA-2236]added sdv test cases for sta...
Posted by kunal642 <gi...@git.apache.org>.
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2042#discussion_r173373691
--- Diff: integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/StandardPartitionTestCase.scala ---
@@ -0,0 +1,436 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.cluster.sdv.generated
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util._
+import org.scalatest.BeforeAndAfterAll
+
+/**
+ * Test Class for partitionTestCase to verify all scenerios
+ */
+
+class StandardPartitionTestCase extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll = {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd HH:mm:ss")
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
+ }
+ //Verify exception if column in partitioned by is already specified in table schema
+
+ test("Standard-Partition_TC001", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (INTEGER_COLUMN1 int)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','LIST_INFO'='1,3')""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify table is created with Partition
+ test("Standard-Partition_TC002", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY (INTEGER_COLUMN1 int)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','LIST_INFO'='1,3')""")
+ val df = sql(s"""DESC uniqdata""")
+ assert(df.collect().reverse.head.get(0).toString.toUpperCase.contains("INTEGER_COLUMN1"))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify table is created with Partition with table comment
+
+ test("Standard-Partition_TC003",Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, decimalField DECIMAL(18,2), charField CHAR(5), floatField FLOAT ) COMMENT 'partition_table' PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ val df = sql(s"""DESC formatted partition_table""")
+ checkExistence(df, true, "partition_table")
+ }
+
+ //Verify WHEN partitioned by is not specified in the DDL, but partition type,number of partitions and list info are given
+ test("Standard-Partition_TC004", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','NUM_PARTITIONS'='3','LIST_INFO'='1,3')""")
+ assert(true)
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Creating a partition table, with ARRAY type partitioned column
+ test("Standard-Partition_TC005", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists partition_table_array""")
+ sql(s"""CREATE TABLE partition_table_array(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, stringField STRING, timestampField TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (complexData ARRAY<STRING>) STORED BY 'carbondata'""")
+ sql(s"""drop table if exists partition_table_array""")
+ }
+ }
+
+ //Verify exception while Creating a partition table with DICTIONARY_INCLUDE
+ test("Standard-Partition_TC006", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists partition_table_string""")
+ sql(s"""CREATE TABLE partition_table_string(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestampField TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT, complexData ARRAY<STRING> ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('DICTIONARY_INCLUDE'='stringField')""")
+ sql(s"""drop table if exists partition_table_string""")
+ }
+ }
+
+ //Creating a partition table with DICTIONARY_EXCLUDE
+ test("Standard-Partition_TC007", Include) {
+ sql(s"""drop table if exists partition_table_string""")
+ sql(s"""CREATE TABLE partition_table_string(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestampField TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT, complexData ARRAY<STRING> ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('DICTIONARY_EXCLUDE'='stringField')""")
+ sql(s"""drop table if exists partition_table_string""")
+ }
+
+ //Verify exception if datatype is not provided with partition column
+ test("Standard-Partition_TC008", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY (DOJ)STORED BY 'org.apache.carbondata.format'""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify exception if non existent file header is provided in partition
+ test("Standard-Partition_TC009", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY (DOJ timestamp)STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata OPTIONS('DELIMITER'=',','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOJ,,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify exception if PARTITION BY is empty
+ test("Standard-Partition_TC010", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY ()STORED BY 'org.apache.carbondata.format'""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=LOCAL_SORT
+ test("Standard-Partition_TC011", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='LOCAL_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=GLOBAL_SORT
+ test("Standard-Partition_TC012", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=BATCH_SORT
+ test("Standard-Partition_TC013", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='BATCH_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=NO_SORT
+ test("Standard-Partition_TC014", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='NO_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with Bad Records Action = FORCE
+ test("Standard-Partition_TC015", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='FORCE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""select count(*) from partition_table""").show()
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Verify Exception when Loading data into a partitioned table with Bad Records Action = FAIL
+ test("Standard-Partition_TC016", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='FAIL')""")
+ sql(s"""drop table if exists partition_table""")
+ }
+ }
+
+ //Loading data into a partitioned table with Bad Records Action = IGNORE
+ test("Standard-Partition_TC017", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='IGNORE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(0)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with Bad Records Action = REDIRECT
+ test("Standard-Partition_TC018", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='REDIRECT')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(0)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with SINGLE_PASS=TRUE
+ test("Standard-Partition_TC019", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','SINGLE_PASS'='TRUE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with SINGLE_PASS=FALSE
+ test("Standard-Partition_TC020", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','SINGLE_PASS'='FALSE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Verify load with Standard Partition
+ test("Standard-Partition_TC021", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select count(*) from uniqdata"""), Seq(Row(28)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify load with Standard Partition with limit 1
+ test("Standard-Partition_TC022", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select * from uniqdata limit 1"""),Seq(Row("CUST_NAME_00000","ACTIVE_EMUI_VERSION_00000",null,null,null,12345678901.0000000000,22345678901.0000000000,1.123456749E10,-1.123456749E10,1,null,1)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify load with Standard Partition with select partition column
+ test("Standard-Partition_TC023", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select CUST_ID from uniqdata limit 1"""), Seq(Row(1)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify table creation if 2 partition columns are provided
+ test("Standard-Partition_TC024", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (CUST_ID int,DOJ timestamp) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1',doj) OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ val df = sql(s"""DESC uniqdata""").collect()
+ assert(df(df.indexWhere(_.get(0).toString.contains("# Partition")) + 3).get(0).toString.contains("doj"))
+ assert(df(df.indexWhere(_.get(0).toString.contains("# Partition")) + 2).get(0).toString.contains("cust_id"))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify load with Standard Partition after compaction
+ test("Standard-Partition_TC025", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""alter table uniqdata compact 'minor'""")
+ checkAnswer(sql(s"""select count(*) from uniqdata"""), Seq(Row(84)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify join operation on Standard Partition
+ test("Standard-Partition_TC026", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""drop table if exists uniqdata1""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""CREATE TABLE uniqdata1 (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata1 partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select a.cust_id, b.cust_id from uniqdata a, uniqdata1 b where a.cust_id >= b.cust_id limit 1"""),Seq(Row(1,1)))
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""drop table if exists uniqdata1""")
+ }
+
+ //Verify exception if partition column is dropped
+ test("Standard-Partition_TC027", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""alter table uniqdata drop columns(CUST_ID)""")
+ sql(s"""drop table if exists uniqdata""")
+ }
+ }
+
+ //Verify INSERT operation on standard partition
+ test("Standard-Partition_TC028", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""insert into table uniqdata values ('a', '1','2015-07-01 00:00:00', 5678,7654,23.4, 55.6, 7654, 8765,33,'2015-07-01 00:00:00', 1)""")
+ sql(s"""insert into table uniqdata values ('a', '1','2015-07-01 00:00:00', 5678,7654,23.4, 55.6, 7654, 8765,33,'2015-07-01 00:00:00', 0)""")
+ checkAnswer(sql(s"""select count(*) from uniqdata"""), Seq(Row(2)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify INSERT INTO SELECT operation on standard partition
+ test("Standard-Partition_TC029", Include) {
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE_load""")
+ sql(s"""CREATE TABLE partition_table_load (shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table_load options ('BAD_RECORDS_ACTION'='FORCE')""")
+ sql(s"""INSERT into TABLE partition_table PARTITION (stringfield = 'Hello') SELECT * FROM partition_table_load au WHERE au.intfield = 25""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""),Seq(Row(2)))
+ sql(s"""drop table if exists PARTITION_TABLE""")
+ sql(s"""drop table if exists PARTITION_TABLE_load""")
+ }
+
+ //Verify INSERT overwrite operation on standard partition
+ test("Standard-Partition_TC030", Include) {
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE_load""")
+ sql(s"""CREATE TABLE partition_table_load (shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table_load options ('BAD_RECORDS_ACTION'='FORCE')""")
+ sql(s"""INSERT OVERWRITE TABLE partition_table PARTITION (stringfield = 'Hello') SELECT * FROM partition_table_load au WHERE au.intField = 25""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""),Seq(Row(2)))
+ sql(s"""drop table if exists PARTITION_TABLE""")
+ sql(s"""drop table if exists PARTITION_TABLE_load""")
+ }
+
+ //Verify date with > filter condition and standard partition
+ test("Standard-Partition_TC031", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='4') OPTIONS('DELIMITER'=',' , 'BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select count(*) from uniqdata where CUST_ID>3"""),Seq(Row(28)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify date with = filter condition and standard partition
+ test("Standard-Partition_TC032", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='4')OPTIONS('DELIMITER'=',' , 'BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select count(*) from uniqdata where CUST_ID=3"""),Seq(Row(0)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify date with = value and standard partition
+ test("Standard-Partition_TC033", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='4') OPTIONS('DELIMITER'=',' , 'BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select count(*) from uniqdata where CUST_ID=10"""),Seq(Row(0)))
+
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify describe partition_table on standard Partition
+ test("Standard-Partition_TC034", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ val df = sql(s"""DESC partition_table""")
+ assert(df.collect().reverse.head.get(0).toString.toUpperCase.contains("STRINGFIELD"))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Verify update partition_table on standard Partition
+ test("Standard-Partition_TC035", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table PARTITION (stringField = "Hello")""")
+ sql("""update partition_table set (stringfield)=('China') where stringfield = 'Hello'""")
+ checkAnswer(sql(s"""show partitions partition_table"""),Seq(Row("stringfield=Hello")))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Verify update partition_table on standard Partition
+ test("Standard-Partition_TC036", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table PARTITION (stringField = 'Hello')""")
+ sql("""update partition_table set (stringfield)=('China') where stringfield = 'Hello'""")
+ sql("""update partition_table set (stringfield)=('China123') where stringfield != 'China'""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""),Seq(Row(10)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Verify update partition_table on standard Partition
+ test("Standard-Partition_TC037", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table PARTITION (stringField = 'Hello')""")
+ sql("""update partition_table set (stringfield)=('China') where stringfield = 'Hello'""")
+ sql("""update partition_table set (stringfield)=('China123') where stringfield != 'China'""")
+ sql("""update partition_table set (stringfield)=('Japan') where stringfield > 'China'""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""),Seq(Row(10)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Verify update partition_table on standard Partition
+ test("Standard-Partition_TC038", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table PARTITION (stringField = 'Hello')""")
+ sql("""update partition_table set (stringfield)=('Asia') where stringfield < 'hello'""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""),Seq(Row(10)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Verify update partition_table on standard Partition
+ test("Standard-Partition_TC039", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table PARTITION (stringField = 'Hello')""")
+ sql("""update partition_table set (stringfield)=('China') where stringfield = 'Hello'""")
+ sql("""update partition_table set (stringfield)=('Europe') where stringfield LIKE 'C%'""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""),Seq(Row(10)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Verify update partition_table on standard Partition
+ test("Standard-Partition_TC040", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table PARTITION (stringField = 'Hello')""")
+ sql("""update partition_table set (stringfield)=('China') where stringfield = 'Hello'""")
+ sql("""update partition_table set (stringfield)=('Europe') where stringfield LIKE 'C%'""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""),Seq(Row(10)))
--- End diff --
change all update tests to check for the updated value
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on the issue:
https://github.com/apache/carbondata/pull/2042
SDV Build Fail , Please check CI http://144.76.159.231:8080/job/ApacheSDVTests/3815/
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2042
Build Failed with Spark 2.2.1, Please check CI http://88.99.58.216:8080/job/ApacheCarbonPRBuilder/2914/
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2042
Build Success with Spark 2.2.1, Please check CI http://88.99.58.216:8080/job/ApacheCarbonPRBuilder/3137/
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by kunal642 <gi...@git.apache.org>.
Github user kunal642 commented on the issue:
https://github.com/apache/carbondata/pull/2042
LGTM
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by praveenmeenakshi56 <gi...@git.apache.org>.
Github user praveenmeenakshi56 commented on the issue:
https://github.com/apache/carbondata/pull/2042
retest this please
---
[GitHub] carbondata pull request #2042: [CARBONDATA-2236]added sdv test cases for sta...
Posted by kunal642 <gi...@git.apache.org>.
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2042#discussion_r173367821
--- Diff: integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/StandardPartitionTestCase.scala ---
@@ -0,0 +1,436 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.cluster.sdv.generated
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util._
+import org.scalatest.BeforeAndAfterAll
+
+/**
+ * Test Class for partitionTestCase to verify all scenerios
+ */
+
+class StandardPartitionTestCase extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll = {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd HH:mm:ss")
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
+ }
+ //Verify exception if column in partitioned by is already specified in table schema
+
+ test("Standard-Partition_TC001", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (INTEGER_COLUMN1 int)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','LIST_INFO'='1,3')""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify table is created with Partition
+ test("Standard-Partition_TC002", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY (INTEGER_COLUMN1 int)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','LIST_INFO'='1,3')""")
+ val df = sql(s"""DESC uniqdata""")
+ assert(df.collect().reverse.head.get(0).toString.toUpperCase.contains("INTEGER_COLUMN1"))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify table is created with Partition with table comment
+
+ test("Standard-Partition_TC003",Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, decimalField DECIMAL(18,2), charField CHAR(5), floatField FLOAT ) COMMENT 'partition_table' PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ val df = sql(s"""DESC formatted partition_table""")
+ checkExistence(df, true, "partition_table")
+ }
+
+ //Verify WHEN partitioned by is not specified in the DDL, but partition type,number of partitions and list info are given
+ test("Standard-Partition_TC004", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','NUM_PARTITIONS'='3','LIST_INFO'='1,3')""")
+ assert(true)
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Creating a partition table, with ARRAY type partitioned column
+ test("Standard-Partition_TC005", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists partition_table_array""")
+ sql(s"""CREATE TABLE partition_table_array(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, stringField STRING, timestampField TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (complexData ARRAY<STRING>) STORED BY 'carbondata'""")
+ sql(s"""drop table if exists partition_table_array""")
+ }
+ }
+
+ //Verify exception while Creating a partition table with DICTIONARY_INCLUDE
+ test("Standard-Partition_TC006", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists partition_table_string""")
+ sql(s"""CREATE TABLE partition_table_string(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestampField TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT, complexData ARRAY<STRING> ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('DICTIONARY_INCLUDE'='stringField')""")
+ sql(s"""drop table if exists partition_table_string""")
+ }
+ }
+
+ //Creating a partition table with DICTIONARY_EXCLUDE
+ test("Standard-Partition_TC007", Include) {
+ sql(s"""drop table if exists partition_table_string""")
+ sql(s"""CREATE TABLE partition_table_string(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestampField TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT, complexData ARRAY<STRING> ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('DICTIONARY_EXCLUDE'='stringField')""")
+ sql(s"""drop table if exists partition_table_string""")
+ }
+
+ //Verify exception if datatype is not provided with partition column
+ test("Standard-Partition_TC008", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY (DOJ)STORED BY 'org.apache.carbondata.format'""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify exception if non existent file header is provided in partition
+ test("Standard-Partition_TC009", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY (DOJ timestamp)STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata OPTIONS('DELIMITER'=',','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOJ,,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify exception if PARTITION BY is empty
+ test("Standard-Partition_TC010", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY ()STORED BY 'org.apache.carbondata.format'""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=LOCAL_SORT
+ test("Standard-Partition_TC011", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='LOCAL_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=GLOBAL_SORT
+ test("Standard-Partition_TC012", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=BATCH_SORT
+ test("Standard-Partition_TC013", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='BATCH_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=NO_SORT
+ test("Standard-Partition_TC014", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='NO_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with Bad Records Action = FORCE
+ test("Standard-Partition_TC015", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='FORCE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""select count(*) from partition_table""").show()
--- End diff --
remove this line
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by praveenmeenakshi56 <gi...@git.apache.org>.
Github user praveenmeenakshi56 commented on the issue:
https://github.com/apache/carbondata/pull/2042
retest SDV please
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2042
Build Failed with Spark 2.2.1, Please check CI http://88.99.58.216:8080/job/ApacheCarbonPRBuilder/2913/
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on the issue:
https://github.com/apache/carbondata/pull/2042
SDV Build Fail , Please check CI http://144.76.159.231:8080/job/ApacheSDVTests/3817/
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2042
Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder1/4243/
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on the issue:
https://github.com/apache/carbondata/pull/2042
Can one of the admins verify this patch?
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on the issue:
https://github.com/apache/carbondata/pull/2042
SDV Build Success , Please check CI http://144.76.159.231:8080/job/ApacheSDVTests/3894/
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2042
Can one of the admins verify this patch?
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2042
Build Success with Spark 2.2.1, Please check CI http://88.99.58.216:8080/job/ApacheCarbonPRBuilder/3149/
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on the issue:
https://github.com/apache/carbondata/pull/2042
SDV Build Fail , Please check CI http://144.76.159.231:8080/job/ApacheSDVTests/3818/
---
[GitHub] carbondata pull request #2042: [CARBONDATA-2236]added sdv test cases for sta...
Posted by kunal642 <gi...@git.apache.org>.
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2042#discussion_r173367335
--- Diff: integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/StandardPartitionTestCase.scala ---
@@ -0,0 +1,436 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.cluster.sdv.generated
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util._
+import org.scalatest.BeforeAndAfterAll
+
+/**
+ * Test Class for partitionTestCase to verify all scenerios
+ */
+
+class StandardPartitionTestCase extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll = {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd HH:mm:ss")
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
+ }
+ //Verify exception if column in partitioned by is already specified in table schema
+
+ test("Standard-Partition_TC001", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (INTEGER_COLUMN1 int)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','LIST_INFO'='1,3')""")
--- End diff --
remove carbon partition syntax from all test cases
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2042
Build Success with Spark 2.2.1, Please check CI http://88.99.58.216:8080/job/ApacheCarbonPRBuilder/3025/
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on the issue:
https://github.com/apache/carbondata/pull/2042
SDV Build Success , Please check CI http://144.76.159.231:8080/job/ApacheSDVTests/3885/
---
[GitHub] carbondata pull request #2042: [CARBONDATA-2236]added sdv test cases for sta...
Posted by kunal642 <gi...@git.apache.org>.
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2042#discussion_r173367883
--- Diff: integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/StandardPartitionTestCase.scala ---
@@ -0,0 +1,436 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.cluster.sdv.generated
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util._
+import org.scalatest.BeforeAndAfterAll
+
+/**
+ * Test Class for partitionTestCase to verify all scenerios
+ */
+
+class StandardPartitionTestCase extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll = {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd HH:mm:ss")
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
+ }
+ //Verify exception if column in partitioned by is already specified in table schema
+
+ test("Standard-Partition_TC001", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
--- End diff --
move drop command outside intercept block from all tests
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2042
Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder1/4126/
---
[GitHub] carbondata pull request #2042: [CARBONDATA-2236]added sdv test cases for sta...
Posted by kunal642 <gi...@git.apache.org>.
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2042#discussion_r173369982
--- Diff: integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/StandardPartitionTestCase.scala ---
@@ -0,0 +1,436 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.cluster.sdv.generated
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util._
+import org.scalatest.BeforeAndAfterAll
+
+/**
+ * Test Class for partitionTestCase to verify all scenerios
+ */
+
+class StandardPartitionTestCase extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll = {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd HH:mm:ss")
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
+ }
+ //Verify exception if column in partitioned by is already specified in table schema
+
+ test("Standard-Partition_TC001", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (INTEGER_COLUMN1 int)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','LIST_INFO'='1,3')""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify table is created with Partition
+ test("Standard-Partition_TC002", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY (INTEGER_COLUMN1 int)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','LIST_INFO'='1,3')""")
+ val df = sql(s"""DESC uniqdata""")
+ assert(df.collect().reverse.head.get(0).toString.toUpperCase.contains("INTEGER_COLUMN1"))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify table is created with Partition with table comment
+
+ test("Standard-Partition_TC003",Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, decimalField DECIMAL(18,2), charField CHAR(5), floatField FLOAT ) COMMENT 'partition_table' PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ val df = sql(s"""DESC formatted partition_table""")
+ checkExistence(df, true, "partition_table")
+ }
+
+ //Verify WHEN partitioned by is not specified in the DDL, but partition type,number of partitions and list info are given
+ test("Standard-Partition_TC004", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','NUM_PARTITIONS'='3','LIST_INFO'='1,3')""")
+ assert(true)
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Creating a partition table, with ARRAY type partitioned column
+ test("Standard-Partition_TC005", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists partition_table_array""")
+ sql(s"""CREATE TABLE partition_table_array(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, stringField STRING, timestampField TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (complexData ARRAY<STRING>) STORED BY 'carbondata'""")
+ sql(s"""drop table if exists partition_table_array""")
+ }
+ }
+
+ //Verify exception while Creating a partition table with DICTIONARY_INCLUDE
+ test("Standard-Partition_TC006", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists partition_table_string""")
+ sql(s"""CREATE TABLE partition_table_string(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestampField TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT, complexData ARRAY<STRING> ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('DICTIONARY_INCLUDE'='stringField')""")
+ sql(s"""drop table if exists partition_table_string""")
+ }
+ }
+
+ //Creating a partition table with DICTIONARY_EXCLUDE
+ test("Standard-Partition_TC007", Include) {
+ sql(s"""drop table if exists partition_table_string""")
+ sql(s"""CREATE TABLE partition_table_string(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestampField TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT, complexData ARRAY<STRING> ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('DICTIONARY_EXCLUDE'='stringField')""")
+ sql(s"""drop table if exists partition_table_string""")
+ }
+
+ //Verify exception if datatype is not provided with partition column
+ test("Standard-Partition_TC008", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY (DOJ)STORED BY 'org.apache.carbondata.format'""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify exception if non existent file header is provided in partition
+ test("Standard-Partition_TC009", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY (DOJ timestamp)STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata OPTIONS('DELIMITER'=',','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOJ,,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify exception if PARTITION BY is empty
+ test("Standard-Partition_TC010", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY ()STORED BY 'org.apache.carbondata.format'""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=LOCAL_SORT
+ test("Standard-Partition_TC011", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='LOCAL_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=GLOBAL_SORT
+ test("Standard-Partition_TC012", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=BATCH_SORT
+ test("Standard-Partition_TC013", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='BATCH_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=NO_SORT
+ test("Standard-Partition_TC014", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='NO_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with Bad Records Action = FORCE
+ test("Standard-Partition_TC015", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='FORCE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""select count(*) from partition_table""").show()
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Verify Exception when Loading data into a partitioned table with Bad Records Action = FAIL
+ test("Standard-Partition_TC016", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='FAIL')""")
+ sql(s"""drop table if exists partition_table""")
+ }
+ }
+
+ //Loading data into a partitioned table with Bad Records Action = IGNORE
+ test("Standard-Partition_TC017", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='IGNORE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(0)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with Bad Records Action = REDIRECT
+ test("Standard-Partition_TC018", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='REDIRECT')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(0)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with SINGLE_PASS=TRUE
+ test("Standard-Partition_TC019", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','SINGLE_PASS'='TRUE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with SINGLE_PASS=FALSE
+ test("Standard-Partition_TC020", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','SINGLE_PASS'='FALSE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Verify load with Standard Partition
+ test("Standard-Partition_TC021", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select count(*) from uniqdata"""), Seq(Row(28)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify load with Standard Partition with limit 1
+ test("Standard-Partition_TC022", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select * from uniqdata limit 1"""),Seq(Row("CUST_NAME_00000","ACTIVE_EMUI_VERSION_00000",null,null,null,12345678901.0000000000,22345678901.0000000000,1.123456749E10,-1.123456749E10,1,null,1)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify load with Standard Partition with select partition column
+ test("Standard-Partition_TC023", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select CUST_ID from uniqdata limit 1"""), Seq(Row(1)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify table creation if 2 partition columns are provided
+ test("Standard-Partition_TC024", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (CUST_ID int,DOJ timestamp) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1',doj) OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ val df = sql(s"""DESC uniqdata""").collect()
+ assert(df(df.indexWhere(_.get(0).toString.contains("# Partition")) + 3).get(0).toString.contains("doj"))
+ assert(df(df.indexWhere(_.get(0).toString.contains("# Partition")) + 2).get(0).toString.contains("cust_id"))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify load with Standard Partition after compaction
+ test("Standard-Partition_TC025", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""alter table uniqdata compact 'minor'""")
+ checkAnswer(sql(s"""select count(*) from uniqdata"""), Seq(Row(84)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify join operation on Standard Partition
+ test("Standard-Partition_TC026", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""drop table if exists uniqdata1""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""CREATE TABLE uniqdata1 (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata1 partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select a.cust_id, b.cust_id from uniqdata a, uniqdata1 b where a.cust_id >= b.cust_id limit 1"""),Seq(Row(1,1)))
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""drop table if exists uniqdata1""")
+ }
+
+ //Verify exception if partition column is dropped
+ test("Standard-Partition_TC027", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""alter table uniqdata drop columns(CUST_ID)""")
+ sql(s"""drop table if exists uniqdata""")
+ }
+ }
+
+ //Verify INSERT operation on standard partition
+ test("Standard-Partition_TC028", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""insert into table uniqdata values ('a', '1','2015-07-01 00:00:00', 5678,7654,23.4, 55.6, 7654, 8765,33,'2015-07-01 00:00:00', 1)""")
+ sql(s"""insert into table uniqdata values ('a', '1','2015-07-01 00:00:00', 5678,7654,23.4, 55.6, 7654, 8765,33,'2015-07-01 00:00:00', 0)""")
+ checkAnswer(sql(s"""select count(*) from uniqdata"""), Seq(Row(2)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify INSERT INTO SELECT operation on standard partition
+ test("Standard-Partition_TC029", Include) {
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE_load""")
+ sql(s"""CREATE TABLE partition_table_load (shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table_load options ('BAD_RECORDS_ACTION'='FORCE')""")
+ sql(s"""INSERT into TABLE partition_table PARTITION (stringfield = 'Hello') SELECT * FROM partition_table_load au WHERE au.intfield = 25""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""),Seq(Row(2)))
+ sql(s"""drop table if exists PARTITION_TABLE""")
+ sql(s"""drop table if exists PARTITION_TABLE_load""")
+ }
+
+ //Verify INSERT overwrite operation on standard partition
+ test("Standard-Partition_TC030", Include) {
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE_load""")
+ sql(s"""CREATE TABLE partition_table_load (shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table_load options ('BAD_RECORDS_ACTION'='FORCE')""")
+ sql(s"""INSERT OVERWRITE TABLE partition_table PARTITION (stringfield = 'Hello') SELECT * FROM partition_table_load au WHERE au.intField = 25""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""),Seq(Row(2)))
+ sql(s"""drop table if exists PARTITION_TABLE""")
+ sql(s"""drop table if exists PARTITION_TABLE_load""")
+ }
+
+ //Verify date with > filter condition and standard partition
+ test("Standard-Partition_TC031", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='4') OPTIONS('DELIMITER'=',' , 'BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select count(*) from uniqdata where CUST_ID>3"""),Seq(Row(28)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify date with = filter condition and standard partition
+ test("Standard-Partition_TC032", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='4')OPTIONS('DELIMITER'=',' , 'BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select count(*) from uniqdata where CUST_ID=3"""),Seq(Row(0)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify date with = value and standard partition
+ test("Standard-Partition_TC033", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='4') OPTIONS('DELIMITER'=',' , 'BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select count(*) from uniqdata where CUST_ID=10"""),Seq(Row(0)))
+
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify describe partition_table on standard Partition
+ test("Standard-Partition_TC034", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ val df = sql(s"""DESC partition_table""")
+ assert(df.collect().reverse.head.get(0).toString.toUpperCase.contains("STRINGFIELD"))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Verify update partition_table on standard Partition
+ test("Standard-Partition_TC035", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table PARTITION (stringField = "Hello")""")
+ sql("""update partition_table set (stringfield)=('China') where stringfield = 'Hello'""")
+ checkAnswer(sql(s"""show partitions partition_table"""),Seq(Row("stringfield=Hello")))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Verify update partition_table on standard Partition
+ test("Standard-Partition_TC036", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table PARTITION (stringField = 'Hello')""")
+ sql("""update partition_table set (stringfield)=('China') where stringfield = 'Hello'""")
+ sql("""update partition_table set (stringfield)=('China123') where stringfield != 'China'""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""),Seq(Row(10)))
--- End diff --
check for the updated value instead of count(*)
---
[GitHub] carbondata pull request #2042: [CARBONDATA-2236]added sdv test cases for sta...
Posted by kunal642 <gi...@git.apache.org>.
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2042#discussion_r173369434
--- Diff: integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/StandardPartitionTestCase.scala ---
@@ -0,0 +1,436 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.cluster.sdv.generated
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util._
+import org.scalatest.BeforeAndAfterAll
+
+/**
+ * Test Class for partitionTestCase to verify all scenerios
+ */
+
+class StandardPartitionTestCase extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll = {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd HH:mm:ss")
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
+ }
+ //Verify exception if column in partitioned by is already specified in table schema
+
+ test("Standard-Partition_TC001", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (INTEGER_COLUMN1 int)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','LIST_INFO'='1,3')""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify table is created with Partition
+ test("Standard-Partition_TC002", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY (INTEGER_COLUMN1 int)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','LIST_INFO'='1,3')""")
+ val df = sql(s"""DESC uniqdata""")
+ assert(df.collect().reverse.head.get(0).toString.toUpperCase.contains("INTEGER_COLUMN1"))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify table is created with Partition with table comment
+
+ test("Standard-Partition_TC003",Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, decimalField DECIMAL(18,2), charField CHAR(5), floatField FLOAT ) COMMENT 'partition_table' PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ val df = sql(s"""DESC formatted partition_table""")
+ checkExistence(df, true, "partition_table")
+ }
+
+ //Verify WHEN partitioned by is not specified in the DDL, but partition type,number of partitions and list info are given
+ test("Standard-Partition_TC004", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','NUM_PARTITIONS'='3','LIST_INFO'='1,3')""")
+ assert(true)
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Creating a partition table, with ARRAY type partitioned column
+ test("Standard-Partition_TC005", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists partition_table_array""")
+ sql(s"""CREATE TABLE partition_table_array(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, stringField STRING, timestampField TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (complexData ARRAY<STRING>) STORED BY 'carbondata'""")
+ sql(s"""drop table if exists partition_table_array""")
+ }
+ }
+
+ //Verify exception while Creating a partition table with DICTIONARY_INCLUDE
+ test("Standard-Partition_TC006", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists partition_table_string""")
+ sql(s"""CREATE TABLE partition_table_string(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestampField TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT, complexData ARRAY<STRING> ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('DICTIONARY_INCLUDE'='stringField')""")
+ sql(s"""drop table if exists partition_table_string""")
+ }
+ }
+
+ //Creating a partition table with DICTIONARY_EXCLUDE
+ test("Standard-Partition_TC007", Include) {
+ sql(s"""drop table if exists partition_table_string""")
+ sql(s"""CREATE TABLE partition_table_string(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestampField TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT, complexData ARRAY<STRING> ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('DICTIONARY_EXCLUDE'='stringField')""")
+ sql(s"""drop table if exists partition_table_string""")
+ }
+
+ //Verify exception if datatype is not provided with partition column
+ test("Standard-Partition_TC008", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY (DOJ)STORED BY 'org.apache.carbondata.format'""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify exception if non existent file header is provided in partition
+ test("Standard-Partition_TC009", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY (DOJ timestamp)STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata OPTIONS('DELIMITER'=',','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOJ,,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify exception if PARTITION BY is empty
+ test("Standard-Partition_TC010", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY ()STORED BY 'org.apache.carbondata.format'""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=LOCAL_SORT
+ test("Standard-Partition_TC011", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='LOCAL_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=GLOBAL_SORT
+ test("Standard-Partition_TC012", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=BATCH_SORT
+ test("Standard-Partition_TC013", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='BATCH_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=NO_SORT
+ test("Standard-Partition_TC014", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='NO_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with Bad Records Action = FORCE
+ test("Standard-Partition_TC015", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='FORCE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""select count(*) from partition_table""").show()
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Verify Exception when Loading data into a partitioned table with Bad Records Action = FAIL
+ test("Standard-Partition_TC016", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='FAIL')""")
+ sql(s"""drop table if exists partition_table""")
+ }
+ }
+
+ //Loading data into a partitioned table with Bad Records Action = IGNORE
+ test("Standard-Partition_TC017", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='IGNORE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(0)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with Bad Records Action = REDIRECT
+ test("Standard-Partition_TC018", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='REDIRECT')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(0)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with SINGLE_PASS=TRUE
+ test("Standard-Partition_TC019", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','SINGLE_PASS'='TRUE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with SINGLE_PASS=FALSE
+ test("Standard-Partition_TC020", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','SINGLE_PASS'='FALSE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Verify load with Standard Partition
+ test("Standard-Partition_TC021", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select count(*) from uniqdata"""), Seq(Row(28)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify load with Standard Partition with limit 1
+ test("Standard-Partition_TC022", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select * from uniqdata limit 1"""),Seq(Row("CUST_NAME_00000","ACTIVE_EMUI_VERSION_00000",null,null,null,12345678901.0000000000,22345678901.0000000000,1.123456749E10,-1.123456749E10,1,null,1)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify load with Standard Partition with select partition column
+ test("Standard-Partition_TC023", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select CUST_ID from uniqdata limit 1"""), Seq(Row(1)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify table creation if 2 partition columns are provided
+ test("Standard-Partition_TC024", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (CUST_ID int,DOJ timestamp) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1',doj) OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ val df = sql(s"""DESC uniqdata""").collect()
+ assert(df(df.indexWhere(_.get(0).toString.contains("# Partition")) + 3).get(0).toString.contains("doj"))
+ assert(df(df.indexWhere(_.get(0).toString.contains("# Partition")) + 2).get(0).toString.contains("cust_id"))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify load with Standard Partition after compaction
+ test("Standard-Partition_TC025", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""alter table uniqdata compact 'minor'""")
+ checkAnswer(sql(s"""select count(*) from uniqdata"""), Seq(Row(84)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify join operation on Standard Partition
+ test("Standard-Partition_TC026", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""drop table if exists uniqdata1""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""CREATE TABLE uniqdata1 (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata1 partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select a.cust_id, b.cust_id from uniqdata a, uniqdata1 b where a.cust_id >= b.cust_id limit 1"""),Seq(Row(1,1)))
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""drop table if exists uniqdata1""")
+ }
+
+ //Verify exception if partition column is dropped
+ test("Standard-Partition_TC027", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""alter table uniqdata drop columns(CUST_ID)""")
+ sql(s"""drop table if exists uniqdata""")
+ }
+ }
+
+ //Verify INSERT operation on standard partition
+ test("Standard-Partition_TC028", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""insert into table uniqdata values ('a', '1','2015-07-01 00:00:00', 5678,7654,23.4, 55.6, 7654, 8765,33,'2015-07-01 00:00:00', 1)""")
+ sql(s"""insert into table uniqdata values ('a', '1','2015-07-01 00:00:00', 5678,7654,23.4, 55.6, 7654, 8765,33,'2015-07-01 00:00:00', 0)""")
+ checkAnswer(sql(s"""select count(*) from uniqdata"""), Seq(Row(2)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify INSERT INTO SELECT operation on standard partition
+ test("Standard-Partition_TC029", Include) {
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE_load""")
+ sql(s"""CREATE TABLE partition_table_load (shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table_load options ('BAD_RECORDS_ACTION'='FORCE')""")
+ sql(s"""INSERT into TABLE partition_table PARTITION (stringfield = 'Hello') SELECT * FROM partition_table_load au WHERE au.intfield = 25""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""),Seq(Row(2)))
+ sql(s"""drop table if exists PARTITION_TABLE""")
+ sql(s"""drop table if exists PARTITION_TABLE_load""")
+ }
+
+ //Verify INSERT overwrite operation on standard partition
+ test("Standard-Partition_TC030", Include) {
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE_load""")
+ sql(s"""CREATE TABLE partition_table_load (shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table_load options ('BAD_RECORDS_ACTION'='FORCE')""")
+ sql(s"""INSERT OVERWRITE TABLE partition_table PARTITION (stringfield = 'Hello') SELECT * FROM partition_table_load au WHERE au.intField = 25""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""),Seq(Row(2)))
+ sql(s"""drop table if exists PARTITION_TABLE""")
+ sql(s"""drop table if exists PARTITION_TABLE_load""")
+ }
+
+ //Verify date with > filter condition and standard partition
+ test("Standard-Partition_TC031", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='4') OPTIONS('DELIMITER'=',' , 'BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select count(*) from uniqdata where CUST_ID>3"""),Seq(Row(28)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify date with = filter condition and standard partition
+ test("Standard-Partition_TC032", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='4')OPTIONS('DELIMITER'=',' , 'BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select count(*) from uniqdata where CUST_ID=3"""),Seq(Row(0)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify date with = value and standard partition
+ test("Standard-Partition_TC033", Include) {
--- End diff --
same as TC032
---
[GitHub] carbondata pull request #2042: [CARBONDATA-2236]added sdv test cases for sta...
Posted by kunal642 <gi...@git.apache.org>.
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2042#discussion_r173369404
--- Diff: integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/StandardPartitionTestCase.scala ---
@@ -0,0 +1,436 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.cluster.sdv.generated
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util._
+import org.scalatest.BeforeAndAfterAll
+
+/**
+ * Test Class for partitionTestCase to verify all scenerios
+ */
+
+class StandardPartitionTestCase extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll = {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd HH:mm:ss")
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
+ }
+ //Verify exception if column in partitioned by is already specified in table schema
+
+ test("Standard-Partition_TC001", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (INTEGER_COLUMN1 int)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','LIST_INFO'='1,3')""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify table is created with Partition
+ test("Standard-Partition_TC002", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY (INTEGER_COLUMN1 int)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','LIST_INFO'='1,3')""")
+ val df = sql(s"""DESC uniqdata""")
+ assert(df.collect().reverse.head.get(0).toString.toUpperCase.contains("INTEGER_COLUMN1"))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify table is created with Partition with table comment
+
+ test("Standard-Partition_TC003",Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, decimalField DECIMAL(18,2), charField CHAR(5), floatField FLOAT ) COMMENT 'partition_table' PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ val df = sql(s"""DESC formatted partition_table""")
+ checkExistence(df, true, "partition_table")
+ }
+
+ //Verify WHEN partitioned by is not specified in the DDL, but partition type,number of partitions and list info are given
+ test("Standard-Partition_TC004", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','NUM_PARTITIONS'='3','LIST_INFO'='1,3')""")
+ assert(true)
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Creating a partition table, with ARRAY type partitioned column
+ test("Standard-Partition_TC005", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists partition_table_array""")
+ sql(s"""CREATE TABLE partition_table_array(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, stringField STRING, timestampField TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (complexData ARRAY<STRING>) STORED BY 'carbondata'""")
+ sql(s"""drop table if exists partition_table_array""")
+ }
+ }
+
+ //Verify exception while Creating a partition table with DICTIONARY_INCLUDE
+ test("Standard-Partition_TC006", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists partition_table_string""")
+ sql(s"""CREATE TABLE partition_table_string(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestampField TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT, complexData ARRAY<STRING> ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('DICTIONARY_INCLUDE'='stringField')""")
+ sql(s"""drop table if exists partition_table_string""")
+ }
+ }
+
+ //Creating a partition table with DICTIONARY_EXCLUDE
+ test("Standard-Partition_TC007", Include) {
+ sql(s"""drop table if exists partition_table_string""")
+ sql(s"""CREATE TABLE partition_table_string(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestampField TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT, complexData ARRAY<STRING> ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('DICTIONARY_EXCLUDE'='stringField')""")
+ sql(s"""drop table if exists partition_table_string""")
+ }
+
+ //Verify exception if datatype is not provided with partition column
+ test("Standard-Partition_TC008", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY (DOJ)STORED BY 'org.apache.carbondata.format'""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify exception if non existent file header is provided in partition
+ test("Standard-Partition_TC009", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY (DOJ timestamp)STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata OPTIONS('DELIMITER'=',','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOJ,,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify exception if PARTITION BY is empty
+ test("Standard-Partition_TC010", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY ()STORED BY 'org.apache.carbondata.format'""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=LOCAL_SORT
+ test("Standard-Partition_TC011", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='LOCAL_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=GLOBAL_SORT
+ test("Standard-Partition_TC012", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=BATCH_SORT
+ test("Standard-Partition_TC013", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='BATCH_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=NO_SORT
+ test("Standard-Partition_TC014", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='NO_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with Bad Records Action = FORCE
+ test("Standard-Partition_TC015", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='FORCE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""select count(*) from partition_table""").show()
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Verify Exception when Loading data into a partitioned table with Bad Records Action = FAIL
+ test("Standard-Partition_TC016", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='FAIL')""")
+ sql(s"""drop table if exists partition_table""")
+ }
+ }
+
+ //Loading data into a partitioned table with Bad Records Action = IGNORE
+ test("Standard-Partition_TC017", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='IGNORE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(0)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with Bad Records Action = REDIRECT
+ test("Standard-Partition_TC018", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='REDIRECT')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(0)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with SINGLE_PASS=TRUE
+ test("Standard-Partition_TC019", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','SINGLE_PASS'='TRUE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with SINGLE_PASS=FALSE
+ test("Standard-Partition_TC020", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','SINGLE_PASS'='FALSE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Verify load with Standard Partition
+ test("Standard-Partition_TC021", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select count(*) from uniqdata"""), Seq(Row(28)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify load with Standard Partition with limit 1
+ test("Standard-Partition_TC022", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select * from uniqdata limit 1"""),Seq(Row("CUST_NAME_00000","ACTIVE_EMUI_VERSION_00000",null,null,null,12345678901.0000000000,22345678901.0000000000,1.123456749E10,-1.123456749E10,1,null,1)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify load with Standard Partition with select partition column
+ test("Standard-Partition_TC023", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select CUST_ID from uniqdata limit 1"""), Seq(Row(1)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify table creation if 2 partition columns are provided
+ test("Standard-Partition_TC024", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (CUST_ID int,DOJ timestamp) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1',doj) OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ val df = sql(s"""DESC uniqdata""").collect()
+ assert(df(df.indexWhere(_.get(0).toString.contains("# Partition")) + 3).get(0).toString.contains("doj"))
+ assert(df(df.indexWhere(_.get(0).toString.contains("# Partition")) + 2).get(0).toString.contains("cust_id"))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify load with Standard Partition after compaction
+ test("Standard-Partition_TC025", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""alter table uniqdata compact 'minor'""")
+ checkAnswer(sql(s"""select count(*) from uniqdata"""), Seq(Row(84)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify join operation on Standard Partition
+ test("Standard-Partition_TC026", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""drop table if exists uniqdata1""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""CREATE TABLE uniqdata1 (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata1 partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select a.cust_id, b.cust_id from uniqdata a, uniqdata1 b where a.cust_id >= b.cust_id limit 1"""),Seq(Row(1,1)))
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""drop table if exists uniqdata1""")
+ }
+
+ //Verify exception if partition column is dropped
+ test("Standard-Partition_TC027", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""alter table uniqdata drop columns(CUST_ID)""")
+ sql(s"""drop table if exists uniqdata""")
+ }
+ }
+
+ //Verify INSERT operation on standard partition
+ test("Standard-Partition_TC028", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""insert into table uniqdata values ('a', '1','2015-07-01 00:00:00', 5678,7654,23.4, 55.6, 7654, 8765,33,'2015-07-01 00:00:00', 1)""")
+ sql(s"""insert into table uniqdata values ('a', '1','2015-07-01 00:00:00', 5678,7654,23.4, 55.6, 7654, 8765,33,'2015-07-01 00:00:00', 0)""")
+ checkAnswer(sql(s"""select count(*) from uniqdata"""), Seq(Row(2)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify INSERT INTO SELECT operation on standard partition
+ test("Standard-Partition_TC029", Include) {
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE_load""")
+ sql(s"""CREATE TABLE partition_table_load (shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table_load options ('BAD_RECORDS_ACTION'='FORCE')""")
+ sql(s"""INSERT into TABLE partition_table PARTITION (stringfield = 'Hello') SELECT * FROM partition_table_load au WHERE au.intfield = 25""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""),Seq(Row(2)))
+ sql(s"""drop table if exists PARTITION_TABLE""")
+ sql(s"""drop table if exists PARTITION_TABLE_load""")
+ }
+
+ //Verify INSERT overwrite operation on standard partition
+ test("Standard-Partition_TC030", Include) {
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE_load""")
+ sql(s"""CREATE TABLE partition_table_load (shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table_load options ('BAD_RECORDS_ACTION'='FORCE')""")
+ sql(s"""INSERT OVERWRITE TABLE partition_table PARTITION (stringfield = 'Hello') SELECT * FROM partition_table_load au WHERE au.intField = 25""")
--- End diff --
do one load into partition table before insert overwrite to check the functionality properly
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2042
Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder1/4352/
---
[GitHub] carbondata pull request #2042: [CARBONDATA-2236]added sdv test cases for sta...
Posted by asfgit <gi...@git.apache.org>.
Github user asfgit closed the pull request at:
https://github.com/apache/carbondata/pull/2042
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2042
Build Failed with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder1/4159/
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2042
Can one of the admins verify this patch?
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on the issue:
https://github.com/apache/carbondata/pull/2042
add to whitelist
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on the issue:
https://github.com/apache/carbondata/pull/2042
SDV Build Success , Please check CI http://144.76.159.231:8080/job/ApacheSDVTests/3903/
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2042
Build Failed with Spark 2.2.1, Please check CI http://88.99.58.216:8080/job/ApacheCarbonPRBuilder/3118/
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on the issue:
https://github.com/apache/carbondata/pull/2042
SDV Build Fail , Please check CI http://144.76.159.231:8080/job/ApacheSDVTests/3816/
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by praveenmeenakshi56 <gi...@git.apache.org>.
Github user praveenmeenakshi56 commented on the issue:
https://github.com/apache/carbondata/pull/2042
retest this please
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2042
Build Success with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder1/4381/
---
[GitHub] carbondata pull request #2042: [CARBONDATA-2236]added sdv test cases for sta...
Posted by kunal642 <gi...@git.apache.org>.
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2042#discussion_r173369924
--- Diff: integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/StandardPartitionTestCase.scala ---
@@ -0,0 +1,436 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.cluster.sdv.generated
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util._
+import org.scalatest.BeforeAndAfterAll
+
+/**
+ * Test Class for partitionTestCase to verify all scenerios
+ */
+
+class StandardPartitionTestCase extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll = {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd HH:mm:ss")
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
+ }
+ //Verify exception if column in partitioned by is already specified in table schema
+
+ test("Standard-Partition_TC001", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (INTEGER_COLUMN1 int)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','LIST_INFO'='1,3')""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify table is created with Partition
+ test("Standard-Partition_TC002", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY (INTEGER_COLUMN1 int)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','LIST_INFO'='1,3')""")
+ val df = sql(s"""DESC uniqdata""")
+ assert(df.collect().reverse.head.get(0).toString.toUpperCase.contains("INTEGER_COLUMN1"))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify table is created with Partition with table comment
+
+ test("Standard-Partition_TC003",Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, decimalField DECIMAL(18,2), charField CHAR(5), floatField FLOAT ) COMMENT 'partition_table' PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ val df = sql(s"""DESC formatted partition_table""")
+ checkExistence(df, true, "partition_table")
+ }
+
+ //Verify WHEN partitioned by is not specified in the DDL, but partition type,number of partitions and list info are given
+ test("Standard-Partition_TC004", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','NUM_PARTITIONS'='3','LIST_INFO'='1,3')""")
+ assert(true)
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Creating a partition table, with ARRAY type partitioned column
+ test("Standard-Partition_TC005", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists partition_table_array""")
+ sql(s"""CREATE TABLE partition_table_array(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, stringField STRING, timestampField TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (complexData ARRAY<STRING>) STORED BY 'carbondata'""")
+ sql(s"""drop table if exists partition_table_array""")
+ }
+ }
+
+ //Verify exception while Creating a partition table with DICTIONARY_INCLUDE
+ test("Standard-Partition_TC006", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists partition_table_string""")
+ sql(s"""CREATE TABLE partition_table_string(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestampField TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT, complexData ARRAY<STRING> ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('DICTIONARY_INCLUDE'='stringField')""")
+ sql(s"""drop table if exists partition_table_string""")
+ }
+ }
+
+ //Creating a partition table with DICTIONARY_EXCLUDE
+ test("Standard-Partition_TC007", Include) {
+ sql(s"""drop table if exists partition_table_string""")
+ sql(s"""CREATE TABLE partition_table_string(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestampField TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT, complexData ARRAY<STRING> ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('DICTIONARY_EXCLUDE'='stringField')""")
+ sql(s"""drop table if exists partition_table_string""")
+ }
+
+ //Verify exception if datatype is not provided with partition column
+ test("Standard-Partition_TC008", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY (DOJ)STORED BY 'org.apache.carbondata.format'""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify exception if non existent file header is provided in partition
+ test("Standard-Partition_TC009", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY (DOJ timestamp)STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata OPTIONS('DELIMITER'=',','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOJ,,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify exception if PARTITION BY is empty
+ test("Standard-Partition_TC010", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY ()STORED BY 'org.apache.carbondata.format'""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=LOCAL_SORT
+ test("Standard-Partition_TC011", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='LOCAL_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=GLOBAL_SORT
+ test("Standard-Partition_TC012", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=BATCH_SORT
+ test("Standard-Partition_TC013", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='BATCH_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=NO_SORT
+ test("Standard-Partition_TC014", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='NO_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with Bad Records Action = FORCE
+ test("Standard-Partition_TC015", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='FORCE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""select count(*) from partition_table""").show()
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Verify Exception when Loading data into a partitioned table with Bad Records Action = FAIL
+ test("Standard-Partition_TC016", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='FAIL')""")
+ sql(s"""drop table if exists partition_table""")
+ }
+ }
+
+ //Loading data into a partitioned table with Bad Records Action = IGNORE
+ test("Standard-Partition_TC017", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='IGNORE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(0)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with Bad Records Action = REDIRECT
+ test("Standard-Partition_TC018", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='REDIRECT')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(0)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with SINGLE_PASS=TRUE
+ test("Standard-Partition_TC019", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','SINGLE_PASS'='TRUE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with SINGLE_PASS=FALSE
+ test("Standard-Partition_TC020", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','SINGLE_PASS'='FALSE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Verify load with Standard Partition
+ test("Standard-Partition_TC021", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select count(*) from uniqdata"""), Seq(Row(28)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify load with Standard Partition with limit 1
+ test("Standard-Partition_TC022", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select * from uniqdata limit 1"""),Seq(Row("CUST_NAME_00000","ACTIVE_EMUI_VERSION_00000",null,null,null,12345678901.0000000000,22345678901.0000000000,1.123456749E10,-1.123456749E10,1,null,1)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify load with Standard Partition with select partition column
+ test("Standard-Partition_TC023", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select CUST_ID from uniqdata limit 1"""), Seq(Row(1)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify table creation if 2 partition columns are provided
+ test("Standard-Partition_TC024", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (CUST_ID int,DOJ timestamp) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1',doj) OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ val df = sql(s"""DESC uniqdata""").collect()
+ assert(df(df.indexWhere(_.get(0).toString.contains("# Partition")) + 3).get(0).toString.contains("doj"))
+ assert(df(df.indexWhere(_.get(0).toString.contains("# Partition")) + 2).get(0).toString.contains("cust_id"))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify load with Standard Partition after compaction
+ test("Standard-Partition_TC025", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""alter table uniqdata compact 'minor'""")
+ checkAnswer(sql(s"""select count(*) from uniqdata"""), Seq(Row(84)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify join operation on Standard Partition
+ test("Standard-Partition_TC026", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""drop table if exists uniqdata1""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""CREATE TABLE uniqdata1 (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata1 partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select a.cust_id, b.cust_id from uniqdata a, uniqdata1 b where a.cust_id >= b.cust_id limit 1"""),Seq(Row(1,1)))
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""drop table if exists uniqdata1""")
+ }
+
+ //Verify exception if partition column is dropped
+ test("Standard-Partition_TC027", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""alter table uniqdata drop columns(CUST_ID)""")
+ sql(s"""drop table if exists uniqdata""")
+ }
+ }
+
+ //Verify INSERT operation on standard partition
+ test("Standard-Partition_TC028", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""insert into table uniqdata values ('a', '1','2015-07-01 00:00:00', 5678,7654,23.4, 55.6, 7654, 8765,33,'2015-07-01 00:00:00', 1)""")
+ sql(s"""insert into table uniqdata values ('a', '1','2015-07-01 00:00:00', 5678,7654,23.4, 55.6, 7654, 8765,33,'2015-07-01 00:00:00', 0)""")
+ checkAnswer(sql(s"""select count(*) from uniqdata"""), Seq(Row(2)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify INSERT INTO SELECT operation on standard partition
+ test("Standard-Partition_TC029", Include) {
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE_load""")
+ sql(s"""CREATE TABLE partition_table_load (shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table_load options ('BAD_RECORDS_ACTION'='FORCE')""")
+ sql(s"""INSERT into TABLE partition_table PARTITION (stringfield = 'Hello') SELECT * FROM partition_table_load au WHERE au.intfield = 25""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""),Seq(Row(2)))
+ sql(s"""drop table if exists PARTITION_TABLE""")
+ sql(s"""drop table if exists PARTITION_TABLE_load""")
+ }
+
+ //Verify INSERT overwrite operation on standard partition
+ test("Standard-Partition_TC030", Include) {
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE_load""")
+ sql(s"""CREATE TABLE partition_table_load (shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table_load options ('BAD_RECORDS_ACTION'='FORCE')""")
+ sql(s"""INSERT OVERWRITE TABLE partition_table PARTITION (stringfield = 'Hello') SELECT * FROM partition_table_load au WHERE au.intField = 25""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""),Seq(Row(2)))
+ sql(s"""drop table if exists PARTITION_TABLE""")
+ sql(s"""drop table if exists PARTITION_TABLE_load""")
+ }
+
+ //Verify date with > filter condition and standard partition
+ test("Standard-Partition_TC031", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='4') OPTIONS('DELIMITER'=',' , 'BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select count(*) from uniqdata where CUST_ID>3"""),Seq(Row(28)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify date with = filter condition and standard partition
+ test("Standard-Partition_TC032", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='4')OPTIONS('DELIMITER'=',' , 'BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select count(*) from uniqdata where CUST_ID=3"""),Seq(Row(0)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify date with = value and standard partition
+ test("Standard-Partition_TC033", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='4') OPTIONS('DELIMITER'=',' , 'BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select count(*) from uniqdata where CUST_ID=10"""),Seq(Row(0)))
+
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify describe partition_table on standard Partition
+ test("Standard-Partition_TC034", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ val df = sql(s"""DESC partition_table""")
+ assert(df.collect().reverse.head.get(0).toString.toUpperCase.contains("STRINGFIELD"))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Verify update partition_table on standard Partition
+ test("Standard-Partition_TC035", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table PARTITION (stringField = "Hello")""")
+ sql("""update partition_table set (stringfield)=('China') where stringfield = 'Hello'""")
+ checkAnswer(sql(s"""show partitions partition_table"""),Seq(Row("stringfield=Hello")))
--- End diff --
check for the updated value instead of show partition
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2042
Build Failed with Spark 2.1.0, Please check CI http://136.243.101.176:8080/job/ApacheCarbonPRBuilder1/4160/
---
[GitHub] carbondata pull request #2042: [CARBONDATA-2236]added sdv test cases for sta...
Posted by kunal642 <gi...@git.apache.org>.
Github user kunal642 commented on a diff in the pull request:
https://github.com/apache/carbondata/pull/2042#discussion_r173369580
--- Diff: integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/StandardPartitionTestCase.scala ---
@@ -0,0 +1,436 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.cluster.sdv.generated
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util._
+import org.scalatest.BeforeAndAfterAll
+
+/**
+ * Test Class for partitionTestCase to verify all scenerios
+ */
+
+class StandardPartitionTestCase extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll = {
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd HH:mm:ss")
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
+ }
+ //Verify exception if column in partitioned by is already specified in table schema
+
+ test("Standard-Partition_TC001", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (INTEGER_COLUMN1 int)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','LIST_INFO'='1,3')""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify table is created with Partition
+ test("Standard-Partition_TC002", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY (INTEGER_COLUMN1 int)STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','LIST_INFO'='1,3')""")
+ val df = sql(s"""DESC uniqdata""")
+ assert(df.collect().reverse.head.get(0).toString.toUpperCase.contains("INTEGER_COLUMN1"))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify table is created with Partition with table comment
+
+ test("Standard-Partition_TC003",Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, decimalField DECIMAL(18,2), charField CHAR(5), floatField FLOAT ) COMMENT 'partition_table' PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ val df = sql(s"""DESC formatted partition_table""")
+ checkExistence(df, true, "partition_table")
+ }
+
+ //Verify WHEN partitioned by is not specified in the DDL, but partition type,number of partitions and list info are given
+ test("Standard-Partition_TC004", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('PARTITION_TYPE'='List','NUM_PARTITIONS'='3','LIST_INFO'='1,3')""")
+ assert(true)
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Creating a partition table, with ARRAY type partitioned column
+ test("Standard-Partition_TC005", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists partition_table_array""")
+ sql(s"""CREATE TABLE partition_table_array(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, stringField STRING, timestampField TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (complexData ARRAY<STRING>) STORED BY 'carbondata'""")
+ sql(s"""drop table if exists partition_table_array""")
+ }
+ }
+
+ //Verify exception while Creating a partition table with DICTIONARY_INCLUDE
+ test("Standard-Partition_TC006", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists partition_table_string""")
+ sql(s"""CREATE TABLE partition_table_string(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestampField TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT, complexData ARRAY<STRING> ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('DICTIONARY_INCLUDE'='stringField')""")
+ sql(s"""drop table if exists partition_table_string""")
+ }
+ }
+
+ //Creating a partition table with DICTIONARY_EXCLUDE
+ test("Standard-Partition_TC007", Include) {
+ sql(s"""drop table if exists partition_table_string""")
+ sql(s"""CREATE TABLE partition_table_string(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestampField TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT, complexData ARRAY<STRING> ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('DICTIONARY_EXCLUDE'='stringField')""")
+ sql(s"""drop table if exists partition_table_string""")
+ }
+
+ //Verify exception if datatype is not provided with partition column
+ test("Standard-Partition_TC008", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY (DOJ)STORED BY 'org.apache.carbondata.format'""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify exception if non existent file header is provided in partition
+ test("Standard-Partition_TC009", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY (DOJ timestamp)STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata OPTIONS('DELIMITER'=',','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOJ,,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify exception if PARTITION BY is empty
+ test("Standard-Partition_TC010", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double) PARTITIONED BY ()STORED BY 'org.apache.carbondata.format'""")
+ }
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=LOCAL_SORT
+ test("Standard-Partition_TC011", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='LOCAL_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=GLOBAL_SORT
+ test("Standard-Partition_TC012", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='GLOBAL_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=BATCH_SORT
+ test("Standard-Partition_TC013", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='BATCH_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into partitioned table with SORT_SCOPE=NO_SORT
+ test("Standard-Partition_TC014", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata' TBLPROPERTIES('SORT_SCOPE'='NO_SORT')""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with Bad Records Action = FORCE
+ test("Standard-Partition_TC015", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='FORCE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""select count(*) from partition_table""").show()
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Verify Exception when Loading data into a partitioned table with Bad Records Action = FAIL
+ test("Standard-Partition_TC016", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='FAIL')""")
+ sql(s"""drop table if exists partition_table""")
+ }
+ }
+
+ //Loading data into a partitioned table with Bad Records Action = IGNORE
+ test("Standard-Partition_TC017", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='IGNORE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(0)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with Bad Records Action = REDIRECT
+ test("Standard-Partition_TC018", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','BAD_RECORDS_ACTION'='REDIRECT')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(0)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with SINGLE_PASS=TRUE
+ test("Standard-Partition_TC019", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','SINGLE_PASS'='TRUE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Loading data into a partitioned table with SINGLE_PASS=FALSE
+ test("Standard-Partition_TC020", Include) {
+ sql(s"""drop table if exists partition_table""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2),dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table options('FILEHEADER'='shortfield,intfield,bigintfield,doublefield,stringfield,timestamp,decimalfield,datefield,charfield,floatfield','SINGLE_PASS'='FALSE')""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""), Seq(Row(11)))
+ sql(s"""drop table if exists partition_table""")
+ }
+
+ //Verify load with Standard Partition
+ test("Standard-Partition_TC021", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select count(*) from uniqdata"""), Seq(Row(28)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify load with Standard Partition with limit 1
+ test("Standard-Partition_TC022", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select * from uniqdata limit 1"""),Seq(Row("CUST_NAME_00000","ACTIVE_EMUI_VERSION_00000",null,null,null,12345678901.0000000000,22345678901.0000000000,1.123456749E10,-1.123456749E10,1,null,1)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify load with Standard Partition with select partition column
+ test("Standard-Partition_TC023", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select CUST_ID from uniqdata limit 1"""), Seq(Row(1)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify table creation if 2 partition columns are provided
+ test("Standard-Partition_TC024", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) PARTITIONED BY (CUST_ID int,DOJ timestamp) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1',doj) OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ val df = sql(s"""DESC uniqdata""").collect()
+ assert(df(df.indexWhere(_.get(0).toString.contains("# Partition")) + 3).get(0).toString.contains("doj"))
+ assert(df(df.indexWhere(_.get(0).toString.contains("# Partition")) + 2).get(0).toString.contains("cust_id"))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify load with Standard Partition after compaction
+ test("Standard-Partition_TC025", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""alter table uniqdata compact 'minor'""")
+ checkAnswer(sql(s"""select count(*) from uniqdata"""), Seq(Row(84)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify join operation on Standard Partition
+ test("Standard-Partition_TC026", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""drop table if exists uniqdata1""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""CREATE TABLE uniqdata1 (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata1 partition(CUST_ID='1') OPTIONS('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select a.cust_id, b.cust_id from uniqdata a, uniqdata1 b where a.cust_id >= b.cust_id limit 1"""),Seq(Row(1,1)))
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""drop table if exists uniqdata1""")
+ }
+
+ //Verify exception if partition column is dropped
+ test("Standard-Partition_TC027", Include) {
+ intercept[Exception] {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""alter table uniqdata drop columns(CUST_ID)""")
+ sql(s"""drop table if exists uniqdata""")
+ }
+ }
+
+ //Verify INSERT operation on standard partition
+ test("Standard-Partition_TC028", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""insert into table uniqdata values ('a', '1','2015-07-01 00:00:00', 5678,7654,23.4, 55.6, 7654, 8765,33,'2015-07-01 00:00:00', 1)""")
+ sql(s"""insert into table uniqdata values ('a', '1','2015-07-01 00:00:00', 5678,7654,23.4, 55.6, 7654, 8765,33,'2015-07-01 00:00:00', 0)""")
+ checkAnswer(sql(s"""select count(*) from uniqdata"""), Seq(Row(2)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify INSERT INTO SELECT operation on standard partition
+ test("Standard-Partition_TC029", Include) {
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE_load""")
+ sql(s"""CREATE TABLE partition_table_load (shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table_load options ('BAD_RECORDS_ACTION'='FORCE')""")
+ sql(s"""INSERT into TABLE partition_table PARTITION (stringfield = 'Hello') SELECT * FROM partition_table_load au WHERE au.intfield = 25""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""),Seq(Row(2)))
+ sql(s"""drop table if exists PARTITION_TABLE""")
+ sql(s"""drop table if exists PARTITION_TABLE_load""")
+ }
+
+ //Verify INSERT overwrite operation on standard partition
+ test("Standard-Partition_TC030", Include) {
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE""")
+ sql(s"""CREATE TABLE partition_table(shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) PARTITIONED BY (stringField STRING) STORED BY 'carbondata'""")
+ sql(s"""DROP TABLE IF EXISTS PARTITION_TABLE_load""")
+ sql(s"""CREATE TABLE partition_table_load (shortField SHORT, intField INT, bigintField LONG, doubleField DOUBLE, timestamp TIMESTAMP, decimalField DECIMAL(18,2), dateField DATE, charField CHAR(5), floatField FLOAT ) STORED BY 'carbondata'""")
+ sql(s"""load data inpath '$resourcesPath/Data/partition/list_partition_table.csv' into table partition_table_load options ('BAD_RECORDS_ACTION'='FORCE')""")
+ sql(s"""INSERT OVERWRITE TABLE partition_table PARTITION (stringfield = 'Hello') SELECT * FROM partition_table_load au WHERE au.intField = 25""")
+ checkAnswer(sql(s"""select count(*) from partition_table"""),Seq(Row(2)))
+ sql(s"""drop table if exists PARTITION_TABLE""")
+ sql(s"""drop table if exists PARTITION_TABLE_load""")
+ }
+
+ //Verify date with > filter condition and standard partition
+ test("Standard-Partition_TC031", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='4') OPTIONS('DELIMITER'=',' , 'BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select count(*) from uniqdata where CUST_ID>3"""),Seq(Row(28)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify date with = filter condition and standard partition
+ test("Standard-Partition_TC032", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='4')OPTIONS('DELIMITER'=',' , 'BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select count(*) from uniqdata where CUST_ID=3"""),Seq(Row(0)))
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify date with = value and standard partition
+ test("Standard-Partition_TC033", Include) {
+ sql(s"""drop table if exists uniqdata""")
+ sql(s"""CREATE TABLE uniqdata (CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int, DOJ timestamp) PARTITIONED BY (CUST_ID int) STORED BY 'org.apache.carbondata.format'""")
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/partition/2000_UniqData_partition.csv' into table uniqdata partition(CUST_ID='4') OPTIONS('DELIMITER'=',' , 'BAD_RECORDS_ACTION'='FORCE','QUOTECHAR'='"','FILEHEADER'='CUST_NAME,ACTIVE_EMUI_VERSION,DOB,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1,DOJ,CUST_ID')""")
+ checkAnswer(sql(s"""select count(*) from uniqdata where CUST_ID=10"""),Seq(Row(0)))
+
+ sql(s"""drop table if exists uniqdata""")
+ }
+
+ //Verify describe partition_table on standard Partition
+ test("Standard-Partition_TC034", Include) {
--- End diff --
same as TC002
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by ravipesala <gi...@git.apache.org>.
Github user ravipesala commented on the issue:
https://github.com/apache/carbondata/pull/2042
SDV Build Success , Please check CI http://144.76.159.231:8080/job/ApacheSDVTests/3785/
---
[GitHub] carbondata issue #2042: [CARBONDATA-2236]added sdv test cases for standard p...
Posted by CarbonDataQA <gi...@git.apache.org>.
Github user CarbonDataQA commented on the issue:
https://github.com/apache/carbondata/pull/2042
Build Failed with Spark 2.2.1, Please check CI http://88.99.58.216:8080/job/ApacheCarbonPRBuilder/2999/
---