You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2018/12/19 11:15:16 UTC
carbondata git commit: [CARBONDATA-3182] Fixed SDV Testcase failures
Repository: carbondata
Updated Branches:
refs/heads/master 96cc1a6ab -> c0d858139
[CARBONDATA-3182] Fixed SDV Testcase failures
SDV Testcases were failing because Delimiter and Complex Delimiter was same.
So changed the Complex Delimiter in Load Option.
This closes #3002
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c0d85813
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c0d85813
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c0d85813
Branch: refs/heads/master
Commit: c0d8581392d6827927616f0a8e164d091b1a72bd
Parents: 96cc1a6
Author: manishnalla1994 <ma...@gmail.com>
Authored: Wed Dec 19 14:15:17 2018 +0530
Committer: kunal642 <ku...@gmail.com>
Committed: Wed Dec 19 16:44:09 2018 +0530
----------------------------------------------------------------------
.../sdv/generated/SinglepassTestCase.scala | 32 ++++++++++----------
1 file changed, 16 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0d85813/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/SinglepassTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/SinglepassTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/SinglepassTestCase.scala
index c57bd04..6487696 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/SinglepassTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/SinglepassTestCase.scala
@@ -173,7 +173,7 @@ class SinglepassTestCase extends QueryTest with BeforeAndAfterAll {
test("Loading-004-01-01-01_001-TC_015", Include) {
sql(s"""CREATE TABLE if not exists uniqdata_file_extn (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'""").collect
- sql(s"""LOAD DATA inpath '$resourcesPath/Data/singlepass/2000_UniqData.xls' into table uniqdata_file_extn OPTIONS('DELIMITER'='\001' , 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
+ sql(s"""LOAD DATA inpath '$resourcesPath/Data/singlepass/2000_UniqData.xls' into table uniqdata_file_extn OPTIONS('DELIMITER'='\001' , 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE','COMPLEX_DELIMITER_LEVEL_1'='#', 'COMPLEX_DELIMITER_LEVEL_2'=':','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
checkAnswer(s"""select count(*) from uniqdata_file_extn""",
Seq(Row(2013)), "singlepassTestCase_Loading-004-01-01-01_001-TC_015")
sql(s"""drop table uniqdata_file_extn""").collect
@@ -227,7 +227,7 @@ class SinglepassTestCase extends QueryTest with BeforeAndAfterAll {
//To check Data loading in proper CSV format with .xls with single_pass=false
test("Loading-004-01-01-01_001-TC_022", Include) {
sql(s"""CREATE TABLE if not exists uniqdata_file_extn (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'""").collect
- sql(s"""LOAD DATA inpath '$resourcesPath/Data/singlepass/2000_UniqData.xls' into table uniqdata_file_extn OPTIONS('DELIMITER'='\001' , 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
+ sql(s"""LOAD DATA inpath '$resourcesPath/Data/singlepass/2000_UniqData.xls' into table uniqdata_file_extn OPTIONS('DELIMITER'='\001' , 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1','COMPLEX_DELIMITER_LEVEL_1'='#', 'COMPLEX_DELIMITER_LEVEL_2'=':')""").collect
checkAnswer(s"""select count(*) from uniqdata_file_extn""",
Seq(Row(2013)), "singlepassTestCase_Loading-004-01-01-01_001-TC_022")
sql(s"""drop table uniqdata_file_extn""").collect
@@ -325,7 +325,7 @@ class SinglepassTestCase extends QueryTest with BeforeAndAfterAll {
test("Loading-004-01-01-01_001-TC_034", Include) {
sql(s"""drop table if exists uniqdata_001""").collect
sql(s"""CREATE TABLE if not exists uniqdata_001 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_INCLUDE'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData_001.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData_001.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1','COMPLEX_DELIMITER_LEVEL_1'='#', 'COMPLEX_DELIMITER_LEVEL_2'=':')""").collect
checkAnswer(s"""select count(*) from uniqdata_001""",
Seq(Row(2013)), "singlepassTestCase_Loading-004-01-01-01_001-TC_034")
sql(s"""drop table uniqdata_001""").collect
@@ -401,7 +401,7 @@ class SinglepassTestCase extends QueryTest with BeforeAndAfterAll {
//To check Data loading with delimiters as \001 and SINGLE_PASS= FALSE
test("Loading-004-01-01-01_001-TC_041", Include) {
sql(s"""CREATE TABLE if not exists uniqdata_001 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_INCLUDE'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData_001.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData_001.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1','COMPLEX_DELIMITER_LEVEL_1'='#', 'COMPLEX_DELIMITER_LEVEL_2'=':')""").collect
checkAnswer(s"""select count(*) from uniqdata_001""",
Seq(Row(2013)), "singlepassTestCase_Loading-004-01-01-01_001-TC_041")
sql(s"""drop table uniqdata_001""").collect
@@ -412,9 +412,9 @@ class SinglepassTestCase extends QueryTest with BeforeAndAfterAll {
test("Loading-004-01-01-01_001-TC_043", Include) {
sql(s"""drop table if exists uniqdata_001""").collect
sql(s"""CREATE TABLE if not exists uniqdata_001 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_INCLUDE'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1','COMPLEX_DELIMITER_LEVEL_1'='#', 'COMPLEX_DELIMITER_LEVEL_2'=':')""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1','COMPLEX_DELIMITER_LEVEL_1'='#', 'COMPLEX_DELIMITER_LEVEL_2'=':')""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1','COMPLEX_DELIMITER_LEVEL_1'='#', 'COMPLEX_DELIMITER_LEVEL_2'=':')""").collect
sql(s"""alter table uniqdata_001 compact 'minor'""").collect
sql(s"""show segments for table uniqdata_001""").collect
sql(s"""drop table uniqdata_001""").collect
@@ -424,9 +424,9 @@ class SinglepassTestCase extends QueryTest with BeforeAndAfterAll {
//To check Auto compaction is successful with carbon.enable.auto.load.merge= True & SINGLE_PASS=FALSE
test("Loading-004-01-01-01_001-TC_044", Include) {
sql(s"""CREATE TABLE if not exists uniqdata_001 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_INCLUDE'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1','COMPLEX_DELIMITER_LEVEL_1'='#', 'COMPLEX_DELIMITER_LEVEL_2'=':')""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1','COMPLEX_DELIMITER_LEVEL_1'='#', 'COMPLEX_DELIMITER_LEVEL_2'=':')""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1','COMPLEX_DELIMITER_LEVEL_1'='#', 'COMPLEX_DELIMITER_LEVEL_2'=':')""").collect
sql(s"""alter table uniqdata_001 compact 'minor'""").collect
sql(s"""show segments for table uniqdata_001""").collect
sql(s"""drop table uniqdata_001""").collect
@@ -437,11 +437,11 @@ class SinglepassTestCase extends QueryTest with BeforeAndAfterAll {
test("Loading-004-01-01-01_001-TC_045", Include) {
sql(s"""CREATE TABLE if not exists uniqdata_001 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_INCLUDE'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1','COMPLEX_DELIMITER_LEVEL_1'='#', 'COMPLEX_DELIMITER_LEVEL_2'=':')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='TRUE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1','COMPLEX_DELIMITER_LEVEL_1'='#', 'COMPLEX_DELIMITER_LEVEL_2'=':')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1','COMPLEX_DELIMITER_LEVEL_1'='#', 'COMPLEX_DELIMITER_LEVEL_2'=':')""").collect
sql(s"""alter table uniqdata_001 compact 'major'""").collect
sql(s"""show segments for table uniqdata_001""").collect
@@ -453,11 +453,11 @@ class SinglepassTestCase extends QueryTest with BeforeAndAfterAll {
test("Loading-004-01-01-01_001-TC_046", Include) {
sql(s"""CREATE TABLE if not exists uniqdata_001 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_INCLUDE'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1','COMPLEX_DELIMITER_LEVEL_1'='#', 'COMPLEX_DELIMITER_LEVEL_2'=':')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1','COMPLEX_DELIMITER_LEVEL_1'='#', 'COMPLEX_DELIMITER_LEVEL_2'=':')""").collect
- sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect
+ sql(s"""LOAD DATA INPATH '$resourcesPath/Data/singlepass/2000_UniqData.csv' into table uniqdata_001 OPTIONS('DELIMITER'="\001" , 'QUOTECHAR'='"', 'BAD_RECORDS_ACTION'='FORCE','SINGLE_PASS'='FALSE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1','COMPLEX_DELIMITER_LEVEL_1'='#', 'COMPLEX_DELIMITER_LEVEL_2'=':')""").collect
sql(s"""alter table uniqdata_001 compact 'major'""").collect
sql(s"""show segments for table uniqdata_001""").collect