You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/01/07 16:37:08 UTC
[34/38] incubator-carbondata git commit: reuse test case for
integration module
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/IUD/T_Hive1.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/IUD/T_Hive1.csv b/integration/spark-common-test/src/test/resources/IUD/T_Hive1.csv
new file mode 100644
index 0000000..418ae5c
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/IUD/T_Hive1.csv
@@ -0,0 +1,10 @@
+TRUE,1,450,304034400,200000343430000000, 121.5,4.99,2.44,SE3423ee,asfdsffdfg,ERTETRWT,2012-01-11 03:04:05.123456719,2012-01-20
+TRUE,2,423,3046340,200000000003454300, 121.5,4.99,2.44,SE3423ee,asfdsffdfg,EtryTRWT,2012-01-12 03:14:05.123456729,2012-01-20
+TRUE,3,453,3003445,200000000000003450, 121.5,4.99,2.44,SE3423ee,asfdsffdfg,ERTEerWT,2012-01-13 03:24:05.123456739,2012-01-20
+TRUE,4,4350,3044364,200000000000000000, 121.5,4.99,2.44,SE3423ee,asfdsffdfg,ERTtryWT,2012-01-14 23:04:05.123456749,2012-01-20
+TRUE,114,4520,30000430,200000000004300000, 121.5,4.99,2.44,RE3423ee,asfdsffdfg,4RTETRWT,2012-01-01 23:04:05.123456819,2012-01-20
+FALSE,123,454,30000040,200000000000000000, 121.5,4.99,2.44,RE3423ee,asfrewerfg,6RTETRWT,2012-01-02 23:04:05.123456829,2012-01-20
+TRUE,11,4530,3000040,200000000000000000, 121.5,4.99,2.44,SE3423ee,asfdsffder,TRTETRWT,2012-01-03 05:04:05.123456839,2012-01-20
+TRUE,14,4590,3000400,200000000000000000, 121.5,4.99,2.44,ASD423ee,asfertfdfg,HRTETRWT,2012-01-04 05:04:05.123456849,2012-01-20
+FALSE,41,4250,00000,200000000000000000, 121.5,4.99,2.44,SAD423ee,asrtsffdfg,HRTETRWT,2012-01-05 05:04:05.123456859,2012-01-20
+TRUE,13,4510,30400,200000000000000000, 121.5,4.99,2.44,DE3423ee,asfrtffdfg,YHTETRWT,2012-01-06 06:04:05.123456869,2012-01-20
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/IUD/comp1.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/IUD/comp1.csv b/integration/spark-common-test/src/test/resources/IUD/comp1.csv
new file mode 100644
index 0000000..9738e06
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/IUD/comp1.csv
@@ -0,0 +1,11 @@
+c1,c2,c3,c5
+a,1,aa,aaa
+b,2,bb,bbb
+c,3,cc,ccc
+d,4,dd,ddd
+e,5,ee,eee
+f,6,ff,fff
+g,7,gg,ggg
+h,8,hh,hhh
+i,9,ii,iii
+j,10,jj,jjj
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/IUD/comp2.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/IUD/comp2.csv b/integration/spark-common-test/src/test/resources/IUD/comp2.csv
new file mode 100644
index 0000000..5a28d5c
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/IUD/comp2.csv
@@ -0,0 +1,11 @@
+c1,c2,c3,c5
+a,11,aa,aaa
+b,12,bb,bbb
+c,13,cc,ccc
+d,14,dd,ddd
+e,15,ee,eee
+f,16,ff,fff
+g,17,gg,ggg
+h,18,hh,hhh
+i,19,ii,iii
+j,20,jj,jjj
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/IUD/comp3.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/IUD/comp3.csv b/integration/spark-common-test/src/test/resources/IUD/comp3.csv
new file mode 100644
index 0000000..a555f71
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/IUD/comp3.csv
@@ -0,0 +1,11 @@
+c1,c2,c3,c5
+a,21,aa,aaa
+b,22,bb,bbb
+c,23,cc,ccc
+d,24,dd,ddd
+e,25,ee,eee
+f,26,ff,fff
+g,27,gg,ggg
+h,28,hh,hhh
+i,29,ii,iii
+j,30,jj,jjj
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/IUD/comp4.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/IUD/comp4.csv b/integration/spark-common-test/src/test/resources/IUD/comp4.csv
new file mode 100644
index 0000000..0450a19
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/IUD/comp4.csv
@@ -0,0 +1,11 @@
+c1,c2,c3,c5
+a,31,aa,aaa
+b,32,bb,bbb
+c,33,cc,ccc
+d,34,dd,ddd
+e,35,ee,eee
+f,36,ff,fff
+g,37,gg,ggg
+h,38,hh,hhh
+i,39,ii,iii
+j,40,jj,jjj
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/IUD/dest.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/IUD/dest.csv b/integration/spark-common-test/src/test/resources/IUD/dest.csv
new file mode 100644
index 0000000..4ef9aa5
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/IUD/dest.csv
@@ -0,0 +1,6 @@
+c1,c2,c3,c5
+a,1,aa,aaa
+b,2,bb,bbb
+c,3,cc,ccc
+d,4,dd,ddd
+e,5,ee,eee
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/IUD/other.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/IUD/other.csv b/integration/spark-common-test/src/test/resources/IUD/other.csv
new file mode 100644
index 0000000..69a0e1c
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/IUD/other.csv
@@ -0,0 +1,3 @@
+c1,c2,c3,c5
+a,1,MGM,Disco
+b,2,RGK,Music
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/IUD/sample.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/IUD/sample.csv b/integration/spark-common-test/src/test/resources/IUD/sample.csv
new file mode 100644
index 0000000..d364136
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/IUD/sample.csv
@@ -0,0 +1,4 @@
+ID,country,name,phonetype,serialname,salary
+4,china,ravz,jio,ASD66902,15003
+1,india,ravi,airtel,ASD90633,15004
+6,usa,manu,alkatel,ASD59961,15005
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/IUD/sample_updated.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/IUD/sample_updated.csv b/integration/spark-common-test/src/test/resources/IUD/sample_updated.csv
new file mode 100644
index 0000000..b0cf165
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/IUD/sample_updated.csv
@@ -0,0 +1,2 @@
+part0/segment_0/part-0-0-1475753917000.carbondata/0/0,4,china,ravz,Aircel,ASD66902,200000
+part0/segment_0/part-0-0-1475753917000.carbondata/0/2,6,usa,manu,Aircel,ASD59961,15005
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/IUD/source2.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/IUD/source2.csv b/integration/spark-common-test/src/test/resources/IUD/source2.csv
new file mode 100644
index 0000000..cc86bdb
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/IUD/source2.csv
@@ -0,0 +1,3 @@
+c11,c22,c33,c55,c66
+a,1,MGM,Disco,10
+b,2,RGK,Music,8
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/IUD/source3.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/IUD/source3.csv b/integration/spark-common-test/src/test/resources/IUD/source3.csv
new file mode 100644
index 0000000..3f92816
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/IUD/source3.csv
@@ -0,0 +1,7 @@
+c11,c22,c33,c55,c66
+a,1,MGM,Disco,10
+b,2,RGK,Music,8
+d,4,YDY,Weather,9
+e,5,ZAZ,Election,11
+g,7,YTY,Hello,12
+h,8,TBT,Yeh,13
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/IUD/update01.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/IUD/update01.csv b/integration/spark-common-test/src/test/resources/IUD/update01.csv
new file mode 100644
index 0000000..35596cb
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/IUD/update01.csv
@@ -0,0 +1,6 @@
+imei,age,task,num,level,name
+imei0,2147,9279,100.05,100.055,fegt
+imei1,-2148,-9807,10.05,100.05,lrhkr
+imei2,2147,9279,100.05,100.055,dfegt
+imei3,-217,-9206,100.005,100.05,lrhkr
+imei4,10,0,15.5,45,Lily
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/OLDFORMATTABLE.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/OLDFORMATTABLE.csv b/integration/spark-common-test/src/test/resources/OLDFORMATTABLE.csv
new file mode 100644
index 0000000..12a6fe8
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/OLDFORMATTABLE.csv
@@ -0,0 +1,34 @@
+country,name,phonetype,serialname,salary
+china,aaa1,phone197,ASD69643,15000
+china,aaa2,phone756,ASD42892,15001
+china,aaa3,phone1904,ASD37014,15002
+china,aaa4,phone2435,ASD66902,15003
+china,aaa5,phone2441,ASD90633,15004
+china,aaa6,phone294,ASD59961,15005
+china,aaa7,phone610,ASD14875,15006
+china,aaa8,phone1848,ASD57308,15007
+china,aaa9,phone706,ASD86717,15008
+usa,aaa10,phone685,ASD30505,15009
+china,aaa11,phone1554,ASD26101,15010
+china,aaa12,phone1781,ASD85711,15011
+china,aaa13,phone943,ASD39200,15012
+china,aaa14,phone1954,ASD80468,15013
+china,aaa15,phone451,ASD1954,15014
+china,aaa16,phone390,ASD38513,15015
+china,aaa17,phone1929,ASD86213,15016
+usa,aaa18,phone910,ASD88812,15017
+china,aaa19,phone2151,ASD9316,15018
+china,aaa20,phone2625,ASD62597,15019
+china,aaa21,phone1371,ASD27896,15020
+china,aaa22,phone945,ASD79760,15021
+china,aaa23,phone2177,ASD45410,15022
+china,aaa24,phone1586,ASD80645,15023
+china,aaa25,phone1310,ASD36408,15024
+china,aaa26,phone1579,ASD14571,15025
+china,aaa27,phone2123,ASD36243,15026
+china,aaa28,phone2334,ASD57825,15027
+china,aaa29,phone1166,ASD26161,15028
+china,aaa30,phone2248,ASD47899,15029
+china,aaa31,phone475,ASD89811,15030
+china,aaa32,phone2499,ASD87974,15031
+china,aaa33,phone2333,ASD62408,15032
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/OLDFORMATTABLEHIVE.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/OLDFORMATTABLEHIVE.csv b/integration/spark-common-test/src/test/resources/OLDFORMATTABLEHIVE.csv
new file mode 100644
index 0000000..55642aa
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/OLDFORMATTABLEHIVE.csv
@@ -0,0 +1,33 @@
+china,aaa1,phone197,ASD69643,15000
+china,aaa2,phone756,ASD42892,15001
+china,aaa3,phone1904,ASD37014,15002
+china,aaa4,phone2435,ASD66902,15003
+china,aaa5,phone2441,ASD90633,15004
+china,aaa6,phone294,ASD59961,15005
+china,aaa7,phone610,ASD14875,15006
+china,aaa8,phone1848,ASD57308,15007
+china,aaa9,phone706,ASD86717,15008
+usa,aaa10,phone685,ASD30505,15009
+china,aaa11,phone1554,ASD26101,15010
+china,aaa12,phone1781,ASD85711,15011
+china,aaa13,phone943,ASD39200,15012
+china,aaa14,phone1954,ASD80468,15013
+china,aaa15,phone451,ASD1954,15014
+china,aaa16,phone390,ASD38513,15015
+china,aaa17,phone1929,ASD86213,15016
+usa,aaa18,phone910,ASD88812,15017
+china,aaa19,phone2151,ASD9316,15018
+china,aaa20,phone2625,ASD62597,15019
+china,aaa21,phone1371,ASD27896,15020
+china,aaa22,phone945,ASD79760,15021
+china,aaa23,phone2177,ASD45410,15022
+china,aaa24,phone1586,ASD80645,15023
+china,aaa25,phone1310,ASD36408,15024
+china,aaa26,phone1579,ASD14571,15025
+china,aaa27,phone2123,ASD36243,15026
+china,aaa28,phone2334,ASD57825,15027
+china,aaa29,phone1166,ASD26161,15028
+china,aaa30,phone2248,ASD47899,15029
+china,aaa31,phone475,ASD89811,15030
+china,aaa32,phone2499,ASD87974,15031
+china,aaa33,phone2333,ASD62408,15032
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/Test_Data1_Logrithmic.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/Test_Data1_Logrithmic.csv b/integration/spark-common-test/src/test/resources/Test_Data1_Logrithmic.csv
new file mode 100644
index 0000000..0f0312d
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/Test_Data1_Logrithmic.csv
@@ -0,0 +1,3 @@
+c1_int,c2_Bigint,c3_Decimal,c4_double,c5_string,c6_Timestamp,c7_Datatype_Desc
+2147483646,9223372036854775807,0.12345678900987654321123456789012345638,1.7976931348623157E308,SQL Capability Carbon is fully compliant with Spark SQL and supports all SQL queries which can run directly on Spark SQL Easy Table Definition: Carbon supports easy to use DDL Data Definition Language statements to define and create tables. Carbon DDL is highly flexible and is very easy to use as well as powerful enough to define complex tables Easy Data anagement Carbon supports a variety of data management functions for loading data to table and maintaining the data in table. Carbon supports bulkloading historical data as well as incrementally loading new data. Loaded data can be deleted based on load time or a specific load can be undone Carbon file format is a columnar store in HDFS it has many features that a modern columnar format hasz such as splittablez compression schema zcomplex data type and so on. Carbon has following unique features Stores data along with index: it can signi
ficantly accelerate query performance and reduces the scans and CPU resources where there are filters in the query. Carbon index consists of multiple level of indicesz a processing framework can leverage this index to reduce the task it needs to schedule and processz and it can also do skip scan in more finer grain unit in task side scanning instead of scanning the whole file.Operable encoded data Through supporting efficient compression and global encoding schemes can query on compressed encoded dataz the data can be converted to encoded data just before returning the results to the usersz which is Column group Allow multiple columns to form a column group that would be stored as row format. This reduces the row reconstruction cost at query time Supports for various use cases with one single Data format like interactive OLAP-style queryz Sequential Access SQL Capability Carbon is fully compliant with Spark SQL and supports all SQL queries which can run directly on Spark SQL Easy
Table Definition: Carbon supports easy to use DDL Data Definition Language statements to define and create tables. Carbon DDL is highly flexible and is very easy to use as well as powerful enough to define complex tables Easy Data anagement Carbon supports a variety of data management functions for loading data to table and maintaining the data in table. Carbon supports bulkloading historical data as well as incrementally loading new data. Loaded data can be deleted based on load time or a specific load can be undone it can significantly accelerate query performance and reduces the scans and CPU resources where there are filters in the query. Carbon index consists of multiple level of indicesz a processing framework can leverage this index to reduce the task it needs to schedule and processz and it can also do skip scan in more finer grain unit in task side scanning instead of scanning the whole file Operable encoded data Through supporting efficient compression and global encodin
g schemes can query on compressed encoded dataz the data can be converted to encoded data just before returning the results to the usersz which is Column group Allow multiple columns to form a column group that would be stored as row format This reduces the row reconstruction cost at query time Supports for various use cases with one single Sequential Access Carbon has following unique features Stores data along with index: it can significantly accelerate query performance and reduces the scans and CPU resources where there are filters in the query. Carbon index consists of multiple level of indicesz a processing framework can leverage this index to reduce the task it needs to schedule and processz and it can also do skip scan in more finer grain unit in task side scanning instead of scanning the whole file Operable encoded data Through supporting efficient compression and global encoding schemes can query on compressed encoded dataz the data can be converted to encoded data just
before returning the results to the users which is Column group Allow multiple columns to form a column group that would be stored as row format Allow multiple columns to form format Allow carbon is fu,2017-07-01 12:07:28,Max_range_values-1
+2147483646,9223372036854775807,12345678900987654321123456789012345678,1.7976931348623157E308,SQL Capability Carbon is fully compliant with Spark SQL and supports all SQL queries which can run directly on Spark SQL Easy Table Definition: Carbon supports easy to use DDL Data Definition Language statements to define and create tables. Carbon DDL is highly flexible and is very easy to use as well as powerful enough to define complex tables Easy Data anagement Carbon supports a variety of data management functions for loading data to table and maintaining the data in table. Carbon supports bulkloading historical data as well as incrementally loading new data. Loaded data can be deleted based on load time or a specific load can be undone Carbon file format is a columnar store in HDFS it has many features that a modern columnar format hasz such as splittablez compression schema zcomplex data type and so on. Carbon has following unique features Stores data along with index: it can signifi
cantly accelerate query performance and reduces the scans and CPU resources where there are filters in the query. Carbon index consists of multiple level of indicesz a processing framework can leverage this index to reduce the task it needs to schedule and processz and it can also do skip scan in more finer grain unit in task side scanning instead of scanning the whole file.Operable encoded data Through supporting efficient compression and global encoding schemes can query on compressed encoded dataz the data can be converted to encoded data just before returning the results to the usersz which is Column group Allow multiple columns to form a column group that would be stored as row format. This reduces the row reconstruction cost at query time Supports for various use cases with one single Data format like interactive OLAP-style queryz Sequential Access SQL Capability Carbon is fully compliant with Spark SQL and supports all SQL queries which can run directly on Spark SQL Easy Ta
ble Definition: Carbon supports easy to use DDL Data Definition Language statements to define and create tables. Carbon DDL is highly flexible and is very easy to use as well as powerful enough to define complex tables Easy Data anagement Carbon supports a variety of data management functions for loading data to table and maintaining the data in table. Carbon supports bulkloading historical data as well as incrementally loading new data. Loaded data can be deleted based on load time or a specific load can be undone it can significantly accelerate query performance and reduces the scans and CPU resources where there are filters in the query. Carbon index consists of multiple level of indicesz a processing framework can leverage this index to reduce the task it needs to schedule and processz and it can also do skip scan in more finer grain unit in task side scanning instead of scanning the whole file Operable encoded data Through supporting efficient compression and global encoding
schemes can query on compressed encoded dataz the data can be converted to encoded data just before returning the results to the usersz which is Column group Allow multiple columns to form a column group that would be stored as row format This reduces the row reconstruction cost at query time Supports for various use cases with one single Sequential Access Carbon has following unique features Stores data along with index: it can significantly accelerate query performance and reduces the scans and CPU resources where there are filters in the query. Carbon index consists of multiple level of indicesz a processing framework can leverage this index to reduce the task it needs to schedule and processz and it can also do skip scan in more finer grain unit in task side scanning instead of scanning the whole file Operable encoded data Through supporting efficient compression and global encoding schemes can query on compressed encoded dataz the data can be converted to encoded data just b
efore returning the results to the users which is Column group Allow multiple columns to form a column group that would be stored as row format Allow multiple columns to form format Allow carbon is f,2017-07-01 12:07:28,Max_range_values-2
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/alldatatypescube.xml
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/alldatatypescube.xml b/integration/spark-common-test/src/test/resources/alldatatypescube.xml
new file mode 100644
index 0000000..84a4a68
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/alldatatypescube.xml
@@ -0,0 +1,109 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<Schema name="default">
+ <Cube name="alldatatypescube" visible="true" cache="true" enabled="true">
+ <Table name="alldatatypescube" alias="">
+ </Table>
+ <Dimension type="StandardDimension" visible="true" highCardinality="false" name="empno">
+ <Hierarchy name="empno" hasAll="true" normalized="false">
+ <Level name="empno" column="empno" type="Integer" levelType="Regular">
+ </Level>
+ </Hierarchy>
+ </Dimension>
+ <Dimension type="StandardDimension" visible="true" highCardinality="false" name="empname">
+ <Hierarchy name="empname" hasAll="true" normalized="false">
+ <Level name="empname" column="empname" type="String" levelType="Regular">
+ </Level>
+ </Hierarchy>
+ </Dimension>
+ <Dimension type="StandardDimension" visible="true" highCardinality="false"
+ name="designation">
+ <Hierarchy name="designation" hasAll="true" normalized="false">
+ <Level name="designation" column="designation" type="String" levelType="Regular">
+ </Level>
+ </Hierarchy>
+ </Dimension>
+ <Dimension type="StandardDimension" visible="true" highCardinality="false" name="doj">
+ <Hierarchy name="doj" hasAll="true" normalized="false">
+ <Level name="doj" column="doj" type="Timestamp" levelType="Regular">
+ </Level>
+ </Hierarchy>
+ </Dimension>
+ <Dimension type="StandardDimension" visible="true" highCardinality="false"
+ name="workgroupcategory">
+ <Hierarchy name="workgroupcategory" hasAll="true" normalized="false">
+ <Level name="workgroupcategory" column="workgroupcategory" type="Integer"
+ levelType="Regular">
+ </Level>
+ </Hierarchy>
+ </Dimension>
+ <Dimension type="StandardDimension" visible="true" highCardinality="false"
+ name="workgroupcategoryname">
+ <Hierarchy name="workgroupcategoryname" hasAll="true" normalized="false">
+ <Level name="workgroupcategoryname" column="workgroupcategoryname" type="String"
+ levelType="Regular">
+ </Level>
+ </Hierarchy>
+ </Dimension>
+ <Dimension type="StandardDimension" visible="true" highCardinality="false" name="deptno">
+ <Hierarchy name="deptno" hasAll="true" normalized="false">
+ <Level name="deptno" column="deptno" type="Integer" levelType="Regular">
+ </Level>
+ </Hierarchy>
+ </Dimension>
+ <Dimension type="StandardDimension" visible="true" highCardinality="false" name="deptname">
+ <Hierarchy name="deptname" hasAll="true" normalized="false">
+ <Level name="deptname" column="deptname" type="String" levelType="Regular">
+ </Level>
+ </Hierarchy>
+ </Dimension>
+ <Dimension type="StandardDimension" visible="true" highCardinality="false"
+ name="projectcode">
+ <Hierarchy name="projectcode" hasAll="true" normalized="false">
+ <Level name="projectcode" column="projectcode" type="Integer" levelType="Regular">
+ </Level>
+ </Hierarchy>
+ </Dimension>
+ <Dimension type="StandardDimension" visible="true" highCardinality="false"
+ name="projectjoindate">
+ <Hierarchy name="projectjoindate" hasAll="true" normalized="false">
+ <Level name="projectjoindate" column="projectjoindate" type="Timestamp"
+ levelType="Regular">
+ </Level>
+ </Hierarchy>
+ </Dimension>
+ <Dimension type="StandardDimension" visible="true" highCardinality="false"
+ name="projectenddate">
+ <Hierarchy name="projectenddate" hasAll="true" normalized="false">
+ <Level name="projectenddate" column="projectenddate" type="Timestamp"
+ levelType="Regular">
+ </Level>
+ </Hierarchy>
+ </Dimension>
+ <Measure name="attendance" column="attendance" datatype="Integer" aggregator="sum"
+ visible="true">
+ </Measure>
+ <Measure name="utilization" column="utilization" datatype="Integer" aggregator="sum"
+ visible="true">
+ </Measure>
+ <Measure name="salary" column="salary" datatype="Integer" aggregator="sum" visible="true">
+ </Measure>
+ </Cube>
+</Schema>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/alldictionary/complex/20160423/1400_1405/complex.dictionary
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/alldictionary/complex/20160423/1400_1405/complex.dictionary b/integration/spark-common-test/src/test/resources/alldictionary/complex/20160423/1400_1405/complex.dictionary
new file mode 100644
index 0000000..223cabd
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/alldictionary/complex/20160423/1400_1405/complex.dictionary
@@ -0,0 +1,20 @@
+0,100080
+1,2355
+2,1ROM size
+3,29-11-2015
+4,1AA100080$2BB100080
+5,MAC283$MAC284$MAC285
+6,1:Chinese:Guangdong Province:shenzhen:longgang:matishan$1:India:Guangdong Province:shenzhen:longgang:matishan
+7,02-03-2016$02-03-2016:02-03-2016
+8,2355
+9,954
+0,100081
+1,1650
+2,6ROM size
+3,29-11-2015
+4,1AA100081$2BB100081
+5,MAC286$MAC287$MAC288
+6,1:Chinese:Guangdong Province:shenzhen:longgang:matishan$1:India:Guangdong Province:shenzhen:longgang:matishan
+7,03-03-2016$03-03-2016:03-03-2016
+8,1650
+9,613
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/alldictionary/sample/20160423/1400_1405/sample.dictionary
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/alldictionary/sample/20160423/1400_1405/sample.dictionary b/integration/spark-common-test/src/test/resources/alldictionary/sample/20160423/1400_1405/sample.dictionary
new file mode 100644
index 0000000..592e634
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/alldictionary/sample/20160423/1400_1405/sample.dictionary
@@ -0,0 +1,9 @@
+0,1
+1,david
+2,shenzhen
+0,2
+1,eason
+0,3
+1,jarry
+2,wuhan
+2,Bangalore
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/array1.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/array1.csv b/integration/spark-common-test/src/test/resources/array1.csv
new file mode 100644
index 0000000..c7a0f89
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/array1.csv
@@ -0,0 +1,2 @@
+1,hello$bye,12345$5678,123456$3456,12.0$13.0,2017-07-09 12:00:00$2016-07-09 13:00:00,123456789$987654321
+2,welcome$hi,9876$1234,13456$356,15.0$18.0,2019-07-09 12:00:00$2015-07-09 13:00:00,8888888888$99999999999
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/arrayColumnEmpty.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/arrayColumnEmpty.csv b/integration/spark-common-test/src/test/resources/arrayColumnEmpty.csv
new file mode 100644
index 0000000..6298196
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/arrayColumnEmpty.csv
@@ -0,0 +1,21 @@
+imei,age,productdate,gamePointId,reserved6,mobile
+ime100004,14,2016-05-02 05:19:24,95,,April/100
+ime100003,10,2016-05-01 13:09:20,12,,Apr-98
+ime100003,13,2016-05-01 16:08:51,67,,Mar-91
+ime100004,13,2016-05-01 19:11:53,67,,Apr-72
+ime100005,10,2016-05-02 02:17:36,12,,May-91
+ime100004,13,2016-05-01 16:37:11,67,,Mar-47
+ime100005,11,2016-05-01 06:27:23,34.5,,Apr-40
+ime100002,10,2016-05-02 05:19:24,12,,Apr-37
+ime100005,10,2016-05-01 14:56:12,12,,May-32
+ime100003,10,2016-05-01 16:39:25,12,,20-Mar
+ime100002,10,2016-05-01 23:00:29,12,,May-36
+ime100002,13,2016-05-01 12:42:14,67,,16-Jul
+ime100005,11,2016-05-01 20:12:10,34.5,,Jun-61
+ime100005,13,2016-05-02 06:06:05,67,,May-99
+ime100005,10,2016-05-01 23:58:34,12,,Mar-55
+ime100005,12,2016-05-01 09:46:31,34.6,,Apr-86
+ime100005,12,2016-05-01 13:27:11,34.6,,Jun-34
+ime100003,14,2016-05-01 05:36:34,95,,Jul-89
+ime100001,14,2016-05-01 08:13:27,95,,Jun-47
+ime100001,14,2016-05-01 13:40:14,95,,Mar-75
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/avgTest.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/avgTest.csv b/integration/spark-common-test/src/test/resources/avgTest.csv
new file mode 100644
index 0000000..9df72fb
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/avgTest.csv
@@ -0,0 +1,16 @@
+ID,date,country,count,phonetype,serialname,salary
+1,2015-7-23 00:00:00,china,1234,phone197,ASD69643,15000
+2,2015-7-24 00:00:00,china,1233,phone756,ASD42892,15001
+3,2015-7-25 00:00:00,china,1000,phone1904,ASD37014,15002
+4,2015-7-26 00:00:00,china,2222,phone2435,ASD66902,15003
+5,2015-7-27 00:00:00,china,1300,phone2441,ASD90633,15004
+6,2015-7-28 00:00:00,china,2100,phone294,ASD59961,15005
+7,2015-7-29 00:00:00,china,1500,phone610,ASD14875,15006
+8,2015-7-30 00:00:00,china,1600,phone1848,ASD57308,15007
+9,2015-7-18 00:00:00,china,1000,phone706,ASD86717,15008
+10,2015-7-19 00:00:00,usa,2100,phone685,ASD30505,15009
+11,2015-7-18 00:00:00,china,2200,phone1554,ASD26101,15010
+12,2015-7-19 00:00:00,china,1300,phone1781,ASD85711,15011
+13,2015-7-20 00:00:00,china,2000,phone943,ASD39200,15012
+14,2015-7-21 00:00:00,china,1500,phone1954,ASD80468,15013
+15,2015-7-22 00:00:00,china,6200,phone451,ASD1954,15014
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/badrecords/datasample.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/badrecords/datasample.csv b/integration/spark-common-test/src/test/resources/badrecords/datasample.csv
new file mode 100644
index 0000000..3895102
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/badrecords/datasample.csv
@@ -0,0 +1,7 @@
+ID,date,country,actual_price,Quantity,sold_price
+10000,2015/7/23,china,120000.45,3,140000.377
+10001,null,china,120000.45,3,140000.377
+10003,2015/7/23,null,120000.45,3,140000.377
+10004,2015/7/23,china,120000.45ghf,3,140000.377
+10005,2015/7/23,china,120000.45,3ghf,140000.377
+10006,2015/7/23,china,120000.45,3,140000.377ghf
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/badrecords/emptyTimeStampValue.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/badrecords/emptyTimeStampValue.csv b/integration/spark-common-test/src/test/resources/badrecords/emptyTimeStampValue.csv
new file mode 100644
index 0000000..751b21b
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/badrecords/emptyTimeStampValue.csv
@@ -0,0 +1,8 @@
+ID,date,country,actual_price,Quantity,sold_price
+\N,,\N,\N,\N,\N
+10003,2015/7/23,xyz,120003.45,3,140000.377
+10003gh,2015/7/23,xyz,120003.45,3,140000.377
+10003,20/7/2016,xyz,120003.45,3,140000.377
+10003,2015/7/23,xyz,120003.45gf,3,140000.377
+
+10003,2015/7/23,xyz,120003.45,3,140000.377gf
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/badrecords/emptyValues.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/badrecords/emptyValues.csv b/integration/spark-common-test/src/test/resources/badrecords/emptyValues.csv
new file mode 100644
index 0000000..492d6c2
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/badrecords/emptyValues.csv
@@ -0,0 +1,8 @@
+ID,date,country,actual_price,Quantity,sold_price
+10001,2015/7/23,,120003.45,3,140000.377
+,2015/7/23,,120003.45,3,140000.377
+,,,120003.45,3,140000.377
+,,,,3,140000.377
+,,,,,140000.377
+,,,,,
+10003,2015/7/23,india,120003.45,3,140000.377
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/badrecords/insufficientColumns.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/badrecords/insufficientColumns.csv b/integration/spark-common-test/src/test/resources/badrecords/insufficientColumns.csv
new file mode 100644
index 0000000..24ac3f4
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/badrecords/insufficientColumns.csv
@@ -0,0 +1,4 @@
+ID,date,country,actual_price,Quantity,sold_price
+100,
+10001,2015/7/23,
+10003,2015/7/23,null,120003.45,3,140000.377
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/badrecords/seriazableValue.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/badrecords/seriazableValue.csv b/integration/spark-common-test/src/test/resources/badrecords/seriazableValue.csv
new file mode 100644
index 0000000..2598a90
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/badrecords/seriazableValue.csv
@@ -0,0 +1,3 @@
+ID,date,country,actual_price,Quantity,sold_price
+\N,\N,\N,\N,\N,\N
+10003,2015/7/23,null,120003.45,3,140000.377
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/bigIntData.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/bigIntData.csv b/integration/spark-common-test/src/test/resources/bigIntData.csv
new file mode 100644
index 0000000..443a514
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/bigIntData.csv
@@ -0,0 +1,14 @@
+ID,date,country,name,phonetype,serialname,salary
+1,2015/7/23,china,aaa1,phone197,ASD69643,10000001
+2,2015/7/24,china,aaa2,phone756,ASD42892,10000000
+3,2015/7/25,china,aaa3,phone1904,ASD37014,10000000
+4,2015/7/26,china,aaa4,phone2435,ASD66902,10000000
+5,2015/7/27,china,aaa5,phone2441,ASD90633,10000000
+6,2015/7/28,china,aaa6,phone294,ASD59961,10000000
+7,2015/7/29,china,aaa7,phone610,ASD14875,10000000
+8,2015/7/30,china,aaa8,phone1848,ASD57308,10000000
+9,2015/7/18,china,aaa9,phone706,ASD86717,10000000
+10,2015/7/19,usa,aaa10,phone685,ASD30505,10000000
+11,2015/7/18,china,aaa11,phone1554,ASD26101,10000000
+12,2015/7/19,china,aaa12,phone1781,ASD85711,10000000
+13,2015/7/19,china,aaa13,phone17851,ASD85721,10000000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/bigIntDataWithHeader.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/bigIntDataWithHeader.csv b/integration/spark-common-test/src/test/resources/bigIntDataWithHeader.csv
new file mode 100644
index 0000000..37f5f07
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/bigIntDataWithHeader.csv
@@ -0,0 +1,13 @@
+ID,date,country,name,phonetype,serialname,salary
+1,2015/7/23,china,aaa1,phone197,ASD69643,150003452628
+2,2015/7/24,china,aaa2,phone756,ASD42892,452345254658824
+3,2015/7/25,china,aaa3,phone1904,ASD37014,6521652222
+4,2015/7/26,china,aaa4,phone2435,ASD66902,15003
+5,2015/7/27,china,aaa5,phone2441,ASD90633,1500043525
+6,2015/7/28,china,aaa6,phone294,ASD59961,2745654656565662345
+7,2015/7/29,china,aaa7,phone610,ASD14875,150064555555
+8,2015/7/30,china,aaa8,phone1848,ASD57308,150074422222
+9,2015/7/18,china,aaa9,phone706,ASD86717,15008023
+10,2015/7/19,usa,aaa10,phone685,ASD30505,1500043525
+11,2015/7/18,china,aaa11,phone1554,ASD26101,1501000
+12,2015/7/19,china,aaa12,phone1781,ASD85711,1501120
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/bigIntDataWithoutHeader.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/bigIntDataWithoutHeader.csv b/integration/spark-common-test/src/test/resources/bigIntDataWithoutHeader.csv
new file mode 100644
index 0000000..a2dd9b3
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/bigIntDataWithoutHeader.csv
@@ -0,0 +1,12 @@
+1,2015/7/23,china,aaa1,phone197,ASD69643,150003452628
+2,2015/7/24,china,aaa2,phone756,ASD42892,452345254658824
+3,2015/7/25,china,aaa3,phone1904,ASD37014,6521652222
+4,2015/7/26,china,aaa4,phone2435,ASD66902,15003
+5,2015/7/27,china,aaa5,phone2441,ASD90633,1500043525
+6,2015/7/28,china,aaa6,phone294,ASD59961,2745654656565662345
+7,2015/7/29,china,aaa7,phone610,ASD14875,150064555555
+8,2015/7/30,china,aaa8,phone1848,ASD57308,150074422222
+9,2015/7/18,china,aaa9,phone706,ASD86717,15008023
+10,2015/7/19,usa,aaa10,phone685,ASD30505,1500043525
+11,2015/7/18,china,aaa11,phone1554,ASD26101,1501000
+12,2015/7/19,china,aaa12,phone1781,ASD85711,1501120
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/big_int_Decimal.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/big_int_Decimal.csv b/integration/spark-common-test/src/test/resources/big_int_Decimal.csv
new file mode 100644
index 0000000..3a16df4
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/big_int_Decimal.csv
@@ -0,0 +1,3 @@
+imei,age,task,name,country,city,sale,num,level,quest,productdate,enddate,pointid,score
+imei0,2147,9279,fegt,china,hangzhou,10000,100.05,100.055,10,2016-05-01 12:25:36,2016-05-01 21:14:48,1,1.005
+imei1,-2148,-9807,lrhkr,America,NewYork,1000,10.05,100.05,100,2016-05-02 19:25:15,2016-05-02 22:25:46,2,1.05
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/channelsId.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/channelsId.csv b/integration/spark-common-test/src/test/resources/channelsId.csv
new file mode 100644
index 0000000..7be3d90
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/channelsId.csv
@@ -0,0 +1,10 @@
+1123
+1234
+1111,
+1431,
+1421
+1421|
+1234,
+1245,
+1231|
+2234|
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/character_carbon.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/character_carbon.csv b/integration/spark-common-test/src/test/resources/character_carbon.csv
new file mode 100644
index 0000000..2ea7b6c
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/character_carbon.csv
@@ -0,0 +1,33 @@
+imei,age,task,num,level,productdate,mark,name
+\u4f60~babu~520,\u4f60~babu~520,\u4f60~babu~520,\u4f60~babu~520,\u4f60~babu~520,\u4f60~babu~520,1,\u4f60~babu~520
+\u4f60`babu`520,\u4f60`babu`520,\u4f60`babu`520,\u4f60`babu`520,\u4f60`babu`520,\u4f60`babu`520,2,\u4f60`babu`520
+\u4f60!babu!520,\u4f60!babu!520,\u4f60!babu!520,\u4f60!babu!520,\u4f60!babu!520,\u4f60!babu!520,3,\u4f60!babu!520
+\u4f60@babu@520,\u4f60@babu@520,\u4f60@babu@520,\u4f60@babu@520,\u4f60@babu@520,\u4f60@babu@520,4,\u4f60@babu@520
+\u4f60#babu#520,\u4f60#babu#520,\u4f60#babu#520,\u4f60#babu#520,\u4f60#babu#520,\u4f60#babu#520,5,\u4f60#babu#520
+\u4f60$babu$520,\u4f60$babu$520,\u4f60$babu$520,\u4f60$babu$520,\u4f60$babu$520,\u4f60$babu$520,6,\u4f60$babu$520
+\u4f60%babu%520,\u4f60%babu%520,\u4f60%babu%520,\u4f60%babu%520,\u4f60%babu%520,\u4f60%babu%520,7,\u4f60%babu%520
+\u4f60^babu^520,\u4f60^babu^520,\u4f60^babu^520,\u4f60^babu^520,\u4f60^babu^520,\u4f60^babu^520,8,\u4f60^babu^520
+\u4f60&babu&520,\u4f60&babu&520,\u4f60&babu&520,\u4f60&babu&520,\u4f60&babu&520,\u4f60&babu&520,9,\u4f60&babu&520
+\u4f60*babu*520,\u4f60*babu*520,\u4f60*babu*520,\u4f60*babu*520,\u4f60*babu*520,\u4f60*babu*520,10,\u4f60*babu*520
+\u4f60(babu(520,\u4f60(babu(520,\u4f60(babu(520,\u4f60(babu(520,\u4f60(babu(520,\u4f60(babu(520,11,\u4f60(babu(520
+\u4f60)babu)520,\u4f60)babu)520,\u4f60)babu)520,\u4f60)babu)520,\u4f60)babu)520,\u4f60)babu)520,12,\u4f60)babu)520
+\u4f60-babu-520,\u4f60-babu-520,\u4f60-babu-520,\u4f60-babu-520,\u4f60-babu-520,\u4f60-babu-520,13,\u4f60-babu-520
+\u4f60_babu_520,\u4f60_babu_520,\u4f60_babu_520,\u4f60_babu_520,\u4f60_babu_520,\u4f60_babu_520,14,\u4f60_babu_520
+\u4f60=babu=520,\u4f60=babu=520,\u4f60=babu=520,\u4f60=babu=520,\u4f60=babu=520,\u4f60=babu=520,15,\u4f60=babu=520
+\u4f60+babu+520,\u4f60+babu+520,\u4f60+babu+520,\u4f60+babu+520,\u4f60+babu+520,\u4f60+babu+520,16,\u4f60+babu+520
+\u4f60{babu{520,\u4f60{babu{520,\u4f60{babu{520,\u4f60{babu{520,\u4f60{babu{520,\u4f60{babu{520,17,\u4f60{babu{520
+\u4f60}babu}520,\u4f60}babu}520,\u4f60}babu}520,\u4f60}babu}520,\u4f60}babu}520,\u4f60}babu}520,18,\u4f60}babu}520
+\u4f60[babu[520,\u4f60[babu[520,\u4f60[babu[520,\u4f60[babu[520,\u4f60[babu[520,\u4f60[babu[520,19,\u4f60[babu[520
+\u4f60]babu]520,\u4f60]babu]520,\u4f60]babu]520,\u4f60]babu]520,\u4f60]babu]520,\u4f60]babu]520,20,\u4f60]babu]520
+\u4f60\babu\520,\u4f60\babu\520,\u4f60\babu\520,\u4f60\babu\520,\u4f60\babu\520,\u4f60\babu\520,21,\u4f60\babu\520
+\u4f60|babu|520,\u4f60|babu|520,\u4f60|babu|520,\u4f60|babu|520,\u4f60|babu|520,\u4f60|babu|520,22,\u4f60|babu|520
+\u4f60;babu;520,\u4f60;babu;520,\u4f60;babu;520,\u4f60;babu;520,\u4f60;babu;520,\u4f60;babu;520,23,\u4f60;babu;520
+\u4f60:babu:520,\u4f60:babu:520,\u4f60:babu:520,\u4f60:babu:520,\u4f60:babu:520,\u4f60:babu:520,24,\u4f60:babu:520
+\u4f60'babu'520,\u4f60'babu'520,\u4f60'babu'520,\u4f60'babu'520,\u4f60'babu'520,\u4f60'babu'520,25,\u4f60'babu'520
+\u4f60"babu"520,\u4f60"babu"520,\u4f60"babu"520,\u4f60"babu"520,\u4f60"babu"520,\u4f60"babu"520,26,\u4f60"babu"520
+\u4f60,babu,520,\u4f60,babu,520,\u4f60,babu,520,\u4f60,babu,520,\u4f60,babu,520,\u4f60,babu,520,27,\u4f60,babu,520
+\u4f60.babu.520,\u4f60.babu.520,\u4f60.babu.520,\u4f60.babu.520,\u4f60.babu.520,\u4f60.babu.520,28,\u4f60.babu.520
+\u4f60<babu<520,\u4f60<babu<520,\u4f60<babu<520,\u4f60<babu<520,\u4f60<babu<520,\u4f60<babu<520,29,\u4f60<babu<520
+\u4f60>babu>520,\u4f60>babu>520,\u4f60>babu>520,\u4f60>babu>520,\u4f60>babu>520,\u4f60>babu>520,30,\u4f60>babu>520
+\u4f60?babu?520,\u4f60?babu?520,\u4f60?babu?520,\u4f60?babu?520,\u4f60?babu?520,\u4f60?babu?520,31,\u4f60?babu?520
+\u4f60/babu/520,\u4f60/babu/520,\u4f60/babu/520,\u4f60/babu/520,\u4f60/babu/520,\u4f60/babu/520,32,\u4f60/babu/520
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/character_hive.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/character_hive.csv b/integration/spark-common-test/src/test/resources/character_hive.csv
new file mode 100644
index 0000000..4362ca0
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/character_hive.csv
@@ -0,0 +1,32 @@
+\u4f60~babu~520,\u4f60~babu~520,\u4f60~babu~520,\u4f60~babu~520,\u4f60~babu~520,\u4f60~babu~520,1,\u4f60~babu~520
+\u4f60`babu`520,\u4f60`babu`520,\u4f60`babu`520,\u4f60`babu`520,\u4f60`babu`520,\u4f60`babu`520,2,\u4f60`babu`520
+\u4f60!babu!520,\u4f60!babu!520,\u4f60!babu!520,\u4f60!babu!520,\u4f60!babu!520,\u4f60!babu!520,3,\u4f60!babu!520
+\u4f60@babu@520,\u4f60@babu@520,\u4f60@babu@520,\u4f60@babu@520,\u4f60@babu@520,\u4f60@babu@520,4,\u4f60@babu@520
+\u4f60#babu#520,\u4f60#babu#520,\u4f60#babu#520,\u4f60#babu#520,\u4f60#babu#520,\u4f60#babu#520,5,\u4f60#babu#520
+\u4f60$babu$520,\u4f60$babu$520,\u4f60$babu$520,\u4f60$babu$520,\u4f60$babu$520,\u4f60$babu$520,6,\u4f60$babu$520
+\u4f60%babu%520,\u4f60%babu%520,\u4f60%babu%520,\u4f60%babu%520,\u4f60%babu%520,\u4f60%babu%520,7,\u4f60%babu%520
+\u4f60^babu^520,\u4f60^babu^520,\u4f60^babu^520,\u4f60^babu^520,\u4f60^babu^520,\u4f60^babu^520,8,\u4f60^babu^520
+\u4f60&babu&520,\u4f60&babu&520,\u4f60&babu&520,\u4f60&babu&520,\u4f60&babu&520,\u4f60&babu&520,9,\u4f60&babu&520
+\u4f60*babu*520,\u4f60*babu*520,\u4f60*babu*520,\u4f60*babu*520,\u4f60*babu*520,\u4f60*babu*520,10,\u4f60*babu*520
+\u4f60(babu(520,\u4f60(babu(520,\u4f60(babu(520,\u4f60(babu(520,\u4f60(babu(520,\u4f60(babu(520,11,\u4f60(babu(520
+\u4f60)babu)520,\u4f60)babu)520,\u4f60)babu)520,\u4f60)babu)520,\u4f60)babu)520,\u4f60)babu)520,12,\u4f60)babu)520
+\u4f60-babu-520,\u4f60-babu-520,\u4f60-babu-520,\u4f60-babu-520,\u4f60-babu-520,\u4f60-babu-520,13,\u4f60-babu-520
+\u4f60_babu_520,\u4f60_babu_520,\u4f60_babu_520,\u4f60_babu_520,\u4f60_babu_520,\u4f60_babu_520,14,\u4f60_babu_520
+\u4f60=babu=520,\u4f60=babu=520,\u4f60=babu=520,\u4f60=babu=520,\u4f60=babu=520,\u4f60=babu=520,15,\u4f60=babu=520
+\u4f60+babu+520,\u4f60+babu+520,\u4f60+babu+520,\u4f60+babu+520,\u4f60+babu+520,\u4f60+babu+520,16,\u4f60+babu+520
+\u4f60{babu{520,\u4f60{babu{520,\u4f60{babu{520,\u4f60{babu{520,\u4f60{babu{520,\u4f60{babu{520,17,\u4f60{babu{520
+\u4f60}babu}520,\u4f60}babu}520,\u4f60}babu}520,\u4f60}babu}520,\u4f60}babu}520,\u4f60}babu}520,18,\u4f60}babu}520
+\u4f60[babu[520,\u4f60[babu[520,\u4f60[babu[520,\u4f60[babu[520,\u4f60[babu[520,\u4f60[babu[520,19,\u4f60[babu[520
+\u4f60]babu]520,\u4f60]babu]520,\u4f60]babu]520,\u4f60]babu]520,\u4f60]babu]520,\u4f60]babu]520,20,\u4f60]babu]520
+\u4f60\babu\520,\u4f60\babu\520,\u4f60\babu\520,\u4f60\babu\520,\u4f60\babu\520,\u4f60\babu\520,21,\u4f60\babu\520
+\u4f60|babu|520,\u4f60|babu|520,\u4f60|babu|520,\u4f60|babu|520,\u4f60|babu|520,\u4f60|babu|520,22,\u4f60|babu|520
+\u4f60;babu;520,\u4f60;babu;520,\u4f60;babu;520,\u4f60;babu;520,\u4f60;babu;520,\u4f60;babu;520,23,\u4f60;babu;520
+\u4f60:babu:520,\u4f60:babu:520,\u4f60:babu:520,\u4f60:babu:520,\u4f60:babu:520,\u4f60:babu:520,24,\u4f60:babu:520
+\u4f60'babu'520,\u4f60'babu'520,\u4f60'babu'520,\u4f60'babu'520,\u4f60'babu'520,\u4f60'babu'520,25,\u4f60'babu'520
+\u4f60"babu"520,\u4f60"babu"520,\u4f60"babu"520,\u4f60"babu"520,\u4f60"babu"520,\u4f60"babu"520,26,\u4f60"babu"520
+\u4f60,babu,520,\u4f60,babu,520,\u4f60,babu,520,\u4f60,babu,520,\u4f60,babu,520,\u4f60,babu,520,27,\u4f60,babu,520
+\u4f60.babu.520,\u4f60.babu.520,\u4f60.babu.520,\u4f60.babu.520,\u4f60.babu.520,\u4f60.babu.520,28,\u4f60.babu.520
+\u4f60<babu<520,\u4f60<babu<520,\u4f60<babu<520,\u4f60<babu<520,\u4f60<babu<520,\u4f60<babu<520,29,\u4f60<babu<520
+\u4f60>babu>520,\u4f60>babu>520,\u4f60>babu>520,\u4f60>babu>520,\u4f60>babu>520,\u4f60>babu>520,30,\u4f60>babu>520
+\u4f60?babu?520,\u4f60?babu?520,\u4f60?babu?520,\u4f60?babu?520,\u4f60?babu?520,\u4f60?babu?520,31,\u4f60?babu?520
+\u4f60/babu/520,\u4f60/babu/520,\u4f60/babu/520,\u4f60/babu/520,\u4f60/babu/520,\u4f60/babu/520,32,\u4f60/babu/520
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/columndictionary/country.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/columndictionary/country.csv b/integration/spark-common-test/src/test/resources/columndictionary/country.csv
new file mode 100644
index 0000000..1fd8032
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/columndictionary/country.csv
@@ -0,0 +1,5 @@
+usa
+china
+uk
+france
+brazil
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/columndictionary/name.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/columndictionary/name.csv b/integration/spark-common-test/src/test/resources/columndictionary/name.csv
new file mode 100644
index 0000000..cbd6eef
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/columndictionary/name.csv
@@ -0,0 +1,10 @@
+aaa1
+aaa2
+aaa3
+aaa4
+aaa5
+aaa6
+aaa7
+aaa8
+aaa9
+aaa10
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/comment.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/comment.csv b/integration/spark-common-test/src/test/resources/comment.csv
new file mode 100644
index 0000000..34ccb12
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/comment.csv
@@ -0,0 +1,5 @@
+.~carbon,.,~carbon,~carbon,~carbon,~carbon,~carbon,~carbon,~carbon
+,carbon,,carbon,,carbon,,carbon,,carbon,,carbon,,carbon,,carbon
+#?carbon,#carbon,#carbon,#carbon,#carbon,#carbon,#carbon,#carbon
+?carbon,#carbon,#carbon,#carbon,#carbon,#carbon,#carbon,#carbon
+".carbon,"carbon,"carbon,"carbon,"carbon,"carbon,"carbon,"carbon
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/compaction/compaction1.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/compaction/compaction1.csv b/integration/spark-common-test/src/test/resources/compaction/compaction1.csv
new file mode 100644
index 0000000..b669eca
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/compaction/compaction1.csv
@@ -0,0 +1,6 @@
+ID,date,country,name,phonetype,serialname,salary
+1,07/23/2015,china,aaa1,phone197,ASD69643,15000
+2,07/24/2015,chile,aaa2,phone756,ASD42892,15001
+3,07/25/2015,america,aaa3,phone1904,ASD37014,15002
+4,07/26/2015,canada,aaa4,phone2435,ASD66902,15003
+5,07/27/2015,england,aaa5,phone2441,ASD90633,15004
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/compaction/compaction1_forhive.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/compaction/compaction1_forhive.csv b/integration/spark-common-test/src/test/resources/compaction/compaction1_forhive.csv
new file mode 100644
index 0000000..07c2508
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/compaction/compaction1_forhive.csv
@@ -0,0 +1,5 @@
+1,07/23/2015,china,aaa1,phone197,ASD69643,15000
+2,07/24/2015,chile,aaa2,phone756,ASD42892,15001
+3,07/25/2015,america,aaa3,phone1904,ASD37014,15002
+4,07/26/2015,canada,aaa4,phone2435,ASD66902,15003
+5,07/27/2015,england,aaa5,phone2441,ASD90633,15004
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/compaction/compaction2.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/compaction/compaction2.csv b/integration/spark-common-test/src/test/resources/compaction/compaction2.csv
new file mode 100644
index 0000000..57233a1
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/compaction/compaction2.csv
@@ -0,0 +1,6 @@
+ID,date,country,name,phonetype,serialname,salary
+6,07/28/2015,newzealand,aaa6,phone294,ASD59961,15005
+7,07/29/2015,mexico,aaa7,phone610,ASD14875,15006
+8,07/30/2015,westindies,aaa8,phone1848,ASD57308,15007
+9,07/18/2015,burma,aaa9,phone706,ASD86717,15008
+10,07/19/2015,butan,aaa10,phone685,ASD30505,1500
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/compaction/compaction3.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/compaction/compaction3.csv b/integration/spark-common-test/src/test/resources/compaction/compaction3.csv
new file mode 100644
index 0000000..14a1522
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/compaction/compaction3.csv
@@ -0,0 +1,6 @@
+ID,date,country,name,phonetype,serialname,salary
+11,07/23/2015,china,aaa1,phone197,ASD69643,15005
+12,07/24/2015,india,aaa2,phone756,ASD42892,15006
+13,07/26/2015,iran,aaa4,phone2435,ASD66902,15008
+14,07/27/2015,iraq,aaa5,phone2441,ASD90633,15009
+15,07/28/2015,ireland,aaa6,phone2441,ASD90633,15010
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/compaction/compactioncard2.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/compaction/compactioncard2.csv b/integration/spark-common-test/src/test/resources/compaction/compactioncard2.csv
new file mode 100644
index 0000000..75fa792
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/compaction/compactioncard2.csv
@@ -0,0 +1,257 @@
+ID,date,country,name,phonetype,serialname,salary
+6,7/28/2015,newzealand,aaa6,phone294,ASD59961,15005
+7,7/29/2015,mexico,aaa7,phone610,ASD14875,15006
+8,7/30/2015,westindies,aaa8,phone1848,ASD57308,15007
+9,7/18/2015,burma,aaa9,phone706,ASD86717,15008
+10,7/19/2015,butan,aaa10,phone685,ASD30505,1500
+11,7/20/2015,burma,aaa11,phone664,ASD25707,1500
+12,7/21/2015,butan,aaa12,phone643,ASD81919,1500
+13,7/22/2015,burma,aaa13,phone622,ASD138131,1500
+14,7/23/2015,burma,aaa14,phone601,ASD194343,1500
+15,7/24/2015,butan,aaa15,phone580,ASD250555,1500
+16,7/25/2015,burma,aaa16,phone559,ASD306767,1500
+17,7/26/2015,butan,aaa17,phone538,ASD362979,1500
+18,7/27/2015,burma,aaa18,phone517,ASD419191,1500
+19,7/28/2015,burma,aaa19,phone496,ASD475403,1500
+20,7/29/2015,butan,aaa20,phone475,ASD531615,1500
+21,7/30/2015,burma,aaa21,phone454,ASD587827,1500
+22,7/31/2015,butan,aaa22,phone433,ASD644039,1500
+23,8/1/2015,burma,aaa23,phone412,ASD700251,1500
+24,8/2/2015,burma,aaa24,phone391,ASD756463,1500
+25,8/3/2015,butan,aaa25,phone370,ASD812675,1500
+26,8/4/2015,burma,aaa26,phone349,ASD868887,1500
+27,8/5/2015,butan,aaa27,phone328,ASD925099,1500
+28,8/6/2015,burma,aaa28,phone307,ASD981311,1500
+29,8/7/2015,burma,aaa29,phone286,ASD1037523,1500
+30,8/8/2015,butan,aaa30,phone265,ASD1093735,1500
+31,8/9/2015,burma,aaa31,phone244,ASD1149947,1500
+32,8/10/2015,butan,aaa32,phone223,ASD1206159,1500
+33,8/11/2015,burma,aaa33,phone202,ASD1262371,1500
+34,8/12/2015,burma,aaa34,phone181,ASD1318583,1500
+35,8/13/2015,butan,aaa35,phone160,ASD1374795,1500
+36,8/14/2015,burma,aaa36,phone139,ASD1431007,1500
+37,8/15/2015,butan,aaa37,phone118,ASD1487219,1500
+38,8/16/2015,burma,aaa38,phone97,ASD1543431,1500
+39,8/17/2015,burma,aaa39,phone76,ASD1599643,1500
+40,8/18/2015,butan,aaa40,phone55,ASD1655855,1500
+41,8/19/2015,burma,aaa41,phone34,ASD1712067,1500
+42,8/20/2015,butan,aaa42,phone13,ASD1768279,1500
+43,8/21/2015,burma,aaa43,phone8,ASD1824491,1500
+44,8/22/2015,burma,aaa44,phone29,ASD1880703,1500
+45,8/23/2015,butan,aaa45,phone50,ASD1936915,1500
+46,8/24/2015,burma,aaa46,phone71,ASD1993127,1500
+47,8/25/2015,butan,aaa47,phone92,ASD2049339,1500
+48,8/26/2015,burma,aaa48,phone113,ASD2105551,1500
+49,8/27/2015,burma,aaa49,phone134,ASD2161763,1500
+50,8/28/2015,butan,aaa50,phone155,ASD2217975,1500
+51,8/29/2015,burma,aaa51,phone176,ASD2274187,1500
+52,8/30/2015,butan,aaa52,phone197,ASD2330399,1500
+53,8/31/2015,burma,aaa53,phone218,ASD2386611,1500
+54,9/1/2015,burma,aaa54,phone239,ASD2442823,1500
+55,9/2/2015,butan,aaa55,phone260,ASD2499035,1500
+56,9/3/2015,burma,aaa56,phone281,ASD2555247,1500
+57,9/4/2015,butan,aaa57,phone302,ASD2611459,1500
+58,9/5/2015,burma,aaa58,phone323,ASD2667671,1500
+59,9/6/2015,burma,aaa59,phone344,ASD2723883,1500
+60,9/7/2015,butan,aaa60,phone365,ASD2780095,1500
+61,9/8/2015,burma,aaa61,phone386,ASD2836307,1500
+62,9/9/2015,butan,aaa62,phone407,ASD2892519,1500
+63,9/10/2015,burma,aaa63,phone428,ASD2948731,1500
+64,9/11/2015,burma,aaa64,phone449,ASD3004943,1500
+65,9/12/2015,butan,aaa65,phone470,ASD3061155,1500
+66,9/13/2015,burma,aaa66,phone491,ASD3117367,1500
+67,9/14/2015,butan,aaa67,phone512,ASD3173579,1500
+68,9/15/2015,burma,aaa68,phone533,ASD3229791,1500
+69,9/16/2015,burma,aaa69,phone554,ASD3286003,1500
+70,9/17/2015,butan,aaa70,phone575,ASD3342215,1500
+71,9/18/2015,burma,aaa71,phone596,ASD3398427,1500
+72,9/19/2015,butan,aaa72,phone617,ASD3454639,1500
+73,9/20/2015,burma,aaa73,phone638,ASD3510851,1500
+74,9/21/2015,burma,aaa74,phone659,ASD3567063,1500
+75,9/22/2015,butan,aaa75,phone680,ASD3623275,1500
+76,9/23/2015,burma,aaa76,phone701,ASD3679487,1500
+77,9/24/2015,butan,aaa77,phone722,ASD3735699,1500
+78,9/25/2015,burma,aaa78,phone743,ASD3791911,1500
+79,9/26/2015,burma,aaa79,phone764,ASD3848123,1500
+80,9/27/2015,butan,aaa80,phone785,ASD3904335,1500
+81,9/28/2015,burma,aaa81,phone806,ASD3960547,1500
+82,9/29/2015,butan,aaa82,phone827,ASD4016759,1500
+83,9/30/2015,burma,aaa83,phone848,ASD4072971,1500
+84,10/1/2015,burma,aaa84,phone869,ASD4129183,1500
+85,10/2/2015,butan,aaa85,phone890,ASD4185395,1500
+86,10/3/2015,burma,aaa86,phone911,ASD4241607,1500
+87,10/4/2015,butan,aaa87,phone932,ASD4297819,1500
+88,10/5/2015,burma,aaa88,phone953,ASD4354031,1500
+89,10/6/2015,burma,aaa89,phone974,ASD4410243,1500
+90,10/7/2015,butan,aaa90,phone995,ASD4466455,1500
+91,10/8/2015,burma,aaa91,phone1016,ASD4522667,1500
+92,10/9/2015,butan,aaa92,phone1037,ASD4578879,1500
+93,10/10/2015,burma,aaa93,phone1058,ASD4635091,1500
+94,10/11/2015,burma,aaa94,phone1079,ASD4691303,1500
+95,10/12/2015,butan,aaa95,phone1100,ASD4747515,1500
+96,10/13/2015,burma,aaa96,phone1121,ASD4803727,1500
+97,10/14/2015,butan,aaa97,phone1142,ASD4859939,1500
+98,10/15/2015,burma,aaa98,phone1163,ASD4916151,1500
+99,10/16/2015,burma,aaa99,phone1184,ASD4972363,1500
+100,10/17/2015,butan,aaa100,phone1205,ASD5028575,1500
+101,10/18/2015,burma,aaa101,phone1226,ASD5084787,1500
+102,10/19/2015,butan,aaa102,phone1247,ASD5140999,1500
+103,10/20/2015,burma,aaa103,phone1268,ASD5197211,1500
+104,10/21/2015,burma,aaa104,phone1289,ASD5253423,1500
+105,10/22/2015,butan,aaa105,phone1310,ASD5309635,1500
+106,10/23/2015,burma,aaa106,phone1331,ASD5365847,1500
+107,10/24/2015,butan,aaa107,phone1352,ASD5422059,1500
+108,10/25/2015,burma,aaa108,phone1373,ASD5478271,1500
+109,10/26/2015,burma,aaa109,phone1394,ASD5534483,1500
+110,10/27/2015,butan,aaa110,phone1415,ASD5590695,1500
+111,10/28/2015,burma,aaa111,phone1436,ASD5646907,1500
+112,10/29/2015,butan,aaa112,phone1457,ASD5703119,1500
+113,10/30/2015,burma,aaa113,phone1478,ASD5759331,1500
+114,10/31/2015,burma,aaa114,phone1499,ASD5815543,1500
+115,11/1/2015,butan,aaa115,phone1520,ASD5871755,1500
+116,11/2/2015,burma,aaa116,phone1541,ASD5927967,1500
+117,11/3/2015,butan,aaa117,phone1562,ASD5984179,1500
+118,11/4/2015,burma,aaa118,phone1583,ASD6040391,1500
+119,11/5/2015,burma,aaa119,phone1604,ASD6096603,1500
+120,11/6/2015,butan,aaa120,phone1625,ASD6152815,1500
+121,11/7/2015,burma,aaa121,phone1646,ASD6209027,1500
+122,11/8/2015,butan,aaa122,phone1667,ASD6265239,1500
+123,11/9/2015,burma,aaa123,phone1688,ASD6321451,1500
+124,11/10/2015,burma,aaa124,phone1709,ASD6377663,1500
+125,11/11/2015,butan,aaa125,phone1730,ASD6433875,1500
+126,11/12/2015,burma,aaa126,phone1751,ASD6490087,1500
+127,11/13/2015,butan,aaa127,phone1772,ASD6546299,1500
+128,11/14/2015,burma,aaa128,phone1793,ASD6602511,1500
+129,11/15/2015,burma,aaa129,phone1814,ASD6658723,1500
+130,11/16/2015,butan,aaa130,phone1835,ASD6714935,1500
+131,11/17/2015,burma,aaa131,phone1856,ASD6771147,1500
+132,11/18/2015,butan,aaa132,phone1877,ASD6827359,1500
+133,11/19/2015,burma,aaa133,phone1898,ASD6883571,1500
+134,11/20/2015,burma,aaa134,phone1919,ASD6939783,1500
+135,11/21/2015,butan,aaa135,phone1940,ASD6995995,1500
+136,11/22/2015,burma,aaa136,phone1961,ASD7052207,1500
+137,11/23/2015,butan,aaa137,phone1982,ASD7108419,1500
+138,11/24/2015,burma,aaa138,phone2003,ASD7164631,1500
+139,11/25/2015,burma,aaa139,phone2024,ASD7220843,1500
+140,11/26/2015,butan,aaa140,phone2045,ASD7277055,1500
+141,11/27/2015,burma,aaa141,phone2066,ASD7333267,1500
+142,11/28/2015,butan,aaa142,phone2087,ASD7389479,1500
+143,11/29/2015,burma,aaa143,phone2108,ASD7445691,1500
+144,11/30/2015,burma,aaa144,phone2129,ASD7501903,1500
+145,12/1/2015,butan,aaa145,phone2150,ASD7558115,1500
+146,12/2/2015,burma,aaa146,phone2171,ASD7614327,1500
+147,12/3/2015,butan,aaa147,phone2192,ASD7670539,1500
+148,12/4/2015,burma,aaa148,phone2213,ASD7726751,1500
+149,12/5/2015,burma,aaa149,phone2234,ASD7782963,1500
+150,12/6/2015,butan,aaa150,phone2255,ASD7839175,1500
+151,12/7/2015,burma,aaa151,phone2276,ASD7895387,1500
+152,12/8/2015,butan,aaa152,phone2297,ASD7951599,1500
+153,12/9/2015,burma,aaa153,phone2318,ASD8007811,1500
+154,12/10/2015,burma,aaa154,phone2339,ASD8064023,1500
+155,12/11/2015,butan,aaa155,phone2360,ASD8120235,1500
+156,12/12/2015,burma,aaa156,phone2381,ASD8176447,1500
+157,12/13/2015,butan,aaa157,phone2402,ASD8232659,1500
+158,12/14/2015,burma,aaa158,phone2423,ASD8288871,1500
+159,12/15/2015,burma,aaa159,phone2444,ASD8345083,1500
+160,12/16/2015,butan,aaa160,phone2465,ASD8401295,1500
+161,12/17/2015,burma,aaa161,phone2486,ASD8457507,1500
+162,12/18/2015,butan,aaa162,phone2507,ASD8513719,1500
+163,12/19/2015,burma,aaa163,phone2528,ASD8569931,1500
+164,12/20/2015,burma,aaa164,phone2549,ASD8626143,1500
+165,12/21/2015,butan,aaa165,phone2570,ASD8682355,1500
+166,12/22/2015,burma,aaa166,phone2591,ASD8738567,1500
+167,12/23/2015,butan,aaa167,phone2612,ASD8794779,1500
+168,12/24/2015,burma,aaa168,phone2633,ASD8850991,1500
+169,12/25/2015,burma,aaa169,phone2654,ASD8907203,1500
+170,12/26/2015,butan,aaa170,phone2675,ASD8963415,1500
+171,12/27/2015,burma,aaa171,phone2696,ASD9019627,1500
+172,12/28/2015,butan,aaa172,phone2717,ASD9075839,1500
+173,12/29/2015,burma,aaa173,phone2738,ASD9132051,1500
+174,12/30/2015,burma,aaa174,phone2759,ASD9188263,1500
+175,12/31/2015,butan,aaa175,phone2780,ASD9244475,1500
+176,1/1/2016,burma,aaa176,phone2801,ASD9300687,1500
+177,1/2/2016,butan,aaa177,phone2822,ASD9356899,1500
+178,1/3/2016,burma,aaa178,phone2843,ASD9413111,1500
+179,1/4/2016,burma,aaa179,phone2864,ASD9469323,1500
+180,1/5/2016,butan,aaa180,phone2885,ASD9525535,1500
+181,1/6/2016,burma,aaa181,phone2906,ASD9581747,1500
+182,1/7/2016,butan,aaa182,phone2927,ASD9637959,1500
+183,1/8/2016,burma,aaa183,phone2948,ASD9694171,1500
+184,1/9/2016,burma,aaa184,phone2969,ASD9750383,1500
+185,1/10/2016,butan,aaa185,phone2990,ASD9806595,1500
+186,1/11/2016,burma,aaa186,phone3011,ASD9862807,1500
+187,1/12/2016,butan,aaa187,phone3032,ASD9919019,1500
+188,1/13/2016,burma,aaa188,phone3053,ASD9975231,1500
+189,1/14/2016,burma,aaa189,phone3074,ASD10031443,1500
+190,1/15/2016,butan,aaa190,phone3095,ASD10087655,1500
+191,1/16/2016,burma,aaa191,phone3116,ASD10143867,1500
+192,1/17/2016,butan,aaa192,phone3137,ASD10200079,1500
+193,1/18/2016,burma,aaa193,phone3158,ASD10256291,1500
+194,1/19/2016,burma,aaa194,phone3179,ASD10312503,1500
+195,1/20/2016,butan,aaa195,phone3200,ASD10368715,1500
+196,1/21/2016,burma,aaa196,phone3221,ASD10424927,1500
+197,1/22/2016,butan,aaa197,phone3242,ASD10481139,1500
+198,1/23/2016,burma,aaa198,phone3263,ASD10537351,1500
+199,1/24/2016,burma,aaa199,phone3284,ASD10593563,1500
+200,1/25/2016,butan,aaa200,phone3305,ASD10649775,1500
+201,1/26/2016,burma,aaa201,phone3326,ASD10705987,1500
+202,1/27/2016,butan,aaa202,phone3347,ASD10762199,1500
+203,1/28/2016,burma,aaa203,phone3368,ASD10818411,1500
+204,1/29/2016,burma,aaa204,phone3389,ASD10874623,1500
+205,1/30/2016,butan,aaa205,phone3410,ASD10930835,1500
+206,1/31/2016,burma,aaa206,phone3431,ASD10987047,1500
+207,2/1/2016,butan,aaa207,phone3452,ASD11043259,1500
+208,2/2/2016,burma,aaa208,phone3473,ASD11099471,1500
+209,2/3/2016,burma,aaa209,phone3494,ASD11155683,1500
+210,2/4/2016,butan,aaa210,phone3515,ASD11211895,1500
+211,2/5/2016,burma,aaa211,phone3536,ASD11268107,1500
+212,2/6/2016,butan,aaa212,phone3557,ASD11324319,1500
+213,2/7/2016,burma,aaa213,phone3578,ASD11380531,1500
+214,2/8/2016,burma,aaa214,phone3599,ASD11436743,1500
+215,2/9/2016,butan,aaa215,phone3620,ASD11492955,1500
+216,2/10/2016,burma,aaa216,phone3641,ASD11549167,1500
+217,2/11/2016,butan,aaa217,phone3662,ASD11605379,1500
+218,2/12/2016,burma,aaa218,phone3683,ASD11661591,1500
+219,2/13/2016,burma,aaa219,phone3704,ASD11717803,1500
+220,2/14/2016,butan,aaa220,phone3725,ASD11774015,1500
+221,2/15/2016,burma,aaa221,phone3746,ASD11830227,1500
+222,2/16/2016,butan,aaa222,phone3767,ASD11886439,1500
+223,2/17/2016,burma,aaa223,phone3788,ASD11942651,1500
+224,2/18/2016,burma,aaa224,phone3809,ASD11998863,1500
+225,2/19/2016,butan,aaa225,phone3830,ASD12055075,1500
+226,2/20/2016,burma,aaa226,phone3851,ASD12111287,1500
+227,2/21/2016,butan,aaa227,phone3872,ASD12167499,1500
+228,2/22/2016,burma,aaa228,phone3893,ASD12223711,1500
+229,2/23/2016,burma,aaa229,phone3914,ASD12279923,1500
+230,2/24/2016,butan,aaa230,phone3935,ASD12336135,1500
+231,2/25/2016,burma,aaa231,phone3956,ASD12392347,1500
+232,2/26/2016,butan,aaa232,phone3977,ASD12448559,1500
+233,2/27/2016,burma,aaa233,phone3998,ASD12504771,1500
+234,2/28/2016,burma,aaa234,phone4019,ASD12560983,1500
+235,2/29/2016,butan,aaa235,phone4040,ASD12617195,1500
+236,3/1/2016,burma,aaa236,phone4061,ASD12673407,1500
+237,3/2/2016,butan,aaa237,phone4082,ASD12729619,1500
+238,3/3/2016,burma,aaa238,phone4103,ASD12785831,1500
+239,3/4/2016,burma,aaa239,phone4124,ASD12842043,1500
+240,3/5/2016,butan,aaa240,phone4145,ASD12898255,1500
+241,3/6/2016,burma,aaa241,phone4166,ASD12954467,1500
+242,3/7/2016,butan,aaa242,phone4187,ASD13010679,1500
+243,3/8/2016,burma,aaa243,phone4208,ASD13066891,1500
+244,3/9/2016,burma,aaa244,phone4229,ASD13123103,1500
+245,3/10/2016,butan,aaa245,phone4250,ASD13179315,1500
+246,3/11/2016,burma,aaa246,phone4271,ASD13235527,1500
+247,3/12/2016,butan,aaa247,phone4292,ASD13291739,1500
+248,3/13/2016,burma,aaa248,phone4313,ASD13347951,1500
+249,3/14/2016,burma,aaa249,phone4334,ASD13404163,1500
+250,3/15/2016,butan,aaa250,phone4355,ASD13460375,1500
+251,3/16/2016,burma,aaa251,phone4376,ASD13516587,1500
+252,3/17/2016,butan,aaa252,phone4397,ASD13572799,1500
+253,3/18/2016,burma,aaa253,phone4418,ASD13629011,1500
+254,3/19/2016,burma,aaa254,phone4439,ASD13685223,1500
+255,3/20/2016,butan,aaa255,phone4460,ASD13741435,1500
+256,3/21/2016,burma,aaa256,phone4481,ASD13797647,1500
+257,3/22/2016,butan,aaa257,phone4502,ASD13853859,1500
+258,3/23/2016,burma,aaa258,phone4523,ASD13910071,1500
+259,3/24/2016,burma,aaa259,phone4544,ASD13966283,1500
+260,3/25/2016,butan,aaa260,phone4565,ASD14022495,1500
+261,3/26/2016,burma,aaa261,phone4586,ASD14078707,1500
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/compaction/compactioncard2_forhive.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/compaction/compactioncard2_forhive.csv b/integration/spark-common-test/src/test/resources/compaction/compactioncard2_forhive.csv
new file mode 100644
index 0000000..7d7673e
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/compaction/compactioncard2_forhive.csv
@@ -0,0 +1,256 @@
+6,7/28/2015,newzealand,aaa6,phone294,ASD59961,15005
+7,7/29/2015,mexico,aaa7,phone610,ASD14875,15006
+8,7/30/2015,westindies,aaa8,phone1848,ASD57308,15007
+9,7/18/2015,burma,aaa9,phone706,ASD86717,15008
+10,7/19/2015,butan,aaa10,phone685,ASD30505,1500
+11,7/20/2015,burma,aaa11,phone664,ASD25707,1500
+12,7/21/2015,butan,aaa12,phone643,ASD81919,1500
+13,7/22/2015,burma,aaa13,phone622,ASD138131,1500
+14,7/23/2015,burma,aaa14,phone601,ASD194343,1500
+15,7/24/2015,butan,aaa15,phone580,ASD250555,1500
+16,7/25/2015,burma,aaa16,phone559,ASD306767,1500
+17,7/26/2015,butan,aaa17,phone538,ASD362979,1500
+18,7/27/2015,burma,aaa18,phone517,ASD419191,1500
+19,7/28/2015,burma,aaa19,phone496,ASD475403,1500
+20,7/29/2015,butan,aaa20,phone475,ASD531615,1500
+21,7/30/2015,burma,aaa21,phone454,ASD587827,1500
+22,7/31/2015,butan,aaa22,phone433,ASD644039,1500
+23,8/1/2015,burma,aaa23,phone412,ASD700251,1500
+24,8/2/2015,burma,aaa24,phone391,ASD756463,1500
+25,8/3/2015,butan,aaa25,phone370,ASD812675,1500
+26,8/4/2015,burma,aaa26,phone349,ASD868887,1500
+27,8/5/2015,butan,aaa27,phone328,ASD925099,1500
+28,8/6/2015,burma,aaa28,phone307,ASD981311,1500
+29,8/7/2015,burma,aaa29,phone286,ASD1037523,1500
+30,8/8/2015,butan,aaa30,phone265,ASD1093735,1500
+31,8/9/2015,burma,aaa31,phone244,ASD1149947,1500
+32,8/10/2015,butan,aaa32,phone223,ASD1206159,1500
+33,8/11/2015,burma,aaa33,phone202,ASD1262371,1500
+34,8/12/2015,burma,aaa34,phone181,ASD1318583,1500
+35,8/13/2015,butan,aaa35,phone160,ASD1374795,1500
+36,8/14/2015,burma,aaa36,phone139,ASD1431007,1500
+37,8/15/2015,butan,aaa37,phone118,ASD1487219,1500
+38,8/16/2015,burma,aaa38,phone97,ASD1543431,1500
+39,8/17/2015,burma,aaa39,phone76,ASD1599643,1500
+40,8/18/2015,butan,aaa40,phone55,ASD1655855,1500
+41,8/19/2015,burma,aaa41,phone34,ASD1712067,1500
+42,8/20/2015,butan,aaa42,phone13,ASD1768279,1500
+43,8/21/2015,burma,aaa43,phone8,ASD1824491,1500
+44,8/22/2015,burma,aaa44,phone29,ASD1880703,1500
+45,8/23/2015,butan,aaa45,phone50,ASD1936915,1500
+46,8/24/2015,burma,aaa46,phone71,ASD1993127,1500
+47,8/25/2015,butan,aaa47,phone92,ASD2049339,1500
+48,8/26/2015,burma,aaa48,phone113,ASD2105551,1500
+49,8/27/2015,burma,aaa49,phone134,ASD2161763,1500
+50,8/28/2015,butan,aaa50,phone155,ASD2217975,1500
+51,8/29/2015,burma,aaa51,phone176,ASD2274187,1500
+52,8/30/2015,butan,aaa52,phone197,ASD2330399,1500
+53,8/31/2015,burma,aaa53,phone218,ASD2386611,1500
+54,9/1/2015,burma,aaa54,phone239,ASD2442823,1500
+55,9/2/2015,butan,aaa55,phone260,ASD2499035,1500
+56,9/3/2015,burma,aaa56,phone281,ASD2555247,1500
+57,9/4/2015,butan,aaa57,phone302,ASD2611459,1500
+58,9/5/2015,burma,aaa58,phone323,ASD2667671,1500
+59,9/6/2015,burma,aaa59,phone344,ASD2723883,1500
+60,9/7/2015,butan,aaa60,phone365,ASD2780095,1500
+61,9/8/2015,burma,aaa61,phone386,ASD2836307,1500
+62,9/9/2015,butan,aaa62,phone407,ASD2892519,1500
+63,9/10/2015,burma,aaa63,phone428,ASD2948731,1500
+64,9/11/2015,burma,aaa64,phone449,ASD3004943,1500
+65,9/12/2015,butan,aaa65,phone470,ASD3061155,1500
+66,9/13/2015,burma,aaa66,phone491,ASD3117367,1500
+67,9/14/2015,butan,aaa67,phone512,ASD3173579,1500
+68,9/15/2015,burma,aaa68,phone533,ASD3229791,1500
+69,9/16/2015,burma,aaa69,phone554,ASD3286003,1500
+70,9/17/2015,butan,aaa70,phone575,ASD3342215,1500
+71,9/18/2015,burma,aaa71,phone596,ASD3398427,1500
+72,9/19/2015,butan,aaa72,phone617,ASD3454639,1500
+73,9/20/2015,burma,aaa73,phone638,ASD3510851,1500
+74,9/21/2015,burma,aaa74,phone659,ASD3567063,1500
+75,9/22/2015,butan,aaa75,phone680,ASD3623275,1500
+76,9/23/2015,burma,aaa76,phone701,ASD3679487,1500
+77,9/24/2015,butan,aaa77,phone722,ASD3735699,1500
+78,9/25/2015,burma,aaa78,phone743,ASD3791911,1500
+79,9/26/2015,burma,aaa79,phone764,ASD3848123,1500
+80,9/27/2015,butan,aaa80,phone785,ASD3904335,1500
+81,9/28/2015,burma,aaa81,phone806,ASD3960547,1500
+82,9/29/2015,butan,aaa82,phone827,ASD4016759,1500
+83,9/30/2015,burma,aaa83,phone848,ASD4072971,1500
+84,10/1/2015,burma,aaa84,phone869,ASD4129183,1500
+85,10/2/2015,butan,aaa85,phone890,ASD4185395,1500
+86,10/3/2015,burma,aaa86,phone911,ASD4241607,1500
+87,10/4/2015,butan,aaa87,phone932,ASD4297819,1500
+88,10/5/2015,burma,aaa88,phone953,ASD4354031,1500
+89,10/6/2015,burma,aaa89,phone974,ASD4410243,1500
+90,10/7/2015,butan,aaa90,phone995,ASD4466455,1500
+91,10/8/2015,burma,aaa91,phone1016,ASD4522667,1500
+92,10/9/2015,butan,aaa92,phone1037,ASD4578879,1500
+93,10/10/2015,burma,aaa93,phone1058,ASD4635091,1500
+94,10/11/2015,burma,aaa94,phone1079,ASD4691303,1500
+95,10/12/2015,butan,aaa95,phone1100,ASD4747515,1500
+96,10/13/2015,burma,aaa96,phone1121,ASD4803727,1500
+97,10/14/2015,butan,aaa97,phone1142,ASD4859939,1500
+98,10/15/2015,burma,aaa98,phone1163,ASD4916151,1500
+99,10/16/2015,burma,aaa99,phone1184,ASD4972363,1500
+100,10/17/2015,butan,aaa100,phone1205,ASD5028575,1500
+101,10/18/2015,burma,aaa101,phone1226,ASD5084787,1500
+102,10/19/2015,butan,aaa102,phone1247,ASD5140999,1500
+103,10/20/2015,burma,aaa103,phone1268,ASD5197211,1500
+104,10/21/2015,burma,aaa104,phone1289,ASD5253423,1500
+105,10/22/2015,butan,aaa105,phone1310,ASD5309635,1500
+106,10/23/2015,burma,aaa106,phone1331,ASD5365847,1500
+107,10/24/2015,butan,aaa107,phone1352,ASD5422059,1500
+108,10/25/2015,burma,aaa108,phone1373,ASD5478271,1500
+109,10/26/2015,burma,aaa109,phone1394,ASD5534483,1500
+110,10/27/2015,butan,aaa110,phone1415,ASD5590695,1500
+111,10/28/2015,burma,aaa111,phone1436,ASD5646907,1500
+112,10/29/2015,butan,aaa112,phone1457,ASD5703119,1500
+113,10/30/2015,burma,aaa113,phone1478,ASD5759331,1500
+114,10/31/2015,burma,aaa114,phone1499,ASD5815543,1500
+115,11/1/2015,butan,aaa115,phone1520,ASD5871755,1500
+116,11/2/2015,burma,aaa116,phone1541,ASD5927967,1500
+117,11/3/2015,butan,aaa117,phone1562,ASD5984179,1500
+118,11/4/2015,burma,aaa118,phone1583,ASD6040391,1500
+119,11/5/2015,burma,aaa119,phone1604,ASD6096603,1500
+120,11/6/2015,butan,aaa120,phone1625,ASD6152815,1500
+121,11/7/2015,burma,aaa121,phone1646,ASD6209027,1500
+122,11/8/2015,butan,aaa122,phone1667,ASD6265239,1500
+123,11/9/2015,burma,aaa123,phone1688,ASD6321451,1500
+124,11/10/2015,burma,aaa124,phone1709,ASD6377663,1500
+125,11/11/2015,butan,aaa125,phone1730,ASD6433875,1500
+126,11/12/2015,burma,aaa126,phone1751,ASD6490087,1500
+127,11/13/2015,butan,aaa127,phone1772,ASD6546299,1500
+128,11/14/2015,burma,aaa128,phone1793,ASD6602511,1500
+129,11/15/2015,burma,aaa129,phone1814,ASD6658723,1500
+130,11/16/2015,butan,aaa130,phone1835,ASD6714935,1500
+131,11/17/2015,burma,aaa131,phone1856,ASD6771147,1500
+132,11/18/2015,butan,aaa132,phone1877,ASD6827359,1500
+133,11/19/2015,burma,aaa133,phone1898,ASD6883571,1500
+134,11/20/2015,burma,aaa134,phone1919,ASD6939783,1500
+135,11/21/2015,butan,aaa135,phone1940,ASD6995995,1500
+136,11/22/2015,burma,aaa136,phone1961,ASD7052207,1500
+137,11/23/2015,butan,aaa137,phone1982,ASD7108419,1500
+138,11/24/2015,burma,aaa138,phone2003,ASD7164631,1500
+139,11/25/2015,burma,aaa139,phone2024,ASD7220843,1500
+140,11/26/2015,butan,aaa140,phone2045,ASD7277055,1500
+141,11/27/2015,burma,aaa141,phone2066,ASD7333267,1500
+142,11/28/2015,butan,aaa142,phone2087,ASD7389479,1500
+143,11/29/2015,burma,aaa143,phone2108,ASD7445691,1500
+144,11/30/2015,burma,aaa144,phone2129,ASD7501903,1500
+145,12/1/2015,butan,aaa145,phone2150,ASD7558115,1500
+146,12/2/2015,burma,aaa146,phone2171,ASD7614327,1500
+147,12/3/2015,butan,aaa147,phone2192,ASD7670539,1500
+148,12/4/2015,burma,aaa148,phone2213,ASD7726751,1500
+149,12/5/2015,burma,aaa149,phone2234,ASD7782963,1500
+150,12/6/2015,butan,aaa150,phone2255,ASD7839175,1500
+151,12/7/2015,burma,aaa151,phone2276,ASD7895387,1500
+152,12/8/2015,butan,aaa152,phone2297,ASD7951599,1500
+153,12/9/2015,burma,aaa153,phone2318,ASD8007811,1500
+154,12/10/2015,burma,aaa154,phone2339,ASD8064023,1500
+155,12/11/2015,butan,aaa155,phone2360,ASD8120235,1500
+156,12/12/2015,burma,aaa156,phone2381,ASD8176447,1500
+157,12/13/2015,butan,aaa157,phone2402,ASD8232659,1500
+158,12/14/2015,burma,aaa158,phone2423,ASD8288871,1500
+159,12/15/2015,burma,aaa159,phone2444,ASD8345083,1500
+160,12/16/2015,butan,aaa160,phone2465,ASD8401295,1500
+161,12/17/2015,burma,aaa161,phone2486,ASD8457507,1500
+162,12/18/2015,butan,aaa162,phone2507,ASD8513719,1500
+163,12/19/2015,burma,aaa163,phone2528,ASD8569931,1500
+164,12/20/2015,burma,aaa164,phone2549,ASD8626143,1500
+165,12/21/2015,butan,aaa165,phone2570,ASD8682355,1500
+166,12/22/2015,burma,aaa166,phone2591,ASD8738567,1500
+167,12/23/2015,butan,aaa167,phone2612,ASD8794779,1500
+168,12/24/2015,burma,aaa168,phone2633,ASD8850991,1500
+169,12/25/2015,burma,aaa169,phone2654,ASD8907203,1500
+170,12/26/2015,butan,aaa170,phone2675,ASD8963415,1500
+171,12/27/2015,burma,aaa171,phone2696,ASD9019627,1500
+172,12/28/2015,butan,aaa172,phone2717,ASD9075839,1500
+173,12/29/2015,burma,aaa173,phone2738,ASD9132051,1500
+174,12/30/2015,burma,aaa174,phone2759,ASD9188263,1500
+175,12/31/2015,butan,aaa175,phone2780,ASD9244475,1500
+176,1/1/2016,burma,aaa176,phone2801,ASD9300687,1500
+177,1/2/2016,butan,aaa177,phone2822,ASD9356899,1500
+178,1/3/2016,burma,aaa178,phone2843,ASD9413111,1500
+179,1/4/2016,burma,aaa179,phone2864,ASD9469323,1500
+180,1/5/2016,butan,aaa180,phone2885,ASD9525535,1500
+181,1/6/2016,burma,aaa181,phone2906,ASD9581747,1500
+182,1/7/2016,butan,aaa182,phone2927,ASD9637959,1500
+183,1/8/2016,burma,aaa183,phone2948,ASD9694171,1500
+184,1/9/2016,burma,aaa184,phone2969,ASD9750383,1500
+185,1/10/2016,butan,aaa185,phone2990,ASD9806595,1500
+186,1/11/2016,burma,aaa186,phone3011,ASD9862807,1500
+187,1/12/2016,butan,aaa187,phone3032,ASD9919019,1500
+188,1/13/2016,burma,aaa188,phone3053,ASD9975231,1500
+189,1/14/2016,burma,aaa189,phone3074,ASD10031443,1500
+190,1/15/2016,butan,aaa190,phone3095,ASD10087655,1500
+191,1/16/2016,burma,aaa191,phone3116,ASD10143867,1500
+192,1/17/2016,butan,aaa192,phone3137,ASD10200079,1500
+193,1/18/2016,burma,aaa193,phone3158,ASD10256291,1500
+194,1/19/2016,burma,aaa194,phone3179,ASD10312503,1500
+195,1/20/2016,butan,aaa195,phone3200,ASD10368715,1500
+196,1/21/2016,burma,aaa196,phone3221,ASD10424927,1500
+197,1/22/2016,butan,aaa197,phone3242,ASD10481139,1500
+198,1/23/2016,burma,aaa198,phone3263,ASD10537351,1500
+199,1/24/2016,burma,aaa199,phone3284,ASD10593563,1500
+200,1/25/2016,butan,aaa200,phone3305,ASD10649775,1500
+201,1/26/2016,burma,aaa201,phone3326,ASD10705987,1500
+202,1/27/2016,butan,aaa202,phone3347,ASD10762199,1500
+203,1/28/2016,burma,aaa203,phone3368,ASD10818411,1500
+204,1/29/2016,burma,aaa204,phone3389,ASD10874623,1500
+205,1/30/2016,butan,aaa205,phone3410,ASD10930835,1500
+206,1/31/2016,burma,aaa206,phone3431,ASD10987047,1500
+207,2/1/2016,butan,aaa207,phone3452,ASD11043259,1500
+208,2/2/2016,burma,aaa208,phone3473,ASD11099471,1500
+209,2/3/2016,burma,aaa209,phone3494,ASD11155683,1500
+210,2/4/2016,butan,aaa210,phone3515,ASD11211895,1500
+211,2/5/2016,burma,aaa211,phone3536,ASD11268107,1500
+212,2/6/2016,butan,aaa212,phone3557,ASD11324319,1500
+213,2/7/2016,burma,aaa213,phone3578,ASD11380531,1500
+214,2/8/2016,burma,aaa214,phone3599,ASD11436743,1500
+215,2/9/2016,butan,aaa215,phone3620,ASD11492955,1500
+216,2/10/2016,burma,aaa216,phone3641,ASD11549167,1500
+217,2/11/2016,butan,aaa217,phone3662,ASD11605379,1500
+218,2/12/2016,burma,aaa218,phone3683,ASD11661591,1500
+219,2/13/2016,burma,aaa219,phone3704,ASD11717803,1500
+220,2/14/2016,butan,aaa220,phone3725,ASD11774015,1500
+221,2/15/2016,burma,aaa221,phone3746,ASD11830227,1500
+222,2/16/2016,butan,aaa222,phone3767,ASD11886439,1500
+223,2/17/2016,burma,aaa223,phone3788,ASD11942651,1500
+224,2/18/2016,burma,aaa224,phone3809,ASD11998863,1500
+225,2/19/2016,butan,aaa225,phone3830,ASD12055075,1500
+226,2/20/2016,burma,aaa226,phone3851,ASD12111287,1500
+227,2/21/2016,butan,aaa227,phone3872,ASD12167499,1500
+228,2/22/2016,burma,aaa228,phone3893,ASD12223711,1500
+229,2/23/2016,burma,aaa229,phone3914,ASD12279923,1500
+230,2/24/2016,butan,aaa230,phone3935,ASD12336135,1500
+231,2/25/2016,burma,aaa231,phone3956,ASD12392347,1500
+232,2/26/2016,butan,aaa232,phone3977,ASD12448559,1500
+233,2/27/2016,burma,aaa233,phone3998,ASD12504771,1500
+234,2/28/2016,burma,aaa234,phone4019,ASD12560983,1500
+235,2/29/2016,butan,aaa235,phone4040,ASD12617195,1500
+236,3/1/2016,burma,aaa236,phone4061,ASD12673407,1500
+237,3/2/2016,butan,aaa237,phone4082,ASD12729619,1500
+238,3/3/2016,burma,aaa238,phone4103,ASD12785831,1500
+239,3/4/2016,burma,aaa239,phone4124,ASD12842043,1500
+240,3/5/2016,butan,aaa240,phone4145,ASD12898255,1500
+241,3/6/2016,burma,aaa241,phone4166,ASD12954467,1500
+242,3/7/2016,butan,aaa242,phone4187,ASD13010679,1500
+243,3/8/2016,burma,aaa243,phone4208,ASD13066891,1500
+244,3/9/2016,burma,aaa244,phone4229,ASD13123103,1500
+245,3/10/2016,butan,aaa245,phone4250,ASD13179315,1500
+246,3/11/2016,burma,aaa246,phone4271,ASD13235527,1500
+247,3/12/2016,butan,aaa247,phone4292,ASD13291739,1500
+248,3/13/2016,burma,aaa248,phone4313,ASD13347951,1500
+249,3/14/2016,burma,aaa249,phone4334,ASD13404163,1500
+250,3/15/2016,butan,aaa250,phone4355,ASD13460375,1500
+251,3/16/2016,burma,aaa251,phone4376,ASD13516587,1500
+252,3/17/2016,butan,aaa252,phone4397,ASD13572799,1500
+253,3/18/2016,burma,aaa253,phone4418,ASD13629011,1500
+254,3/19/2016,burma,aaa254,phone4439,ASD13685223,1500
+255,3/20/2016,butan,aaa255,phone4460,ASD13741435,1500
+256,3/21/2016,burma,aaa256,phone4481,ASD13797647,1500
+257,3/22/2016,butan,aaa257,phone4502,ASD13853859,1500
+258,3/23/2016,burma,aaa258,phone4523,ASD13910071,1500
+259,3/24/2016,burma,aaa259,phone4544,ASD13966283,1500
+260,3/25/2016,butan,aaa260,phone4565,ASD14022495,1500
+261,3/26/2016,burma,aaa261,phone4586,ASD14078707,1500
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/complexTypeDecimal.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/complexTypeDecimal.csv b/integration/spark-common-test/src/test/resources/complexTypeDecimal.csv
new file mode 100644
index 0000000..8c90d27
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/complexTypeDecimal.csv
@@ -0,0 +1,9 @@
+ID,date,country,name,phonetype,serialname,salary,complex
+1.2,2015/7/23,china,aaa1,phone197,ASD69643,15000,3.113$3.33
+2,2015/7/24,china,aaa2,phone756,ASD42892,15001,3.123$7.33
+4.3,2015/7/26,china,aaa4,phone2435,ASD66902,15003,3.123$56.33
+5,2015/7/27,china,aaa5,phone2441,ASD90633,15004,3.133$5.33
+6.5,2015/7/28,china,aaa6,phone294,ASD59961,15005,3.133$54.33
+8,2015/7/30,china,aaa8,phone1848,ASD57308,15007,32.13$56.33
+9.1,2015/7/18,china,aaa9,phone706,ASD86717,15008,3.213$44.33
+10,2015/7/19,usa,aaa10,phone685,ASD30505,15009,32.13$33.33
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/complexTypeDecimalNested.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/complexTypeDecimalNested.csv b/integration/spark-common-test/src/test/resources/complexTypeDecimalNested.csv
new file mode 100644
index 0000000..4951bf8
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/complexTypeDecimalNested.csv
@@ -0,0 +1,9 @@
+ID,date,country,name,phonetype,serialname,salary,complex
+1.2,2015/7/23,china,aaa1,phone197,ASD69643,15000,3.113:imei$3.33:imsi
+2,2015/7/24,china,aaa2,phone756,ASD42892,15001,3.123:imei$7.33:imsi
+4.3,2015/7/26,china,aaa4,phone2435,ASD66902,15003,3.123:imei$56.33:imsi
+5,2015/7/27,china,aaa5,phone2441,ASD90633,15004,3.133:imei$5.33:imsi
+6.5,2015/7/28,china,aaa6,phone294,ASD59961,15005,3.133:imei$54.33:imsi
+8,2015/7/30,china,aaa8,phone1848,ASD57308,15007,32.13:imei$56.33:imsi
+9.1,2015/7/18,china,aaa9,phone706,ASD86717,15008,3.213:imei$44.33:imsi
+10,2015/7/19,usa,aaa10,phone685,ASD30505,15009,32.13:imei$33.33:imsi
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/af2f204e/integration/spark-common-test/src/test/resources/complexTypeDecimalNestedHive.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/complexTypeDecimalNestedHive.csv b/integration/spark-common-test/src/test/resources/complexTypeDecimalNestedHive.csv
new file mode 100644
index 0000000..ae67723
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/complexTypeDecimalNestedHive.csv
@@ -0,0 +1,8 @@
+1.2,2015/7/23,china,aaa1,phone197,ASD69643,15000,3.113:imei$3.33:imsi
+2,2015/7/24,china,aaa2,phone756,ASD42892,15001,3.123:imei$7.33:imsi
+4.3,2015/7/26,china,aaa4,phone2435,ASD66902,15003,3.123:imei$56.33:imsi
+5,2015/7/27,china,aaa5,phone2441,ASD90633,15004,3.133:imei$5.33:imsi
+6.5,2015/7/28,china,aaa6,phone294,ASD59961,15005,3.133:imei$54.33:imsi
+8,2015/7/30,china,aaa8,phone1848,ASD57308,15007,32.13:imei$56.33:imsi
+9.1,2015/7/18,china,aaa9,phone706,ASD86717,15008,3.213:imei$44.33:imsi
+10,2015/7/19,usa,aaa10,phone685,ASD30505,15009,32.13:imei$33.33:imsi
\ No newline at end of file