You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/05/02 14:12:25 UTC
[01/50] [abbrv] incubator-carbondata git commit: [CARBONDATA-943]
Added check to display error message in catch block. This closes #808 [Forced
Update!]
Repository: incubator-carbondata
Updated Branches:
refs/heads/12-dev 1513700b5 -> 62d1cb37b (forced update)
[CARBONDATA-943] Added check to display error message in catch block. This closes #808
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/174bfadc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/174bfadc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/174bfadc
Branch: refs/heads/12-dev
Commit: 174bfadc77bef96db7d9be5536e777882e356038
Parents: 2ac60a0 c331854
Author: ravipesala <ra...@gmail.com>
Authored: Sat Apr 22 00:01:36 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Sat Apr 22 00:01:36 2017 +0530
----------------------------------------------------------------------
.../spark/exception/MalformedCarbonCommandException.java | 2 ++
.../apache/spark/sql/parser/CarbonSparkSqlParser.scala | 11 ++++++++++-
2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
[19/50] [abbrv] incubator-carbondata git commit: fix code style
Posted by ja...@apache.org.
fix code style
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/3e8416d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/3e8416d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/3e8416d9
Branch: refs/heads/12-dev
Commit: 3e8416d9b3a54899c06de3e53f829931157c038d
Parents: e579694
Author: jackylk <ja...@huawei.com>
Authored: Wed Apr 26 10:43:40 2017 -0700
Committer: jackylk <ja...@huawei.com>
Committed: Wed Apr 26 10:43:40 2017 -0700
----------------------------------------------------------------------
.../scala/org/apache/carbondata/examples/AlterTableExample.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3e8416d9/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
index 1d01de9..f37a83d 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
@@ -19,10 +19,10 @@ package org.apache.carbondata.examples
import java.io.File
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
import org.apache.spark.sql.SparkSession
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
object AlterTableExample {
[23/50] [abbrv] incubator-carbondata git commit: Range Filter Check
Rectification
Posted by ja...@apache.org.
Range Filter Check Rectification
Modify ByteUtil Compare
Direct Dictionary test Case Addition
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/105e3a0d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/105e3a0d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/105e3a0d
Branch: refs/heads/12-dev
Commit: 105e3a0d7b58282f7228e4d6c96fd2dc97ef973f
Parents: 2acb310
Author: sounakr <so...@gmail.com>
Authored: Mon Apr 24 14:26:45 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Apr 27 15:16:57 2017 +0530
----------------------------------------------------------------------
.../executer/RangeValueFilterExecuterImpl.java | 43 ++-----
.../apache/carbondata/core/util/ByteUtil.java | 19 ++-
.../src/test/resources/rangedatasample.csv | 12 +-
.../test/resources/rangenodictionarycompare.csv | 21 ++++
.../RangeFilterAllDataTypesTestCases.scala | 116 ++++++++++++++++++-
.../detailquery/RangeFilterTestCase.scala | 80 ++++++++++++-
6 files changed, 245 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/105e3a0d/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
index 9dc6a99..a20f414 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
@@ -24,7 +24,6 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.scan.expression.Expression;
@@ -117,7 +116,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
*/
private void ifDefaultValueMatchesFilter() {
isDefaultValuePresentInFilter = false;
- if (this.isDimensionPresentInCurrentBlock) {
+ if (!this.isDimensionPresentInCurrentBlock) {
CarbonDimension dimension = this.dimColEvaluatorInfo.getDimension();
byte[] defaultValue = dimension.getDefaultValue();
if (null != defaultValue) {
@@ -354,8 +353,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
int numerOfRows) {
- if (dimensionColumnDataChunk.isExplicitSorted()
- && dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
+ if (dimensionColumnDataChunk.isExplicitSorted()) {
return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numerOfRows);
}
return setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows);
@@ -401,13 +399,9 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
// Method will compare the tentative index value after binary search, this tentative
// index needs to be compared by the filter member if its >= filter then from that
// index the bitset will be considered for filtering process.
- if ((greaterThanExp == true) && (ByteUtil.compare(filterValues[0], dimensionColumnDataChunk
+ if ((ByteUtil.compare(filterValues[0], dimensionColumnDataChunk
.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))) > 0) {
start = start + 1;
- } else if ((greaterThanEqualExp == true) && (ByteUtil.compare(filterValues[0],
- dimensionColumnDataChunk
- .getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))) >= 0) {
- start = start + 1;
}
}
@@ -421,6 +415,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
start = CarbonUtil
.getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
filterValues[1], lessThanEqualExp);
+
if (lessThanExp == true && start >= 0) {
start =
CarbonUtil.nextLesserValueToTarget(start, dimensionColumnDataChunk, filterValues[1]);
@@ -431,17 +426,12 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
if (start == numerOfRows) {
start = start - 1;
}
- // Method will compare the tentative index value after binary search, this tentative
- // index needs to be compared by the filter member if its >= filter then from that
- // index the bitset will be considered for filtering process.
- if ((lessThanExp == true) && (ByteUtil.compare(filterValues[1],
+ // In case the start is less than 0, then positive value of start is pointing to the next
+ // value of the searched key. So move to the previous one.
+ if ((ByteUtil.compare(filterValues[1],
dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
< 0)) {
start = start - 1;
- } else if ((lessThanEqualExp == true) && (ByteUtil.compare(filterValues[1],
- dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
- <= 0)) {
- start = start - 1;
}
}
endMax = start;
@@ -511,14 +501,9 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
// Method will compare the tentative index value after binary search, this tentative
// index needs to be compared by the filter member if its >= filter then from that
// index the bitset will be considered for filtering process.
- if ((greaterThanExp == true)
- && (ByteUtil.compare(filterValues[0], dimensionColumnDataChunk.getChunkData(start)))
+ if ((ByteUtil.compare(filterValues[0], dimensionColumnDataChunk.getChunkData(start)))
> 0) {
start = start + 1;
- } else if ((greaterThanEqualExp == true)
- && (ByteUtil.compare(filterValues[0], dimensionColumnDataChunk.getChunkData(start)))
- >= 0) {
- start = start + 1;
}
}
startMin = start;
@@ -541,14 +526,10 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
if (start == numerOfRows) {
start = start - 1;
}
- // Method will compare the tentative index value after binary search, this tentative
- // index needs to be compared by the filter member if its >= filter then from that
- // index the bitset will be considered for filtering process.
- if ((lessThanExp == true) && (
- ByteUtil.compare(filterValues[1], dimensionColumnDataChunk.getChunkData(start)) < 0)) {
- start = start - 1;
- } else if ((lessThanEqualExp == true) && (
- ByteUtil.compare(filterValues[1], dimensionColumnDataChunk.getChunkData(start)) <= 0)) {
+ // In case the start is less than 0, then positive value of start is pointing to the next
+ // value of the searched key. So move to the previous one.
+ if ((ByteUtil.compare(filterValues[1], dimensionColumnDataChunk.getChunkData(start))
+ < 0)) {
start = start - 1;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/105e3a0d/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
index 7e7e01b..16c6760 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
@@ -45,19 +45,16 @@ public final class ByteUtil {
if (buffer1 == buffer2) {
return 0;
}
- // Bring WritableComparator code local
- int i = 0;
- int j = 0;
- for (; i < buffer1.length && j < buffer2.length; i++, j++) {
- int a = (buffer1[i] & 0xff);
- int b = (buffer2[j] & 0xff);
- if (a != b) {
- return a - b;
- }
- }
- return 0;
+ int len1 = buffer1.length;
+ int len2 = buffer2.length;
+ int offset1 = 0;
+ int offset2 = 0;
+ // Call UnsafeComparer compareTo for comparision.
+ return ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(buffer1, offset1, len1, buffer2, offset2, len2);
}
+
/**
* covert the long[] to int[]
*
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/105e3a0d/integration/spark-common-test/src/test/resources/rangedatasample.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/rangedatasample.csv b/integration/spark-common-test/src/test/resources/rangedatasample.csv
index b729470..263d26c 100644
--- a/integration/spark-common-test/src/test/resources/rangedatasample.csv
+++ b/integration/spark-common-test/src/test/resources/rangedatasample.csv
@@ -9,4 +9,14 @@ empno,doj,salary
18,2016-03-14 15:00:16,1046
19,2016-03-14 15:00:17,1047
20,2016-03-14 15:00:18,1048
-21,,1040
+21,2016-03-14 15:00:20,5040
+22,2016-03-14 15:00:25,1040
+23,2016-03-14 15:00:31,1041
+24,2016-03-14 15:00:38,1042
+25,2016-03-14 15:00:39,1043
+26,2016-03-14 15:00:19,1044
+27,2016-03-14 15:00:49,1045
+28,2016-03-14 15:00:50,1046
+29,2016-03-14 15:00:24,1047
+30,2016-03-14 15:00:35,1048
+31,,1040
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/105e3a0d/integration/spark-common-test/src/test/resources/rangenodictionarycompare.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/rangenodictionarycompare.csv b/integration/spark-common-test/src/test/resources/rangenodictionarycompare.csv
new file mode 100644
index 0000000..1cd6c5c
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/rangenodictionarycompare.csv
@@ -0,0 +1,21 @@
+empno,empname,designation,doj,workgroupcategory,workgroupcategoryname,deptno,deptname,projectcode,projectjoindate,projectenddate,attendance,utilization,salary
+11,100,SE,17-01-2007,1,developer,10,network,928478,17-02-2007,29-11-2016,96,96,5040
+12,101,SSE,29-05-2008,1,developer,11,protocol,928378,29-06-2008,30-12-2016,85,95,7124
+13,108,TPL,07-07-2009,2,tester,10,network,928478,07-08-2009,30-12-2016,88,99,9054
+14,109,SA,29-12-2010,3,manager,11,protocol,928278,29-01-2011,29-06-2016,77,92,11248
+15,11111,SSA,09-11-2011,1,developer,12,security,928375,09-12-2011,29-05-2016,99,91,13245
+16,107,SE,14-10-2012,1,developer,13,configManagement,928478,14-11-2012,29-12-2016,86,93,5040
+17,1111119,PL,22-09-2013,2,tester,12,security,928778,22-10-2013,15-11-2016,78,97,9574
+18,1111118,TL,15-08-2014,2,tester,14,Learning,928176,15-09-2014,29-05-2016,84,98,7245
+19,131,PL,12-05-2015,1,developer,10,network,928977,12-06-2015,12-11-2016,88,91,11254
+20,129,PM,01-12-2015,3,manager,14,Learning,928479,01-01-2016,30-11-2016,75,94,13547
+21,124,SE,17-01-2007,1,developer,10,network,928478,17-02-2007,29-11-2016,96,96,5040
+22,106,SSE,29-05-2008,1,developer,11,protocol,928378,29-06-2008,30-12-2016,85,95,7124
+23,122222,TPL,07-07-2009,2,tester,10,network,928478,07-08-2009,30-12-2016,88,99,9054
+24,13333,SA,29-12-2010,3,manager,11,protocol,928278,29-01-2011,29-06-2016,77,92,11248
+25,140,SSA,09-11-2011,1,developer,12,security,928375,09-12-2011,29-05-2016,99,91,13245
+26,149,SE,14-10-2012,1,developer,13,configManagement,928478,14-11-2012,29-12-2016,86,93,5040
+27,132,PL,22-09-2013,2,tester,12,security,928778,22-10-2013,15-11-2016,78,97,9574
+28,13888888,TL,15-08-2014,2,tester,14,Learning,928176,15-09-2014,29-05-2016,84,98,7245
+29,126,PL,12-05-2015,1,developer,10,network,928977,12-06-2015,12-11-2016,88,91,11254
+30,116,PM,01-12-2015,3,manager,14,Learning,928479,01-01-2016,30-11-2016,75,94,13547
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/105e3a0d/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
index 5a72afe..68b08a3 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
@@ -244,8 +244,7 @@ class RangeFilterMyTests extends QueryTest with BeforeAndAfterAll {
test("test range filter for direct dictionary"){
checkAnswer(
sql("select doj from directDictionaryTable where doj > '2016-03-14 15:00:17'"),
- Seq(Row(Timestamp.valueOf("2016-03-14 15:00:18.0"))
- )
+ sql("select doj from directDictionaryTable_hive where doj > '2016-03-14 15:00:17'")
)
}
@@ -265,6 +264,36 @@ class RangeFilterMyTests extends QueryTest with BeforeAndAfterAll {
)
}
+ test("test range filter for direct dictionary and boundary 2"){
+ checkAnswer(
+ sql("select count(*) from directDictionaryTable where doj > '2016-03-14 15:00:23.0' and doj < '2016-03-14 15:00:60.0'"),
+ sql("select count(*) from directDictionaryTable_hive where doj > '2016-03-14 15:00:23.0' and doj < '2016-03-14 15:00:60.0'")
+ )
+ }
+
+
+ test("test range filter for direct dictionary and boundary 3"){
+ checkAnswer(
+ sql("select count(*) from directDictionaryTable where doj >= '2016-03-14 15:00:23.0' and doj < '2016-03-14 15:00:60.0'"),
+ sql("select count(*) from directDictionaryTable_hive where doj >= '2016-03-14 15:00:23.0' and doj < '2016-03-14 15:00:60.0'")
+ )
+ }
+
+
+ test("test range filter for direct dictionary and boundary 4"){
+ checkAnswer(
+ sql("select count(*) from directDictionaryTable where doj > '2016-03-14 15:00:23.0' and doj < '2016-03-14 15:00:40.0'"),
+ sql("select count(*) from directDictionaryTable_hive where doj > '2016-03-14 15:00:23.0' and doj < '2016-03-14 15:00:40.0'")
+ )
+ }
+
+ test("test range filter for direct dictionary and boundary 5"){
+ checkAnswer(
+ sql("select count(*) from directDictionaryTable where doj > '2016-03-14 15:00:23.0' and doj <= '2016-03-14 15:00:40.0'"),
+ sql("select count(*) from directDictionaryTable_hive where doj > '2016-03-14 15:00:23.0' and doj <= '2016-03-14 15:00:40.0'")
+ )
+ }
+
test("test range filter for direct dictionary more values after filter"){
checkAnswer(
sql("select doj from directDictionaryTable where doj > '2016-03-14 15:00:09'"),
@@ -293,6 +322,89 @@ class RangeFilterMyTests extends QueryTest with BeforeAndAfterAll {
)
}
+ // use cast for range
+ test("test range filter for direct dictionary cast"){
+ checkAnswer(
+ sql("select doj from directDictionaryTable where doj > cast ('2016-03-14 15:00:17' as timestamp)"),
+ sql("select doj from directDictionaryTable_hive where doj > cast ('2016-03-14 15:00:17' as timestamp)")
+ )
+ }
+
+ test("test range filter for direct dictionary cast and"){
+ checkAnswer(
+ sql("select doj from directDictionaryTable where doj > cast ('2016-03-14 15:00:16' as timestamp) and doj < cast ('2016-03-14 15:00:18' as timestamp)"),
+ Seq(Row(Timestamp.valueOf("2016-03-14 15:00:17.0"))
+ )
+ )
+ }
+
+ test("test range filter for direct dictionary and boundary cast "){
+ checkAnswer(
+ sql("select count(*) from directDictionaryTable where doj > cast ('2016-03-14 15:00:18.0' as timestamp) and doj < cast ('2016-03-14 15:00:09.0' as timestamp)"),
+ Seq(Row(0)
+ )
+ )
+ }
+
+ test("test range filter for direct dictionary and boundary 2 cast "){
+ checkAnswer(
+ sql("select count(*) from directDictionaryTable where doj > cast('2016-03-14 15:00:23.0' as timestamp) and doj < cast ('2016-03-14 15:00:60.0' as timestamp)"),
+ sql("select count(*) from directDictionaryTable_hive where doj > cast('2016-03-14 15:00:23.0' as timestamp) and doj < cast ('2016-03-14 15:00:60.0' as timestamp)")
+ )
+ }
+
+
+ test("test range filter for direct dictionary and boundary 3 cast"){
+ checkAnswer(
+ sql("select count(*) from directDictionaryTable where doj >= cast('2016-03-14 15:00:23.0' as timestamp) and doj < cast('2016-03-14 15:00:60.0' as timestamp)"),
+ sql("select count(*) from directDictionaryTable_hive where doj >= cast('2016-03-14 15:00:23.0' as timestamp) and doj < cast('2016-03-14 15:00:60.0' as timestamp)")
+ )
+ }
+
+
+ test("test range filter for direct dictionary and boundary 4 cast"){
+ checkAnswer(
+ sql("select count(*) from directDictionaryTable where doj > cast('2016-03-14 15:00:23.0' as timestamp) and doj < cast('2016-03-14 15:00:40.0' as timestamp)"),
+ sql("select count(*) from directDictionaryTable_hive where doj > cast('2016-03-14 15:00:23.0' as timestamp) and doj < cast('2016-03-14 15:00:40.0' as timestamp)")
+ )
+ }
+
+ test("test range filter for direct dictionary and boundary 5 cast "){
+ checkAnswer(
+ sql("select count(*) from directDictionaryTable where doj > cast('2016-03-14 15:00:23.0' as timestamp) and doj <= cast('2016-03-14 15:00:40.0' as timestamp)"),
+ sql("select count(*) from directDictionaryTable_hive where doj > cast('2016-03-14 15:00:23.0' as timestamp) and doj <= cast('2016-03-14 15:00:40.0' as timestamp)")
+ )
+ }
+
+ test("test range filter for direct dictionary more values after filter cast"){
+ checkAnswer(
+ sql("select doj from directDictionaryTable where doj > cast('2016-03-14 15:00:09' as timestamp)"),
+ sql("select doj from directDictionaryTable_hive where doj > cast('2016-03-14 15:00:09' as timestamp)")
+ )
+ }
+
+ test("test range filter for direct dictionary or condition cast"){
+ checkAnswer(
+ sql("select doj from directDictionaryTable where doj > cast('2016-03-14 15:00:09' as timestamp) or doj > cast('2016-03-14 15:00:15' as timestamp)"),
+ sql("select doj from directDictionaryTable_hive where doj > cast('2016-03-14 15:00:09' as timestamp) or doj > cast('2016-03-14 15:00:15' as timestamp)")
+ )
+ }
+
+ test("test range filter for direct dictionary or and condition cast"){
+ checkAnswer(
+ sql("select doj from directDictionaryTable where doj > cast('2016-03-14 15:00:09' as timestamp) or doj > cast('2016-03-14 15:00:15' as timestamp) and doj < cast('2016-03-14 15:00:13' as timestamp)"),
+ sql("select doj from directDictionaryTable_hive where doj > cast('2016-03-14 15:00:09' as timestamp) or doj > cast('2016-03-14 15:00:15' as timestamp) and doj < cast('2016-03-14 15:00:13' as timestamp)")
+ )
+ }
+
+ test("test range filter for direct dictionary with no data in csv cast"){
+ checkAnswer(
+ sql("select doj from directDictionaryTable where doj > cast('2016-03-14 15:05:09' as timestamp) or doj > cast('2016-03-14 15:05:15' as timestamp) and doj < cast('2016-03-14 15:50:13' as timestamp)"),
+ sql("select doj from directDictionaryTable_hive where doj > cast('2016-03-14 15:05:09' as timestamp) or doj > cast('2016-03-14 15:05:15' as timestamp) and doj < cast('2016-03-14 15:50:13' as timestamp)")
+ )
+ }
+
+
test("test range filter for measure in dictionary include"){
checkAnswer(
sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_6 where deptno > 10 "),
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/105e3a0d/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterTestCase.scala
index 89b4db9..3dd01cc 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterTestCase.scala
@@ -40,6 +40,7 @@ class RangeFilterTestCase extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists NO_DICTIONARY_CARBON_6")
sql("drop table if exists DICTIONARY_CARBON_6")
sql("drop table if exists NO_DICTIONARY_CARBON_7")
+ sql("drop table if exists NO_DICTIONARY_HIVE_8")
sql("CREATE TABLE NO_DICTIONARY_HIVE_1 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION " +
"string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint," +
@@ -117,9 +118,25 @@ class RangeFilterTestCase extends QueryTest with BeforeAndAfterAll {
"TBLPROPERTIES('DICTIONARY_EXCLUDE'='empno,empname,designation')"
)
sql(
- s"LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE NO_DICTIONARY_CARBON_7 " +
+ s"LOAD DATA LOCAL INPATH '$resourcesPath/rangenodictionarycompare.csv' INTO TABLE NO_DICTIONARY_CARBON_7 " +
"OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
)
+
+ sql(
+ "create table NO_DICTIONARY_HIVE_8(empno string,empname string,designation string,doj " +
+ "Timestamp,workgroupcategory int, " +
+ "workgroupcategoryname string,deptno int, deptname string, projectcode int, " +
+ "projectjoindate Timestamp,projectenddate Timestamp,attendance int, "
+ + "utilization int,salary int) row format delimited fields terminated by ',' " +
+ "tblproperties(\"skip.header.line.count\"=\"1\") " +
+ ""
+ )
+
+ sql(
+ s"load data local inpath '$resourcesPath/rangenodictionarycompare.csv' into table " +
+ "NO_DICTIONARY_HIVE_8"
+ );
+
sql("CREATE TABLE filtertestTable (ID string,date Timestamp, country String, " +
"name String, phonetype String, serialname String, salary Int) " +
"STORED BY 'org.apache.carbondata.format' " + "TBLPROPERTIES('DICTIONARY_EXCLUDE'='ID')"
@@ -429,6 +446,67 @@ class RangeFilterTestCase extends QueryTest with BeforeAndAfterAll {
sql("select s.empno, s.empname, t.empno, t.empname from NO_DICTIONARY_HIVE_6 s, NO_DICTIONARY_HIVE_7 t where s.empno > '09' and t.empno < '30' and s.empname = t.empname"))
}
+ test("Range with name comparision") {
+ checkAnswer(
+ sql("select empname from NO_DICTIONARY_CARBON_7 where empname >= '11' and empname < '12'"),
+ sql("select empname from NO_DICTIONARY_HIVE_8 where empname >= '11' and empname < '12'"))
+ }
+
+ test("Range with name comparision 1") {
+ checkAnswer(
+ sql("select empname from NO_DICTIONARY_CARBON_7 where empname > '10' and empname < '11'"),
+ sql("select empname from NO_DICTIONARY_HIVE_8 where empname > '10' and empname < '11'"))
+ }
+
+ test("Range with name comparision 3") {
+ checkAnswer(
+ sql("select empname from NO_DICTIONARY_CARBON_7 where empname > '10' and empname <= '11'"),
+ sql("select empname from NO_DICTIONARY_HIVE_8 where empname > '10' and empname <= '11'"))
+ }
+
+
+ test("Range with name comparision 4") {
+ checkAnswer(
+ sql("select empname from NO_DICTIONARY_CARBON_7 where empname >= '10' and empno <= '11'"),
+ sql("select empname from NO_DICTIONARY_HIVE_8 where empname >= '10' and empno <= '11'"))
+ }
+
+ test("Range with name comparision 5") {
+ checkAnswer(
+ sql("select empname from NO_DICTIONARY_CARBON_7 where empname >= '09' and empno <= '50'"),
+ sql("select empname from NO_DICTIONARY_HIVE_8 where empname >= '09' and empno <= '50'"))
+ }
+
+ test("Range with name comparision 6") {
+ checkAnswer(
+ sql("select empname from NO_DICTIONARY_CARBON_7 where empname >= '03' and empno <= '09'"),
+ sql("select empname from NO_DICTIONARY_HIVE_8 where empname >= '03' and empno <= '09'"))
+ }
+
+ test("Range with name comparision 7") {
+ checkAnswer(
+ sql("select empname from NO_DICTIONARY_CARBON_7 where empname >= '10' and empno <= '50'"),
+ sql("select empname from NO_DICTIONARY_HIVE_8 where empname >= '10' and empno <= '50'"))
+ }
+
+ test("Range with name comparision 8") {
+ checkAnswer(
+ sql("select empname from NO_DICTIONARY_CARBON_7 where empname > '10' and empno <= '50'"),
+ sql("select empname from NO_DICTIONARY_HIVE_8 where empname > '10' and empno <= '50'"))
+ }
+
+ test("Range with name comparision 9") {
+ checkAnswer(
+ sql("select empname from NO_DICTIONARY_CARBON_7 where empname > '10' and empno <= '13'"),
+ sql("select empname from NO_DICTIONARY_HIVE_8 where empname > '10' and empno <= '13'"))
+ }
+
+ test("Range with name comparision 10") {
+ checkAnswer(
+ sql("select empname from NO_DICTIONARY_CARBON_7 where empname > '10' and empno <= '14'"),
+ sql("select empname from NO_DICTIONARY_HIVE_8 where empname > '10' and empno <= '14'"))
+ }
+
override def afterAll {
sql("drop table if exists filtertestTable")
[36/50] [abbrv] incubator-carbondata git commit: Range filter test fix
Posted by ja...@apache.org.
Range filter test fix
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/2b5d566c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/2b5d566c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/2b5d566c
Branch: refs/heads/12-dev
Commit: 2b5d566c604932c9b094befecf50d851f9ed955f
Parents: b99c2c1
Author: ravipesala <ra...@gmail.com>
Authored: Mon May 1 08:23:15 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Mon May 1 08:23:15 2017 +0530
----------------------------------------------------------------------
.../RangeFilterAllDataTypesTestCases.scala | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2b5d566c/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
index 3f42d56..e293d72 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
@@ -242,7 +242,6 @@ class RangeFilterMyTests extends QueryTest with BeforeAndAfterAll {
}
test("test range filter for direct dictionary"){
- sql("select doj from directDictionaryTable where doj > cast('2016-03-14 15:00:17' as timestamp)").show(2000,false)
checkAnswer(
sql("select doj from directDictionaryTable where doj > '2016-03-14 15:00:17'"),
sql("select doj from directDictionaryTable_hive where doj > '2016-03-14 15:00:17'")
@@ -266,7 +265,6 @@ class RangeFilterMyTests extends QueryTest with BeforeAndAfterAll {
}
test("test range filter for direct dictionary not equality"){
- sql("select doj from directDictionaryTable where doj != '2016-03-14 15:00:16'").show(2000, false)
checkAnswer(
sql("select doj from directDictionaryTable where doj != '2016-03-14 15:00:16'"),
Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0")),
@@ -277,12 +275,21 @@ class RangeFilterMyTests extends QueryTest with BeforeAndAfterAll {
Row(Timestamp.valueOf("2016-03-14 15:00:14.0")),
Row(Timestamp.valueOf("2016-03-14 15:00:15.0")),
Row(Timestamp.valueOf("2016-03-14 15:00:17.0")),
- Row(Timestamp.valueOf("2016-03-14 15:00:18.0")))
+ Row(Timestamp.valueOf("2016-03-14 15:00:18.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:19.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:20.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:24.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:25.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:31.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:35.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:38.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:39.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:49.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:50.0")))
)
}
test("test range filter for direct dictionary and with explicit casts"){
- sql("select doj from directDictionaryTable where doj > cast ('2016-03-14 15:00:16' as timestamp) and doj < cast ('2016-03-14 15:00:18' as timestamp)").show(2000, false)
checkAnswer(
sql("select doj from directDictionaryTable where doj > cast ('2016-03-14 15:00:16' as timestamp) and doj < cast ('2016-03-14 15:00:18' as timestamp)"),
Seq(Row(Timestamp.valueOf("2016-03-14 15:00:17.0"))
@@ -336,8 +343,6 @@ class RangeFilterMyTests extends QueryTest with BeforeAndAfterAll {
}
test("test range filter for different Timestamp In format "){
- sql("select doj from directDictionaryTable").show(200,false)
- sql("select * from directDictionaryTable where doj in ('2016-03-14 15:00:18', '2016-03-14 15:00:17' )").show(200, false)
checkAnswer(
sql("select doj from directDictionaryTable where doj in ('2016-03-14 15:00:18', '2016-03-14 15:00:17')"),
Seq(Row(Timestamp.valueOf("2016-03-14 15:00:17.0")),
[38/50] [abbrv] incubator-carbondata git commit: fix metaStorePath
with hive-site.xml
Posted by ja...@apache.org.
fix metaStorePath with hive-site.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/0f3b6e45
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/0f3b6e45
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/0f3b6e45
Branch: refs/heads/12-dev
Commit: 0f3b6e45829d5f093091363203e78512b405bdcb
Parents: 33bfb49
Author: QiangCai <qi...@qq.com>
Authored: Fri Apr 28 20:10:22 2017 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Mon May 1 15:54:12 2017 +0530
----------------------------------------------------------------------
.../org/apache/spark/sql/CarbonSession.scala | 22 +++++++++++++-------
1 file changed, 14 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/0f3b6e45/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
index 844b3b9..b436891 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
@@ -24,6 +24,7 @@ import org.apache.spark.scheduler.{SparkListener, SparkListenerApplicationEnd}
import org.apache.spark.sql.SparkSession.Builder
import org.apache.spark.sql.hive.CarbonSessionState
import org.apache.spark.sql.internal.{SessionState, SharedState}
+import org.apache.spark.util.Utils
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
@@ -81,6 +82,19 @@ object CarbonSession {
getValue("options", builder).asInstanceOf[scala.collection.mutable.HashMap[String, String]]
val userSuppliedContext: Option[SparkContext] =
getValue("userSuppliedContext", builder).asInstanceOf[Option[SparkContext]]
+ val hadoopConf = new Configuration()
+ val configFile = Utils.getContextOrSparkClassLoader.getResource("hive-site.xml")
+ if (configFile != null) {
+ hadoopConf.addResource(configFile)
+ }
+ if (options.get(CarbonCommonConstants.HIVE_CONNECTION_URL).isEmpty &&
+ hadoopConf.get(CarbonCommonConstants.HIVE_CONNECTION_URL) == null) {
+ val metaStorePathAbsolute = new File(metaStorePath).getCanonicalPath
+ val hiveMetaStoreDB = metaStorePathAbsolute + "/metastore_db"
+ options ++= Map[String, String]((CarbonCommonConstants.HIVE_CONNECTION_URL,
+ s"jdbc:derby:;databaseName=$hiveMetaStoreDB;create=true"))
+ }
+
// Get the session from current thread's active session.
var session: SparkSession = SparkSession.getActiveSession match {
case Some(sparkSession: CarbonSession) =>
@@ -135,14 +149,6 @@ object CarbonSession {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.STORE_LOCATION, storePath)
session = new CarbonSession(sparkContext)
- val hadoopConf = session.sharedState.sparkContext.hadoopConfiguration
- if (options.get(CarbonCommonConstants.HIVE_CONNECTION_URL).isEmpty &&
- hadoopConf.get(CarbonCommonConstants.HIVE_CONNECTION_URL) == null) {
- val metaStorePathAbsolute = new File(metaStorePath).getCanonicalPath
- val hiveMetaStoreDB = metaStorePathAbsolute + "/metastore_db"
- options ++= Map[String, String]((CarbonCommonConstants.HIVE_CONNECTION_URL,
- s"jdbc:derby:;databaseName=$hiveMetaStoreDB;create=true"))
- }
options.foreach { case (k, v) => session.sessionState.conf.setConfString(k, v) }
SparkSession.setDefaultSession(session)
[50/50] [abbrv] incubator-carbondata git commit: fix issue of rebase
Posted by ja...@apache.org.
fix issue of rebase
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/62d1cb37
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/62d1cb37
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/62d1cb37
Branch: refs/heads/12-dev
Commit: 62d1cb37b75352d18d9291d9bd1c6bb03007490c
Parents: ab85ae6
Author: QiangCai <qi...@qq.com>
Authored: Tue May 2 14:58:16 2017 +0800
Committer: QiangCai <qi...@qq.com>
Committed: Tue May 2 19:19:16 2017 +0800
----------------------------------------------------------------------
.../impl/VariableLengthDimensionDataChunk.java | 3 +-
...feVariableLengthDimensionDataChunkStore.java | 32 +++++++++++++++++++-
.../apache/carbondata/core/util/ByteUtil.java | 4 +++
3 files changed, 36 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/62d1cb37/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
index 0ac5655..6c47bf5 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
@@ -18,8 +18,7 @@ package org.apache.carbondata.core.datastore.chunk.impl;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory
- .DimensionStoreType;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/62d1cb37/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
index bb0edea..a65d745 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
@@ -23,6 +23,17 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.spark.sql.types.BooleanType;
+import org.apache.spark.sql.types.DataType;
+import org.apache.spark.sql.types.Decimal;
+import org.apache.spark.sql.types.DecimalType;
+import org.apache.spark.sql.types.DoubleType;
+import org.apache.spark.sql.types.FloatType;
+import org.apache.spark.sql.types.IntegerType;
+import org.apache.spark.sql.types.LongType;
+import org.apache.spark.sql.types.ShortType;
+import org.apache.spark.sql.types.StringType;
+
/**
* Below class is responsible to store variable length dimension data chunk in
* memory Memory occupied can be on heap or offheap using unsafe interface
@@ -137,7 +148,26 @@ public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimens
CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY.length, data, currentDataOffset, length)) {
vector.putNull(vectorRow);
} else {
- vector.putBytes(vectorRow, currentDataOffset, length, data);
+ DataType dt = vector.getType();
+ if (dt instanceof StringType) {
+ vector.putBytes(vectorRow, currentDataOffset, length, data);
+ } else if (dt instanceof BooleanType) {
+ vector.putBoolean(vectorRow, ByteUtil.toBoolean(data[currentDataOffset]));
+ } else if (dt instanceof ShortType) {
+ vector.putShort(vectorRow, ByteUtil.toShort(data, currentDataOffset, length));
+ } else if (dt instanceof IntegerType) {
+ vector.putInt(vectorRow, ByteUtil.toInt(data, currentDataOffset, length));
+ } else if (dt instanceof FloatType) {
+ vector.putFloat(vectorRow, ByteUtil.toFloat(data, currentDataOffset));
+ } else if (dt instanceof DoubleType) {
+ vector.putDouble(vectorRow, ByteUtil.toDouble(data, currentDataOffset));
+ } else if (dt instanceof LongType) {
+ vector.putLong(vectorRow, ByteUtil.toLong(data, currentDataOffset, length));
+ } else if (dt instanceof DecimalType) {
+ vector.putDecimal(vectorRow,
+ Decimal.apply(ByteUtil.toBigDecimal(data, currentDataOffset, length)),
+ DecimalType.MAX_PRECISION());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/62d1cb37/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
index 70ddec9..f4ab982 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
@@ -397,6 +397,10 @@ public final class ByteUtil {
return b[0] != (byte) 0;
}
+ public static boolean toBoolean(final byte b) {
+ return b != (byte) 0;
+ }
+
/**
* short => byte[]
*
[49/50] [abbrv] incubator-carbondata git commit: Added option to
specify single pass load in data frame
Posted by ja...@apache.org.
Added option to specify single pass load in data frame
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/ab85ae68
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/ab85ae68
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/ab85ae68
Branch: refs/heads/12-dev
Commit: ab85ae68dc5563fbb3630344dc1bc23a7aab71ac
Parents: 1814048
Author: Sanoj MG <sa...@gmail.com>
Authored: Wed Apr 12 22:32:51 2017 +0400
Committer: jackylk <ja...@huawei.com>
Committed: Tue May 2 09:43:27 2017 +0800
----------------------------------------------------------------------
.../testsuite/dataload/TestLoadDataFrame.scala | 32 ++++++++++++++++++++
.../spark/CarbonDataFrameWriter.scala | 3 +-
2 files changed, 34 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ab85ae68/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
index 6e79a10..b790131 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
@@ -59,6 +59,8 @@ class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS carbon5")
sql("DROP TABLE IF EXISTS carbon6")
sql("DROP TABLE IF EXISTS carbon7")
+ sql("DROP TABLE IF EXISTS carbon8")
+ sql("DROP TABLE IF EXISTS carbon9")
}
@@ -160,6 +162,36 @@ class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll {
)
}
+ test("test load dataframe with single pass enabled") {
+ // save dataframe to carbon file
+ df.write
+ .format("carbondata")
+ .option("tableName", "carbon8")
+ .option("tempCSV", "false")
+ .option("single_pass", "true")
+ .option("compress", "false")
+ .mode(SaveMode.Overwrite)
+ .save()
+ checkAnswer(
+ sql("select count(*) from carbon8 where c3 > 500"), Row(500)
+ )
+ }
+
+ test("test load dataframe with single pass disabled") {
+ // save dataframe to carbon file
+ df.write
+ .format("carbondata")
+ .option("tableName", "carbon9")
+ .option("tempCSV", "true")
+ .option("single_pass", "false")
+ .option("compress", "false")
+ .mode(SaveMode.Overwrite)
+ .save()
+ checkAnswer(
+ sql("select count(*) from carbon9 where c3 > 500"), Row(500)
+ )
+ }
+
override def afterAll {
dropTable
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ab85ae68/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
index 9f813a8..9089be1 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
@@ -194,7 +194,8 @@ class CarbonDataFrameWriter(val dataFrame: DataFrame) {
s"""
LOAD DATA INPATH '$csvFolder'
INTO TABLE ${options.dbName}.${options.tableName}
- OPTIONS ('FILEHEADER' = '${dataFrame.columns.mkString(",")}')
+ OPTIONS ('FILEHEADER' = '${dataFrame.columns.mkString(",")}',
+ 'SINGLE_PASS' = '${options.singlePass}')
"""
}
[26/50] [abbrv] incubator-carbondata git commit: [CARBONDATA-984]
change columnSchenma to columnSchema This closes #848
Posted by ja...@apache.org.
[CARBONDATA-984] change columnSchenma to columnSchema This closes #848
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/61d3f3c2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/61d3f3c2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/61d3f3c2
Branch: refs/heads/12-dev
Commit: 61d3f3c2e439d2de63f01b8b07dce6cf29b065f8
Parents: 53a0a07 4a11fbd
Author: jackylk <ja...@huawei.com>
Authored: Thu Apr 27 13:21:55 2017 -0700
Committer: jackylk <ja...@huawei.com>
Committed: Thu Apr 27 13:21:55 2017 -0700
----------------------------------------------------------------------
.../core/util/CarbonMetadataUtil.java | 34 ++++++++++----------
1 file changed, 17 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
[28/50] [abbrv] incubator-carbondata git commit: [CARBONDATA-923] Fix
issue of insertInto read from OneRowRelation This closes #795
Posted by ja...@apache.org.
[CARBONDATA-923] Fix issue of insertInto read from OneRowRelation This closes #795
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/58be73fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/58be73fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/58be73fc
Branch: refs/heads/12-dev
Commit: 58be73fcdf51fa54891bff92594c0e4c73313751
Parents: 61d3f3c 2a4ab4b
Author: jackylk <ja...@huawei.com>
Authored: Thu Apr 27 22:08:47 2017 -0700
Committer: jackylk <ja...@huawei.com>
Committed: Thu Apr 27 22:08:47 2017 -0700
----------------------------------------------------------------------
.../spark/sql/hive/CarbonAnalysisRules.scala | 17 ++++++++-
.../InsertIntoCarbonTableTestCase.scala | 40 ++++++++++++++++++++
2 files changed, 56 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
[11/50] [abbrv] incubator-carbondata git commit: get
HIVE_CONNECTION_URL from hadoopConfiguration
Posted by ja...@apache.org.
get HIVE_CONNECTION_URL from hadoopConfiguration
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/075e74e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/075e74e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/075e74e7
Branch: refs/heads/12-dev
Commit: 075e74e7425e39c93d8a0761be6cc28faf072019
Parents: d695a8a
Author: QiangCai <qi...@qq.com>
Authored: Mon Apr 24 21:10:08 2017 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Apr 26 11:23:15 2017 +0530
----------------------------------------------------------------------
.../scala/org/apache/spark/sql/CarbonSession.scala | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/075e74e7/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
index 1ce170c..844b3b9 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSession.scala
@@ -81,13 +81,6 @@ object CarbonSession {
getValue("options", builder).asInstanceOf[scala.collection.mutable.HashMap[String, String]]
val userSuppliedContext: Option[SparkContext] =
getValue("userSuppliedContext", builder).asInstanceOf[Option[SparkContext]]
- if (options.get(CarbonCommonConstants.HIVE_CONNECTION_URL).isEmpty &&
- new Configuration().get(CarbonCommonConstants.HIVE_CONNECTION_URL) == null) {
- val metaStorePathAbsolute = new File(metaStorePath).getCanonicalPath
- val hiveMetaStoreDB = metaStorePathAbsolute + "/metastore_db"
- options ++= Map[String, String]((CarbonCommonConstants.HIVE_CONNECTION_URL,
- s"jdbc:derby:;databaseName=$hiveMetaStoreDB;create=true"))
- }
// Get the session from current thread's active session.
var session: SparkSession = SparkSession.getActiveSession match {
case Some(sparkSession: CarbonSession) =>
@@ -142,6 +135,14 @@ object CarbonSession {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.STORE_LOCATION, storePath)
session = new CarbonSession(sparkContext)
+ val hadoopConf = session.sharedState.sparkContext.hadoopConfiguration
+ if (options.get(CarbonCommonConstants.HIVE_CONNECTION_URL).isEmpty &&
+ hadoopConf.get(CarbonCommonConstants.HIVE_CONNECTION_URL) == null) {
+ val metaStorePathAbsolute = new File(metaStorePath).getCanonicalPath
+ val hiveMetaStoreDB = metaStorePathAbsolute + "/metastore_db"
+ options ++= Map[String, String]((CarbonCommonConstants.HIVE_CONNECTION_URL,
+ s"jdbc:derby:;databaseName=$hiveMetaStoreDB;create=true"))
+ }
options.foreach { case (k, v) => session.sessionState.conf.setConfString(k, v) }
SparkSession.setDefaultSession(session)
[21/50] [abbrv] incubator-carbondata git commit: Remove fs,
it may not be initialized when create a instance using
AbstractDFSCarbonFile(FileStatus fileStatus). Fixed spelling mistakes
Posted by ja...@apache.org.
Remove fs, it may not be initialized when create a instance using AbstractDFSCarbonFile(FileStatus fileStatus).
Fixed spelling mistakes
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8a1cd7dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8a1cd7dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8a1cd7dd
Branch: refs/heads/12-dev
Commit: 8a1cd7dd6896e96755b2acb7086a75075b66b9b5
Parents: e2e21e3
Author: smallyard <li...@live.cn>
Authored: Thu Apr 27 10:31:58 2017 +0800
Committer: smallyard <li...@live.cn>
Committed: Thu Apr 27 10:31:58 2017 +0800
----------------------------------------------------------------------
.../filesystem/AbstractDFSCarbonFile.java | 24 ++++++++++++--------
1 file changed, 15 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8a1cd7dd/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
index eaff6e8..8a86d92 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
@@ -38,25 +38,26 @@ public abstract class AbstractDFSCarbonFile implements CarbonFile {
private static final LogService LOGGER =
LogServiceFactory.getLogService(AbstractDFSCarbonFile.class.getName());
protected FileStatus fileStatus;
- protected FileSystem fs;
public AbstractDFSCarbonFile(String filePath) {
filePath = filePath.replace("\\", "/");
Path path = new Path(filePath);
+ FileSystem fs;
try {
fs = path.getFileSystem(FileFactory.getConfiguration());
fileStatus = fs.getFileStatus(path);
} catch (IOException e) {
- LOGGER.error("Exception occured:" + e.getMessage());
+ LOGGER.error("Exception occurred:" + e.getMessage());
}
}
public AbstractDFSCarbonFile(Path path) {
+ FileSystem fs;
try {
fs = path.getFileSystem(FileFactory.getConfiguration());
fileStatus = fs.getFileStatus(path);
} catch (IOException e) {
- LOGGER.error("Exception occured:" + e.getMessage());
+ LOGGER.error("Exception occurred:" + e.getMessage());
}
}
@@ -66,7 +67,9 @@ public abstract class AbstractDFSCarbonFile implements CarbonFile {
@Override public boolean createNewFile() {
Path path = fileStatus.getPath();
+ FileSystem fs;
try {
+ fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
return fs.createNewFile(path);
} catch (IOException e) {
return false;
@@ -86,13 +89,14 @@ public abstract class AbstractDFSCarbonFile implements CarbonFile {
}
@Override public boolean exists() {
+ FileSystem fs;
try {
if (null != fileStatus) {
fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
return fs.exists(fileStatus.getPath());
}
} catch (IOException e) {
- LOGGER.error("Exception occured:" + e.getMessage());
+ LOGGER.error("Exception occurred:" + e.getMessage());
}
return false;
}
@@ -115,7 +119,7 @@ public abstract class AbstractDFSCarbonFile implements CarbonFile {
fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
return fs.rename(fileStatus.getPath(), new Path(changetoName));
} catch (IOException e) {
- LOGGER.error("Exception occured:" + e.getMessage());
+ LOGGER.error("Exception occurred:" + e.getMessage());
return false;
}
}
@@ -126,7 +130,7 @@ public abstract class AbstractDFSCarbonFile implements CarbonFile {
fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
return fs.delete(fileStatus.getPath(), true);
} catch (IOException e) {
- LOGGER.error("Exception occured:" + e.getMessage());
+ LOGGER.error("Exception occurred:" + e.getMessage());
return false;
}
}
@@ -136,7 +140,9 @@ public abstract class AbstractDFSCarbonFile implements CarbonFile {
}
@Override public boolean setLastModifiedTime(long timestamp) {
+ FileSystem fs;
try {
+ fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
fs.setTimes(fileStatus.getPath(), timestamp, timestamp);
} catch (IOException e) {
return false;
@@ -159,7 +165,7 @@ public abstract class AbstractDFSCarbonFile implements CarbonFile {
String tempWriteFilePath = fileName + CarbonCommonConstants.TEMPWRITEFILEEXTENSION;
FileFactory.FileType fileType = FileFactory.getFileType(fileName);
try {
- CarbonFile tempFile = null;
+ CarbonFile tempFile;
// delete temporary file if it already exists at a given path
if (FileFactory.isFileExist(tempWriteFilePath, fileType)) {
tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
@@ -191,7 +197,7 @@ public abstract class AbstractDFSCarbonFile implements CarbonFile {
tempFile.renameForce(fileName);
fileTruncatedSuccessfully = true;
} catch (IOException e) {
- LOGGER.error("Exception occured while truncating the file " + e.getMessage());
+ LOGGER.error("Exception occurred while truncating the file " + e.getMessage());
} finally {
CarbonUtil.closeStreams(dataOutputStream, dataInputStream);
}
@@ -203,7 +209,7 @@ public abstract class AbstractDFSCarbonFile implements CarbonFile {
*
* @param fileTimeStamp time to be compared with latest timestamp of file
* @param endOffset file length to be compared with current length of file
- * @return
+ * @return whether a file has been modified or not
*/
@Override public boolean isFileModified(long fileTimeStamp, long endOffset) {
boolean isFileModified = false;
[37/50] [abbrv] incubator-carbondata git commit: [CARBONDATA-1006]
Range filter test fix This closes #871
Posted by ja...@apache.org.
[CARBONDATA-1006] Range filter test fix This closes #871
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/33bfb493
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/33bfb493
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/33bfb493
Branch: refs/heads/12-dev
Commit: 33bfb493a92d9d07320d36c10edcadadb15525ca
Parents: b99c2c1 2b5d566
Author: chenliang613 <ch...@huawei.com>
Authored: Mon May 1 17:55:02 2017 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Mon May 1 17:55:02 2017 +0800
----------------------------------------------------------------------
.../RangeFilterAllDataTypesTestCases.scala | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
[20/50] [abbrv] incubator-carbondata git commit: fix code style
Posted by ja...@apache.org.
fix code style
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/e2e21e35
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/e2e21e35
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/e2e21e35
Branch: refs/heads/12-dev
Commit: e2e21e353f970a4095bd0606a8fc188b89cc3f62
Parents: e579694 3e8416d
Author: jackylk <ja...@huawei.com>
Authored: Wed Apr 26 10:44:06 2017 -0700
Committer: jackylk <ja...@huawei.com>
Committed: Wed Apr 26 10:44:06 2017 -0700
----------------------------------------------------------------------
.../scala/org/apache/carbondata/examples/AlterTableExample.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[48/50] [abbrv] incubator-carbondata git commit: fix sort_columns
issue
Posted by ja...@apache.org.
fix sort_columns issue
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/0287e1e9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/0287e1e9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/0287e1e9
Branch: refs/heads/12-dev
Commit: 0287e1e94d3c86fad34cc4114b86a6c88a2e9cd2
Parents: a0d65f6
Author: QiangCai <qi...@qq.com>
Authored: Sat Apr 8 22:10:17 2017 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Tue May 2 09:43:27 2017 +0800
----------------------------------------------------------------------
.../executer/RowLevelFilterExecuterImpl.java | 3 ++-
.../RowLevelRangeFilterResolverImpl.java | 5 +++-
.../apache/carbondata/core/util/ByteUtil.java | 28 +++++++++++---------
.../carbondata/core/util/DataTypeUtil.java | 2 +-
.../testsuite/sortcolumns/TestSortColumns.scala | 11 ++++++--
.../CarbonRowDataWriterProcessorStepImpl.java | 4 +--
6 files changed, 33 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/0287e1e9/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 8e7c95f..38204e9 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -254,7 +254,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
memberBytes = null;
}
record[dimColumnEvaluatorInfo.getRowIndex()] = DataTypeUtil
- .getDataBasedOnDataType(memberBytes, dimColumnEvaluatorInfo.getDimension());
+ .getDataBasedOnDataTypeForNoDictionaryColumn(memberBytes,
+ dimColumnEvaluatorInfo.getDimension().getDataType());
} else {
continue;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/0287e1e9/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
index 2a1af65..6d4e528 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
@@ -43,6 +43,7 @@ import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverImpl {
@@ -147,7 +148,9 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL.getBytes());
continue;
}
- filterValuesList.add(result.getString().getBytes());
+ filterValuesList.add(DataTypeUtil
+ .getBytesBasedOnDataTypeForNoDictionaryColumn(result.getString(),
+ result.getDataType()));
} catch (FilterIllegalMemberException e) {
// Any invalid member while evaluation shall be ignored, system will log the
// error only once since all rows the evaluation happens so inorder to avoid
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/0287e1e9/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
index f3f0fcd..70ddec9 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
@@ -404,6 +404,7 @@ public final class ByteUtil {
* @return
*/
public static byte[] toBytes(short val) {
+ val = (short)(val ^ Short.MIN_VALUE);
byte[] b = new byte[SIZEOF_SHORT];
b[1] = (byte) val;
val >>= 8;
@@ -423,20 +424,21 @@ public final class ByteUtil {
if (length != SIZEOF_SHORT || offset + length > bytes.length) {
throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
}
+ short n = 0;
if (CarbonUnsafe.unsafe != null) {
if (CarbonUnsafe.ISLITTLEENDIAN) {
- return Short.reverseBytes(
+ n = Short.reverseBytes(
CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
} else {
- return CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+ n = CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
}
} else {
- short n = 0;
+
n ^= bytes[offset] & 0xFF;
n <<= 8;
n ^= bytes[offset + 1] & 0xFF;
- return n;
}
+ return (short)(n ^ Short.MIN_VALUE);
}
/**
@@ -446,6 +448,7 @@ public final class ByteUtil {
* @return
*/
public static byte[] toBytes(int val) {
+ val = val ^ Integer.MIN_VALUE;
byte[] b = new byte[4];
for (int i = 3; i > 0; i--) {
b[i] = (byte) val;
@@ -467,21 +470,21 @@ public final class ByteUtil {
if (length != SIZEOF_INT || offset + length > bytes.length) {
throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
}
+ int n = 0;
if (CarbonUnsafe.unsafe != null) {
if (CarbonUnsafe.ISLITTLEENDIAN) {
- return Integer.reverseBytes(
+ n = Integer.reverseBytes(
CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
} else {
- return CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+ n = CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
}
} else {
- int n = 0;
for (int i = offset; i < (offset + length); i++) {
n <<= 8;
n ^= bytes[i] & 0xFF;
}
- return n;
}
+ return n ^ Integer.MIN_VALUE;
}
/**
@@ -513,6 +516,7 @@ public final class ByteUtil {
* @return
*/
public static byte[] toBytes(long val) {
+ val = val ^ Long.MIN_VALUE;
byte[] b = new byte[8];
for (int i = 7; i > 0; i--) {
b[i] = (byte) val;
@@ -529,21 +533,21 @@ public final class ByteUtil {
if (length != SIZEOF_LONG || offset + length > bytes.length) {
throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
}
+ long l = 0;
if (CarbonUnsafe.unsafe != null) {
if (CarbonUnsafe.ISLITTLEENDIAN) {
- return Long.reverseBytes(
+ l = Long.reverseBytes(
CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
} else {
- return CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+ l = CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
}
} else {
- long l = 0;
for (int i = offset; i < offset + length; i++) {
l <<= 8;
l ^= bytes[i] & 0xFF;
}
- return l;
}
+ return l ^ Long.MIN_VALUE;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/0287e1e9/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 2e16b0f..80c9e72 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -324,7 +324,7 @@ public final class DataTypeUtil {
}
public static byte[] getBytesBasedOnDataTypeForNoDictionaryColumn(String dimensionValue,
- DataType actualDataType) throws Throwable {
+ DataType actualDataType) {
switch (actualDataType) {
case STRING:
return ByteUtil.toBytes(dimensionValue);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/0287e1e9/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
index 88b0421..afdca21 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
@@ -220,13 +220,20 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
sql("CREATE TABLE unsortedtable_offheap_inmemory (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno = 11"), sql("select * from origintable1 where empno = 11"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno < 15 order by empno"), sql("select * from origintable1 where empno < 15 order by empno"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno <= 15 order by empno"), sql("select * from origintable1 where empno <= 15 order by empno"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno > 15 order by empno"), sql("select * from origintable1 where empno > 15 order by empno"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno >= 15 order by empno"), sql("select * from origintable1 where empno >= 15 order by empno"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno <> 15 order by empno"), sql("select * from origintable1 where empno <> 15 order by empno"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno in (15, 16, 17) order by empno"), sql("select * from origintable1 where empno in (15, 16, 17) order by empno"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno is null"), sql("select * from origintable1 where empno is null order by empno"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno is not null"), sql("select * from origintable1 where empno is not null order by empno"))
checkAnswer(sql("select * from unsortedtable_offheap_inmemory order by empno"), sql("select * from origintable1 order by empno"))
} finally {
defaultLoadingProperties
}
}
-
-
+
override def afterAll = {
dropTable
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/0287e1e9/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
index 929a09e..c50f335 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
@@ -156,9 +156,7 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
private void doExecute(Iterator<CarbonRowBatch> iterator, int partitionId, int iteratorIndex) {
String storeLocation = getStoreLocation(tableIdentifier, String.valueOf(partitionId));
CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel
- .createCarbonFactDataHandlerModel(configuration, storeLocation, partitionId, 0);
- model.getCarbonDataFileAttributes()
- .setFactTimeStamp(model.getCarbonDataFileAttributes().getFactTimeStamp() + iteratorIndex);
+ .createCarbonFactDataHandlerModel(configuration, storeLocation, partitionId, iteratorIndex);
CarbonFactHandler dataHandler = null;
boolean rowsNotExist = true;
while (iterator.hasNext()) {
[46/50] [abbrv] incubator-carbondata git commit: remove redundant
variable
Posted by ja...@apache.org.
remove redundant variable
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/a0d65f69
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/a0d65f69
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/a0d65f69
Branch: refs/heads/12-dev
Commit: a0d65f693a7beb818f9c7e49e9de041c52d18846
Parents: 49cabec
Author: QiangCai <qi...@qq.com>
Authored: Tue May 2 09:43:23 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Tue May 2 09:43:23 2017 +0800
----------------------------------------------------------------------
.../dictionary/DictionaryCacheLoaderImpl.java | 3 +--
.../dictionary/DoubleArrayTrieDictionary.java | 5 ++--
.../dictionary/ReverseDictionaryCache.java | 3 +--
.../block/TableBlockUniqueIdentifier.java | 3 +--
.../core/datastore/impl/FileFactory.java | 10 +++-----
.../DateDirectDictionaryGenerator.java | 3 +--
.../core/keygenerator/mdkey/Bits.java | 3 +--
.../keygenerator/mdkey/NumberCompressor.java | 7 +++--
.../core/mutate/CarbonUpdateUtil.java | 14 ++++------
.../CarbonDictionaryMetadataReaderImpl.java | 12 +++------
.../core/reader/CarbonIndexFileReader.java | 6 ++---
.../core/scan/expression/ColumnExpression.java | 4 +--
.../core/scan/expression/LiteralExpression.java | 6 ++---
.../executer/RowLevelFilterExecuterImpl.java | 6 ++---
.../executer/ValueBasedFilterExecuterImpl.java | 6 ++---
.../iterator/DetailQueryResultIterator.java | 3 +--
.../SegmentUpdateStatusManager.java | 5 ++--
.../apache/carbondata/core/util/CarbonUtil.java | 15 +++++------
.../core/util/path/CarbonStorePath.java | 3 +--
.../core/util/path/CarbonTablePath.java | 6 ++---
.../dictionary/AbstractDictionaryCacheTest.java | 6 ++---
.../dictionary/ReverseDictionaryCacheTest.java | 4 +--
.../block/SegmentPropertiesTestUtil.java | 3 +--
.../filesystem/AlluxioCarbonFileTest.java | 6 ++---
.../filesystem/HDFSCarbonFileTest.java | 9 +++----
.../filesystem/LocalCarbonFileTest.java | 3 +--
.../filesystem/ViewFsCarbonFileTest.java | 6 ++---
.../ThriftWrapperSchemaConverterImplTest.java | 3 +--
.../core/scan/executor/util/QueryUtilTest.java | 9 +++----
.../core/util/RangeFilterProcessorTest.java | 21 +++++----------
.../processing/merger/CarbonCompactionUtil.java | 3 +--
.../processing/merger/CarbonDataMergerUtil.java | 27 ++++++++------------
.../merger/CompactionResultSortProcessor.java | 3 +--
.../newflow/DataLoadProcessBuilder.java | 12 +++------
.../DictionaryServerClientDictionary.java | 3 +--
...arallelReadMergeSorterWithBucketingImpl.java | 4 +--
.../holder/UnsafeSortTempFileChunkHolder.java | 3 +--
.../steps/DataConverterProcessorStepImpl.java | 3 +--
...ConverterProcessorWithBucketingStepImpl.java | 3 +--
.../newflow/steps/SortProcessorStepImpl.java | 3 +--
.../sortdata/SortTempFileChunkHolder.java | 3 +--
.../store/CarbonFactDataHandlerModel.java | 14 ++++------
.../util/CarbonDataProcessorUtil.java | 3 +--
.../processing/util/NonDictionaryUtil.java | 3 +--
44 files changed, 94 insertions(+), 186 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
index 4a0c2e8..5373ca8 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
@@ -128,8 +128,7 @@ public class DictionaryCacheLoaderImpl implements DictionaryCacheLoader {
throws IOException {
CarbonDictionaryReader dictionaryReader = getDictionaryReader(columnIdentifier);
try {
- Iterator<byte[]> columnDictionaryChunkWrapper = dictionaryReader.read(startOffset, endOffset);
- return columnDictionaryChunkWrapper;
+ return dictionaryReader.read(startOffset, endOffset);
} finally {
dictionaryReader.close();
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
index 26b12bf..aeec846 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
@@ -223,11 +223,10 @@ public class DoubleArrayTrieDictionary {
*/
private int conflict(int start, int bKey) {
int from = start;
- int newKey = bKey;
TreeSet<Integer> children = getChildren(from);
- children.add(new Integer(newKey));
+ children.add(new Integer(bKey));
int newBasePos = findFreeRoom(children);
- children.remove(new Integer(newKey));
+ children.remove(new Integer(bKey));
int oldBasePos = base[start];
base[start] = newBasePos;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
index 55dfa0e..acb9acf 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
@@ -89,8 +89,7 @@ public class ReverseDictionaryCache<K extends DictionaryColumnUniqueIdentifier,
for (final DictionaryColumnUniqueIdentifier uniqueIdent : dictionaryColumnUniqueIdentifiers) {
taskSubmitList.add(executorService.submit(new Callable<Dictionary>() {
@Override public Dictionary call() throws IOException {
- Dictionary dictionary = getDictionary(uniqueIdent);
- return dictionary;
+ return getDictionary(uniqueIdent);
}
}));
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockUniqueIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockUniqueIdentifier.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockUniqueIdentifier.java
index e873b85..0434066 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockUniqueIdentifier.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockUniqueIdentifier.java
@@ -78,11 +78,10 @@ public class TableBlockUniqueIdentifier {
BlockInfo blockInfo = new BlockInfo(this.tableBlockInfo);
CarbonTableIdentifier carbonTableIdentifier =
this.absoluteTableIdentifier.getCarbonTableIdentifier();
- String uniqueTableBlockName = carbonTableIdentifier.getDatabaseName()
+ return carbonTableIdentifier.getDatabaseName()
+ CarbonCommonConstants.FILE_SEPARATOR + carbonTableIdentifier
.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR
+ this.tableBlockInfo.getSegmentId()
+ CarbonCommonConstants.FILE_SEPARATOR + blockInfo.hashCode();
- return uniqueTableBlockName;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
index 27931fe..ae9e010 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
@@ -191,8 +191,7 @@ public final class FileFactory {
case VIEWFS:
Path pt = new Path(path);
FileSystem fs = pt.getFileSystem(configuration);
- FSDataOutputStream stream = fs.create(pt, true);
- return stream;
+ return fs.create(pt, true);
default:
return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path)));
}
@@ -244,9 +243,7 @@ public final class FileFactory {
case VIEWFS:
Path pt = new Path(path);
FileSystem fs = pt.getFileSystem(configuration);
- FSDataOutputStream stream =
- fs.create(pt, true, bufferSize, fs.getDefaultReplication(pt), blockSize);
- return stream;
+ return fs.create(pt, true, bufferSize, fs.getDefaultReplication(pt), blockSize);
default:
path = getUpdatedFilePath(path, fileType);
return new DataOutputStream(
@@ -402,8 +399,7 @@ public final class FileFactory {
case VIEWFS:
Path pt = new Path(path);
FileSystem fs = pt.getFileSystem(configuration);
- FSDataOutputStream stream = fs.append(pt);
- return stream;
+ return fs.append(pt);
default:
return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path)));
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
index 8db42b0..a21fd85 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
@@ -152,8 +152,7 @@ public class DateDirectDictionaryGenerator implements DirectDictionaryGenerator
private int generateKey(long timeValue) {
long milli = timeValue + threadLocalLocalTimeZone.get().getOffset(timeValue);
- int key = (int) Math.floor((double) milli / MILLIS_PER_DAY) + cutOffDate;
- return key;
+ return (int) Math.floor((double) milli / MILLIS_PER_DAY) + cutOffDate;
}
public void initialize() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
index 044a77b..822b343 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
@@ -89,9 +89,8 @@ public class Bits implements Serializable {
* {start, end}
*/
public int[] getKeyByteOffsets(int index) {
- int prefixPaddingBits = length % 8 == 0 ? 0 : (8 - length % 8);
- int priorLen = prefixPaddingBits;
+ int priorLen = length % 8 == 0 ? 0 : (8 - length % 8);
int start = 0;
int end = 0;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/NumberCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/NumberCompressor.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/NumberCompressor.java
index af58f50..d601159 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/NumberCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/NumberCompressor.java
@@ -138,10 +138,9 @@ public class NumberCompressor {
}
public int[] unCompress(byte[] key, int offset, int length) {
- int ls = length;
- int arrayLength = (ls * BYTE_LENGTH) / bitsLength;
- long[] words = new long[getWordsSizeFromBytesSize(ls)];
- unCompressVal(key, ls, words, offset);
+ int arrayLength = (length * BYTE_LENGTH) / bitsLength;
+ long[] words = new long[getWordsSizeFromBytesSize(length)];
+ unCompressVal(key, length, words, offset);
return getArray(words, arrayLength);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index fef5905..af26035 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -340,8 +340,7 @@ public class CarbonUpdateUtil {
*/
public static Long getTimeStampAsLong(String timtstamp) {
try {
- long longValue = Long.parseLong(timtstamp);
- return longValue;
+ return Long.parseLong(timtstamp);
} catch (NumberFormatException nfe) {
String errorMsg = "Invalid timestamp : " + timtstamp;
LOGGER.error(errorMsg);
@@ -358,8 +357,7 @@ public class CarbonUpdateUtil {
*/
public static Integer getIntegerValue(String value) throws Exception {
try {
- int intValue = Integer.parseInt(value);
- return intValue;
+ return Integer.parseInt(value);
} catch (NumberFormatException nfe) {
LOGGER.error("Invalid row : " + value + nfe.getLocalizedMessage());
throw new Exception("Invalid row : " + nfe.getLocalizedMessage());
@@ -373,9 +371,8 @@ public class CarbonUpdateUtil {
* @return
*/
public static String getBlockName(String completeBlockName) {
- String blockName =
- completeBlockName.substring(0, completeBlockName.lastIndexOf(CarbonCommonConstants.HYPHEN));
- return blockName;
+ return completeBlockName
+ .substring(0, completeBlockName.lastIndexOf(CarbonCommonConstants.HYPHEN));
}
/**
@@ -385,8 +382,7 @@ public class CarbonUpdateUtil {
* @return
*/
public static String getSegmentId(String segmentName) {
- String id = segmentName.split(CarbonCommonConstants.UNDERSCORE)[1];
- return id;
+ return segmentName.split(CarbonCommonConstants.UNDERSCORE)[1];
}
public static int getLatestTaskIdForSegment(String segmentId, CarbonTablePath tablePath) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
index 79ca689..98c0ddb 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
@@ -130,9 +130,7 @@ public class CarbonDictionaryMetadataReaderImpl implements CarbonDictionaryMetad
dictionaryChunkMeta = (ColumnDictionaryChunkMeta) dictionaryMetadataFileReader.read();
}
// create a new instance of chunk meta wrapper using thrift object
- CarbonDictionaryColumnMetaChunk columnMetaChunkForLastSegment =
- getNewInstanceOfCarbonDictionaryColumnMetaChunk(dictionaryChunkMeta);
- return columnMetaChunkForLastSegment;
+ return getNewInstanceOfCarbonDictionaryColumnMetaChunk(dictionaryChunkMeta);
}
/**
@@ -190,10 +188,8 @@ public class CarbonDictionaryMetadataReaderImpl implements CarbonDictionaryMetad
*/
private CarbonDictionaryColumnMetaChunk getNewInstanceOfCarbonDictionaryColumnMetaChunk(
ColumnDictionaryChunkMeta dictionaryChunkMeta) {
- CarbonDictionaryColumnMetaChunk columnMetaChunk =
- new CarbonDictionaryColumnMetaChunk(dictionaryChunkMeta.getMin_surrogate_key(),
- dictionaryChunkMeta.getMax_surrogate_key(), dictionaryChunkMeta.getStart_offset(),
- dictionaryChunkMeta.getEnd_offset(), dictionaryChunkMeta.getChunk_count());
- return columnMetaChunk;
+ return new CarbonDictionaryColumnMetaChunk(dictionaryChunkMeta.getMin_surrogate_key(),
+ dictionaryChunkMeta.getMax_surrogate_key(), dictionaryChunkMeta.getStart_offset(),
+ dictionaryChunkMeta.getEnd_offset(), dictionaryChunkMeta.getChunk_count());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/main/java/org/apache/carbondata/core/reader/CarbonIndexFileReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonIndexFileReader.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonIndexFileReader.java
index b2c2cb5..dc46242 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonIndexFileReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonIndexFileReader.java
@@ -40,12 +40,11 @@ public class CarbonIndexFileReader {
* @throws IOException if any problem while reader the header
*/
public IndexHeader readIndexHeader() throws IOException {
- IndexHeader indexHeader = (IndexHeader) thriftReader.read(new ThriftReader.TBaseCreator() {
+ return (IndexHeader) thriftReader.read(new ThriftReader.TBaseCreator() {
@Override public TBase create() {
return new IndexHeader();
}
});
- return indexHeader;
}
/**
@@ -62,12 +61,11 @@ public class CarbonIndexFileReader {
* @throws IOException if problem while reading the block index
*/
public BlockIndex readBlockIndexInfo() throws IOException {
- BlockIndex blockInfo = (BlockIndex) thriftReader.read(new ThriftReader.TBaseCreator() {
+ return (BlockIndex) thriftReader.read(new ThriftReader.TBaseCreator() {
@Override public TBase create() {
return new BlockIndex();
}
});
- return blockInfo;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
index 69fa9d6..d27d789 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
@@ -86,9 +86,7 @@ public class ColumnExpression extends LeafExpression {
}
@Override public ExpressionResult evaluate(RowIntf value) {
- ExpressionResult expressionResult =
- new ExpressionResult(dataType, (null == value ? null : value.getVal(colIndex)));
- return expressionResult;
+ return new ExpressionResult(dataType, (null == value ? null : value.getVal(colIndex)));
}
@Override public ExpressionType getFilterExpressionType() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java
index 112bf03..b194e3c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java
@@ -37,13 +37,11 @@ public class LiteralExpression extends LeafExpression {
}
@Override public ExpressionResult evaluate(RowIntf value) {
- ExpressionResult expressionResult = new ExpressionResult(dataType, this.value, true);
- return expressionResult;
+ return new ExpressionResult(dataType, this.value, true);
}
public ExpressionResult getExpressionResult() {
- ExpressionResult expressionResult = new ExpressionResult(dataType, this.value, true);
- return expressionResult;
+ return new ExpressionResult(dataType, this.value, true);
}
@Override public ExpressionType getFilterExpressionType() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 470de89..8e7c95f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -430,8 +430,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
if (dimColumnEvaluatorInfo.getDimension().isColumnar()) {
byte[] rawData = dataChunk.getChunkData(index);
ByteBuffer byteBuffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE);
- int dictionaryValue = CarbonUtil.getSurrogateKey(rawData, byteBuffer);
- return dictionaryValue;
+ return CarbonUtil.getSurrogateKey(rawData, byteBuffer);
} else {
return readSurrogatesFromColumnGroupBlock(dataChunk, index, dimColumnEvaluatorInfo);
}
@@ -452,9 +451,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
long[] result = keyStructureInfo.getKeyGenerator().getKeyArray(colData);
int colGroupId =
QueryUtil.getColumnGroupId(segmentProperties, dimensionBlocksIndex[0]);
- int dictionaryValue = (int) result[segmentProperties
+ return (int) result[segmentProperties
.getColumnGroupMdKeyOrdinal(colGroupId, dimensionBlocksIndex[0])];
- return dictionaryValue;
} catch (KeyGenException e) {
LOGGER.error(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
index 003d15e..3e33d04 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
@@ -28,13 +28,11 @@ public class ValueBasedFilterExecuterImpl implements FilterExecuter {
@Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
throws FilterUnsupportedException, IOException {
- BitSetGroup bitSetGroup = new BitSetGroup(0);
- return bitSetGroup;
+ return new BitSetGroup(0);
}
@Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
- BitSet bitSet = new BitSet(1);
- return bitSet;
+ return new BitSet(1);
}
@Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
index e39b93f..747f5a9 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
@@ -38,8 +38,7 @@ public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator
}
@Override public BatchResult next() {
- BatchResult batchResult = getBatchResult();
- return batchResult;
+ return getBatchResult();
}
private BatchResult getBatchResult() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index c822935..70e1993 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -427,7 +427,8 @@ public class SegmentUpdateStatusManager {
getStartTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
final long deltaEndTimeStamp =
getEndTimeOfDeltaFile(CarbonCommonConstants.DELETE_DELTA_FILE_EXT, block);
- CarbonFile[] files = segDir.listFiles(new CarbonFileFilter() {
+
+ return segDir.listFiles(new CarbonFileFilter() {
@Override public boolean accept(CarbonFile pathName) {
String fileName = pathName.getName();
@@ -444,8 +445,6 @@ public class SegmentUpdateStatusManager {
return false;
}
});
-
- return files;
}
}
return null;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 95ee94b..99463de 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -722,9 +722,8 @@ public final class CarbonUtil {
if (null == prop) {
return null;
}
- String basePath = prop.getProperty(CarbonCommonConstants.STORE_LOCATION,
+ return prop.getProperty(CarbonCommonConstants.STORE_LOCATION,
CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL);
- return basePath;
}
/**
@@ -1057,9 +1056,8 @@ public final class CarbonUtil {
isDictionaryDimensions.add(false);
}
}
- boolean[] primitive = ArrayUtils
+ return ArrayUtils
.toPrimitive(isDictionaryDimensions.toArray(new Boolean[isDictionaryDimensions.size()]));
- return primitive;
}
/**
@@ -1594,14 +1592,13 @@ public final class CarbonUtil {
return true;
}
- UpdateVO updatedVODetails = invalidBlockVOForSegmentId;
- if (null != updatedVODetails) {
+ if (null != invalidBlockVOForSegmentId) {
Long blockTimeStamp = Long.parseLong(tableInfo.getFilePath()
.substring(tableInfo.getFilePath().lastIndexOf('-') + 1,
tableInfo.getFilePath().lastIndexOf('.')));
- if ((blockTimeStamp > updatedVODetails.getFactTimestamp() && (
- updatedVODetails.getUpdateDeltaStartTimestamp() != null
- && blockTimeStamp < updatedVODetails.getUpdateDeltaStartTimestamp()))) {
+ if ((blockTimeStamp > invalidBlockVOForSegmentId.getFactTimestamp() && (
+ invalidBlockVOForSegmentId.getUpdateDeltaStartTimestamp() != null
+ && blockTimeStamp < invalidBlockVOForSegmentId.getUpdateDeltaStartTimestamp()))) {
return true;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java
index 54de175..3aece69 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonStorePath.java
@@ -40,10 +40,9 @@ public class CarbonStorePath extends Path {
*/
public static CarbonTablePath getCarbonTablePath(String storePath,
CarbonTableIdentifier tableIdentifier) {
- CarbonTablePath carbonTablePath = new CarbonTablePath(tableIdentifier,
+ return new CarbonTablePath(tableIdentifier,
storePath + File.separator + tableIdentifier.getDatabaseName() + File.separator
+ tableIdentifier.getTableName());
- return carbonTablePath;
}
public static CarbonTablePath getCarbonTablePath(String storePath,
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index 918772a..6a60a65 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -590,12 +590,11 @@ public class CarbonTablePath extends Path {
* @return sort index carbon files
*/
public CarbonFile[] getSortIndexFiles(CarbonFile sortIndexDir, final String columnUniqueId) {
- CarbonFile[] files = sortIndexDir.listFiles(new CarbonFileFilter() {
+ return sortIndexDir.listFiles(new CarbonFileFilter() {
@Override public boolean accept(CarbonFile file) {
return file.getName().startsWith(columnUniqueId) && file.getName().endsWith(SORT_INDEX_EXT);
}
});
- return files;
}
/**
@@ -605,10 +604,9 @@ public class CarbonTablePath extends Path {
* @return
*/
public static String getCarbonDataFileName(String carbonDataFilePath) {
- String carbonDataFileName = carbonDataFilePath
+ return carbonDataFilePath
.substring(carbonDataFilePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR) + 1,
carbonDataFilePath.indexOf(CARBON_DATA_EXT));
- return carbonDataFileName;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
index 2bbe458..2fa2b93 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
@@ -104,10 +104,8 @@ public class AbstractDictionaryCacheTest {
protected DictionaryColumnUniqueIdentifier createDictionaryColumnUniqueIdentifier(
String columnId) {
ColumnIdentifier columnIdentifier = new ColumnIdentifier(columnId, null, DataType.STRING);
- DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier =
- new DictionaryColumnUniqueIdentifier(carbonTableIdentifier, columnIdentifier,
- DataType.STRING);
- return dictionaryColumnUniqueIdentifier;
+ return new DictionaryColumnUniqueIdentifier(carbonTableIdentifier, columnIdentifier,
+ DataType.STRING);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java
index f5eff67..628c3ff 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java
@@ -271,8 +271,6 @@ public class ReverseDictionaryCacheTest extends AbstractDictionaryCacheTest {
protected DictionaryColumnUniqueIdentifier createDictionaryColumnUniqueIdentifier(
String columnId) {
ColumnIdentifier columnIdentifier = new ColumnIdentifier(columnId, null, DataType.DOUBLE);
- DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier =
- new DictionaryColumnUniqueIdentifier(carbonTableIdentifier, columnIdentifier);
- return dictionaryColumnUniqueIdentifier;
+ return new DictionaryColumnUniqueIdentifier(carbonTableIdentifier, columnIdentifier);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
index 2bfe40d..5a941b3 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
@@ -47,8 +47,7 @@ public class SegmentPropertiesTestUtil {
cardinality[i] = x;
x++;
}
- SegmentProperties segmentProperties = new SegmentProperties(columnSchema, cardinality);
- return segmentProperties;
+ return new SegmentProperties(columnSchema, cardinality);
}
public static ColumnSchema getDimensionColumn1() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java
index a691dc7..eff0d8b 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java
@@ -135,8 +135,7 @@ public class AlluxioCarbonFileTest {
@Mock
public FileStatus[] listStatus(Path var1) throws IOException {
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
+ return new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
}
};
@@ -207,8 +206,7 @@ public class AlluxioCarbonFileTest {
@Mock
public FileStatus[] listStatus(Path var1) throws IOException {
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
+ return new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
}
};
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
index b9c1802..5ca7fa8 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
@@ -176,8 +176,7 @@ public class HDFSCarbonFileTest {
@Mock
public FileStatus[] listStatus(Path var1) throws IOException {
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
+ return new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
}
};
@@ -251,8 +250,7 @@ public class HDFSCarbonFileTest {
@Mock
public FileStatus[] listStatus(Path var1) throws IOException {
- FileStatus fileStatus[] = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
+ return new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
}
};
@@ -280,8 +278,7 @@ public class HDFSCarbonFileTest {
@Mock
public FileStatus[] listStatus(Path var1) throws IOException {
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
+ return new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
}
};
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
index 498f4fc..3925e9e 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
@@ -294,8 +294,7 @@ public class LocalCarbonFileTest {
@Mock
public File[] listFiles(FileFilter filter) {
- File[] file = new File[]{dir};
- return file;
+ return new File[]{dir};
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
index 5218043..3cb8198 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
@@ -145,8 +145,7 @@ public class ViewFsCarbonFileTest {
@Mock
public FileStatus[] listStatus(Path var1) throws IOException {
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
+ return new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
}
};
@@ -217,8 +216,7 @@ public class ViewFsCarbonFileTest {
@Mock
public FileStatus[] listStatus(Path var1) throws IOException {
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
+ return new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
}
};
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
index b044b6f..e8851e0 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
@@ -105,8 +105,7 @@ public class ThriftWrapperSchemaConverterImplTest {
}
@Mock public long getTime_stamp() {
- long time = 1112745600000L;
- return time;
+ return 1112745600000L;
}
@Mock public List<org.apache.carbondata.format.ColumnSchema> getAdded() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
index 288c148..23504d3 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
@@ -149,8 +149,7 @@ public class QueryUtilTest extends TestCase {
int ordinal = 0;
new MockUp<SegmentProperties>() {
@Mock public int[][] getColumnGroups() {
- int columnGroups[][] = { { 1, 1 }, { 2, 2 }, { 3, 3 }, { 4, 4 }, { 5, 5 } };
- return columnGroups;
+ return new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 }, { 4, 4 }, { 5, 5 } };
}
};
int actualValue = QueryUtil.getColumnGroupId(segmentProperties, ordinal);
@@ -162,8 +161,7 @@ public class QueryUtilTest extends TestCase {
int ordinal = 1;
new MockUp<SegmentProperties>() {
@Mock public int[][] getColumnGroups() {
- int columnGroups[][] = { { 1, 1 }, { 2, 2 }, { 3, 3 }, { 4, 4 }, { 5, 5 } };
- return columnGroups;
+ return new int[][] { { 1, 1 }, { 2, 2 }, { 3, 3 }, { 4, 4 }, { 5, 5 } };
}
};
int actualValue = QueryUtil.getColumnGroupId(segmentProperties, ordinal);
@@ -175,8 +173,7 @@ public class QueryUtilTest extends TestCase {
int ordinal = 1;
new MockUp<SegmentProperties>() {
@Mock public int[][] getColumnGroups() {
- int columnGroups[][] = { { 1 } };
- return columnGroups;
+ return new int[][] { { 1 } };
}
};
int actualValue = QueryUtil.getColumnGroupId(segmentProperties, ordinal);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java b/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
index 2d8927b..952b7f7 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/RangeFilterProcessorTest.java
@@ -94,8 +94,7 @@ public class RangeFilterProcessorTest {
colb.setDimension(empDimension);
Expression lessThan =
new LessThanEqualToExpression(colb, new LiteralExpression("20", DataType.STRING));
- Expression andExp = new AndExpression(greaterThan, lessThan);
- inputFilter = andExp;
+ inputFilter = new AndExpression(greaterThan, lessThan);
Expression output = new AndExpression(new RangeExpression(
new GreaterThanEqualToExpression(new ColumnExpression("a", DataType.STRING),
@@ -136,8 +135,7 @@ public class RangeFilterProcessorTest {
colb.setDimension(empDimension);
Expression lessThan =
new LessThanEqualToExpression(colb, new LiteralExpression("05", DataType.STRING));
- Expression andExp = new AndExpression(greaterThan, lessThan);
- inputFilter = andExp;
+ inputFilter = new AndExpression(greaterThan, lessThan);
Expression output = new AndExpression(
new GreaterThanEqualToExpression(new ColumnExpression("a", DataType.STRING),
@@ -195,8 +193,7 @@ public class RangeFilterProcessorTest {
Expression And1 = new AndExpression(new NotEqualsExpression(null, null), greaterThan2);
Expression And2 = new AndExpression(And1, greaterThan1);
Expression And3 = new AndExpression(And2, lessThan2);
- Expression And4 = new AndExpression(And3, lessThan1);
- inputFilter = And4;
+ inputFilter = new AndExpression(And3, lessThan1);
// Build The output
@@ -219,14 +216,11 @@ public class RangeFilterProcessorTest {
Expression Andb2 = new AndExpression(Andb1, new RangeExpression(greaterThanb1, lessThanb1));
Expression Andb3 = new AndExpression(Andb2, new TrueExpression(null));
- Expression Andb4 = new AndExpression(Andb3, new TrueExpression(null));
-
- Expression output = Andb4;
FilterOptimizer rangeFilterOptimizer =
new RangeFilterOptmizer(new FilterOptimizerBasic(), inputFilter);
rangeFilterOptimizer.optimizeFilter();
- result = checkBothTrees(inputFilter, output);
+ result = checkBothTrees(inputFilter, new AndExpression(Andb3, new TrueExpression(null)));
// no change
Assert.assertTrue(result);
}
@@ -274,8 +268,7 @@ public class RangeFilterProcessorTest {
Expression Or1 = new OrExpression(new NotEqualsExpression(null, null), greaterThan2);
Expression Or2 = new OrExpression(Or1, greaterThan1);
Expression Or3 = new OrExpression(Or2, lessThan2);
- Expression Or4 = new OrExpression(Or3, lessThan1);
- inputFilter = Or4;
+ inputFilter = new OrExpression(Or3, lessThan1);
// Build The output
@@ -307,13 +300,11 @@ public class RangeFilterProcessorTest {
Expression Orb1 = new OrExpression(new NotEqualsExpression(null, null), greaterThanb2);
Expression Orb2 = new OrExpression(Orb1, greaterThanb1);
Expression Orb3 = new OrExpression(Orb2, lessThanb2);
- Expression Orb4 = new OrExpression(Orb3, lessThanb1);
- Expression output = Orb4;
FilterOptimizer rangeFilterOptimizer =
new RangeFilterOptmizer(new FilterOptimizerBasic(), inputFilter);
rangeFilterOptimizer.optimizeFilter();
- result = checkBothTrees(inputFilter, output);
+ result = checkBothTrees(inputFilter, new OrExpression(Orb3, lessThanb1));
// no change
Assert.assertTrue(result);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
index 2ad83a4..b3d5b5c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
@@ -325,9 +325,8 @@ public class CarbonCompactionUtil {
for (CarbonMeasure measure : masterSchemaMeasures) {
updatedColumnSchemaList.add(measure.getColumnSchema());
}
- int[] updatedCardinality = ArrayUtils
+ return ArrayUtils
.toPrimitive(updatedCardinalityList.toArray(new Integer[updatedCardinalityList.size()]));
- return updatedCardinality;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
index b9ea447..ce9c433 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
@@ -139,16 +139,13 @@ public final class CarbonDataMergerUtil {
* @return
*/
public static String getMergedLoadName(final String segmentToBeMerged) {
- String firstSegmentName = segmentToBeMerged;
- if (firstSegmentName.contains(".")) {
- String beforeDecimal = firstSegmentName.substring(0, firstSegmentName.indexOf("."));
- String afterDecimal = firstSegmentName.substring(firstSegmentName.indexOf(".") + 1);
+ if (segmentToBeMerged.contains(".")) {
+ String beforeDecimal = segmentToBeMerged.substring(0, segmentToBeMerged.indexOf("."));
+ String afterDecimal = segmentToBeMerged.substring(segmentToBeMerged.indexOf(".") + 1);
int fraction = Integer.parseInt(afterDecimal) + 1;
- String mergedSegmentName = beforeDecimal + "." + fraction;
- return mergedSegmentName;
+ return beforeDecimal + "." + fraction;
} else {
- String mergeName = firstSegmentName + "." + 1;
- return mergeName;
+ return segmentToBeMerged + "." + 1;
}
}
@@ -225,9 +222,9 @@ public final class CarbonDataMergerUtil {
for (String compactedBlocks : updatedDeltaFilesList) {
// Try to BlockName
- String fullBlock = compactedBlocks;
- int endIndex = fullBlock.lastIndexOf(File.separator);
- String blkNoExt = fullBlock.substring(endIndex + 1, fullBlock.lastIndexOf("-"));
+ int endIndex = compactedBlocks.lastIndexOf(File.separator);
+ String blkNoExt =
+ compactedBlocks.substring(endIndex + 1, compactedBlocks.lastIndexOf("-"));
blockNames.add(blkNoExt);
}
@@ -422,10 +419,7 @@ public final class CarbonDataMergerUtil {
// Check for segments which are qualified for IUD compaction.
if (compactionType.equals(CompactionType.IUD_UPDDEL_DELTA_COMPACTION)) {
- List<LoadMetadataDetails> listOfSegmentsToBeMerged =
- identifySegmentsToBeMergedBasedOnIUD(sortedSegments, carbonLoadModel);
-
- return listOfSegmentsToBeMerged;
+ return identifySegmentsToBeMergedBasedOnIUD(sortedSegments, carbonLoadModel);
}
// check preserve property and preserve the configured number of latest loads.
@@ -987,8 +981,7 @@ public final class CarbonDataMergerUtil {
List<String> list = segmentUpdateStatusManager.getUpdateDeltaFiles(seg);
- String fullBlock = blkName;
- String[] FileParts = fullBlock.split(CarbonCommonConstants.FILE_SEPARATOR);
+ String[] FileParts = blkName.split(CarbonCommonConstants.FILE_SEPARATOR);
String blockName = FileParts[FileParts.length - 1];
for (String str : list) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index a8f9520..81ee408 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -344,12 +344,11 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
* @return
*/
private SortParameters createSortParameters() {
- SortParameters parameters = SortParameters
+ return SortParameters
.createSortParameters(carbonLoadModel.getDatabaseName(), tableName, dimensionColumnCount,
segmentProperties.getComplexDimensions().size(), measureCount, noDictionaryCount,
carbonLoadModel.getPartitionId(), segmentId, carbonLoadModel.getTaskNo(),
noDictionaryColMapping, true);
- return parameters;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
index de469f9..0978a02 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
@@ -83,9 +83,7 @@ public final class DataLoadProcessBuilder {
AbstractDataLoadProcessorStep sortProcessorStep =
new SortProcessorStepImpl(configuration, converterProcessorStep);
// 4. Writes the sorted data in carbondata format.
- AbstractDataLoadProcessorStep writerProcessorStep =
- new DataWriterProcessorStepImpl(configuration, sortProcessorStep);
- return writerProcessorStep;
+ return new DataWriterProcessorStepImpl(configuration, sortProcessorStep);
}
private AbstractDataLoadProcessorStep buildInternalForNoSort(CarbonIterator[] inputIterators,
@@ -116,9 +114,7 @@ public final class DataLoadProcessBuilder {
AbstractDataLoadProcessorStep sortProcessorStep =
new SortProcessorStepImpl(configuration, converterProcessorStep);
// 4. Writes the sorted data in carbondata format.
- AbstractDataLoadProcessorStep writerProcessorStep =
- new DataWriterBatchProcessorStepImpl(configuration, sortProcessorStep);
- return writerProcessorStep;
+ return new DataWriterBatchProcessorStepImpl(configuration, sortProcessorStep);
}
private AbstractDataLoadProcessorStep buildInternalForBucketing(CarbonIterator[] inputIterators,
@@ -134,9 +130,7 @@ public final class DataLoadProcessBuilder {
AbstractDataLoadProcessorStep sortProcessorStep =
new SortProcessorStepImpl(configuration, converterProcessorStep);
// 4. Writes the sorted data in carbondata format.
- AbstractDataLoadProcessorStep writerProcessorStep =
- new DataWriterProcessorStepImpl(configuration, sortProcessorStep);
- return writerProcessorStep;
+ return new DataWriterProcessorStepImpl(configuration, sortProcessorStep);
}
private CarbonDataLoadConfiguration createConfiguration(CarbonLoadModel loadModel,
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/processing/src/main/java/org/apache/carbondata/processing/newflow/dictionary/DictionaryServerClientDictionary.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/dictionary/DictionaryServerClientDictionary.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/dictionary/DictionaryServerClientDictionary.java
index c7ff1c7..82605a5 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/dictionary/DictionaryServerClientDictionary.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/dictionary/DictionaryServerClientDictionary.java
@@ -84,7 +84,6 @@ public class DictionaryServerClientDictionary implements BiDictionary<Integer, O
@Override public int size() {
dictionaryMessage.setType(DictionaryMessageType.SIZE);
- int size = client.getDictionary(dictionaryMessage).getDictionaryValue() + base;
- return size;
+ return client.getDictionary(dictionaryMessage).getDictionaryValue() + base;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
index 19b1cfe..60231c5 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
@@ -139,13 +139,11 @@ public class ParallelReadMergeSorterWithBucketingImpl extends AbstractMergeSorte
// Set the data file location
String dataFolderLocation =
storeLocation + File.separator + CarbonCommonConstants.SORT_TEMP_FILE_LOCATION;
- SingleThreadFinalSortFilesMerger finalMerger =
- new SingleThreadFinalSortFilesMerger(dataFolderLocation, sortParameters.getTableName(),
+ return new SingleThreadFinalSortFilesMerger(dataFolderLocation, sortParameters.getTableName(),
sortParameters.getDimColCount(), sortParameters.getComplexDimColCount(),
sortParameters.getMeasureColCount(), sortParameters.getNoDictionaryCount(),
sortParameters.getAggType(), sortParameters.getNoDictionaryDimnesionColumn(),
this.sortParameters.getNoDictionarySortColumn());
- return finalMerger;
}
@Override public void close() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
index 84aa029..cfdb69a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
@@ -255,8 +255,7 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
prefetchRecordsProceesed++;
returnRow = currentBuffer[bufferRowCounter++];
} else {
- Object[] outRow = getRowFromStream();
- this.returnRow = outRow;
+ this.returnRow = getRowFromStream();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
index cc99469..2348d54 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
@@ -150,13 +150,12 @@ public class DataConverterProcessorStepImpl extends AbstractDataLoadProcessorSte
}
CarbonTableIdentifier identifier =
configuration.getTableIdentifier().getCarbonTableIdentifier();
- BadRecordsLogger badRecordsLogger = new BadRecordsLogger(identifier.getBadRecordLoggerKey(),
+ return new BadRecordsLogger(identifier.getBadRecordLoggerKey(),
identifier.getTableName() + '_' + System.currentTimeMillis(), getBadLogStoreLocation(
identifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + identifier
.getTableName() + CarbonCommonConstants.FILE_SEPARATOR + configuration.getSegmentId()
+ CarbonCommonConstants.FILE_SEPARATOR + configuration.getTaskNo()),
badRecordsLogRedirect, badRecordsLoggerEnable, badRecordConvertNullDisable, isDataLoadFail);
- return badRecordsLogger;
}
private String getBadLogStoreLocation(String storeLocation) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
index 86971c3..314b8c4 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
@@ -177,13 +177,12 @@ public class DataConverterProcessorWithBucketingStepImpl extends AbstractDataLoa
}
CarbonTableIdentifier identifier =
configuration.getTableIdentifier().getCarbonTableIdentifier();
- BadRecordsLogger badRecordsLogger = new BadRecordsLogger(identifier.getBadRecordLoggerKey(),
+ return new BadRecordsLogger(identifier.getBadRecordLoggerKey(),
identifier.getTableName() + '_' + System.currentTimeMillis(), getBadLogStoreLocation(
identifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + identifier
.getTableName() + CarbonCommonConstants.FILE_SEPARATOR + configuration.getSegmentId()
+ CarbonCommonConstants.FILE_SEPARATOR + configuration.getTaskNo()),
badRecordsLogRedirect, badRecordsLoggerEnable, badRecordConvertNullDisable, isDataLoadFail);
- return badRecordsLogger;
}
private String getBadLogStoreLocation(String storeLocation) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
index 17cc01e..08b8b73 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
@@ -79,8 +79,7 @@ public class SortProcessorStepImpl extends AbstractDataLoadProcessorStep {
@Override
public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
final Iterator<CarbonRowBatch>[] iterators = child.execute();
- Iterator<CarbonRowBatch>[] sortedIterators = sorter.sort(iterators);
- return sortedIterators;
+ return sorter.sort(iterators);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
index 2b0b8ae..6695a5b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
@@ -267,8 +267,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
prefetchRecordsProceesed++;
returnRow = currentBuffer[bufferRowCounter++];
} else {
- Object[] outRow = getRowFromStream();
- this.returnRow = outRow;
+ this.returnRow = getRowFromStream();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index 61322bc..6fd29d7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -198,8 +198,6 @@ public class CarbonFactDataHandlerModel {
CarbonTableIdentifier identifier =
configuration.getTableIdentifier().getCarbonTableIdentifier();
- CarbonTableIdentifier tableIdentifier =
- identifier;
boolean[] isUseInvertedIndex =
CarbonDataProcessorUtil.getIsUseInvertedIndex(configuration.getDataFields());
@@ -232,11 +230,11 @@ public class CarbonFactDataHandlerModel {
}
CarbonTable carbonTable = CarbonMetadata.getInstance().getCarbonTable(
- tableIdentifier.getDatabaseName() + CarbonCommonConstants.UNDERSCORE + tableIdentifier
+ identifier.getDatabaseName() + CarbonCommonConstants.UNDERSCORE + identifier
.getTableName());
List<ColumnSchema> wrapperColumnSchema = CarbonUtil
- .getColumnSchemaList(carbonTable.getDimensionByTableName(tableIdentifier.getTableName()),
- carbonTable.getMeasureByTableName(tableIdentifier.getTableName()));
+ .getColumnSchemaList(carbonTable.getDimensionByTableName(identifier.getTableName()),
+ carbonTable.getMeasureByTableName(identifier.getTableName()));
int[] colCardinality =
CarbonUtil.getFormattedCardinality(dimLensWithComplex, wrapperColumnSchema);
SegmentProperties segmentProperties =
@@ -388,10 +386,8 @@ public class CarbonFactDataHandlerModel {
.getTableName());
CarbonTablePath carbonTablePath =
CarbonStorePath.getCarbonTablePath(carbonStorePath, carbonTable.getCarbonTableIdentifier());
- String carbonDataDirectoryPath = carbonTablePath
- .getCarbonDataDirectoryPath(configuration.getPartitionId(),
- configuration.getSegmentId() + "");
- return carbonDataDirectoryPath;
+ return carbonTablePath.getCarbonDataDirectoryPath(configuration.getPartitionId(),
+ configuration.getSegmentId() + "");
}
public int[] getColCardinality() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 41bfbed..c6eeb4c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -203,8 +203,7 @@ public final class CarbonDataProcessorUtil {
CarbonStorePath.getCarbonTablePath(baseStorePath, carbonTable.getCarbonTableIdentifier());
String carbonDataDirectoryPath =
carbonTablePath.getCarbonDataDirectoryPath(partitionId, segmentId + "");
- String localDataLoadFolderLocation = carbonDataDirectoryPath + File.separator + taskId;
- return localDataLoadFolderLocation;
+ return carbonDataDirectoryPath + File.separator + taskId;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a0d65f69/processing/src/main/java/org/apache/carbondata/processing/util/NonDictionaryUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/NonDictionaryUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/NonDictionaryUtil.java
index 54b4951..d6f992c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/NonDictionaryUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/NonDictionaryUtil.java
@@ -404,8 +404,7 @@ public class NonDictionaryUtil {
builder.append(CarbonCommonConstants.COMA_SPC_CHARACTER);
}
int lastIndex = builder.lastIndexOf(CarbonCommonConstants.COMA_SPC_CHARACTER);
- String str = -1 != lastIndex ? builder.substring(0, lastIndex) : builder.toString();
- return str;
+ return -1 != lastIndex ? builder.substring(0, lastIndex) : builder.toString();
}
/**
[22/50] [abbrv] incubator-carbondata git commit:
[CARBONDATA-993]Remove confused variable fs & Fixed spelling mistakes This
closes #862
Posted by ja...@apache.org.
[CARBONDATA-993]Remove confused variable fs & Fixed spelling mistakes This closes #862
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/2acb310e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/2acb310e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/2acb310e
Branch: refs/heads/12-dev
Commit: 2acb310ec218a736bc17b411f08db2dcfef2064c
Parents: e2e21e3 8a1cd7d
Author: chenliang613 <ch...@huawei.com>
Authored: Thu Apr 27 15:09:53 2017 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Thu Apr 27 15:09:53 2017 +0800
----------------------------------------------------------------------
.../filesystem/AbstractDFSCarbonFile.java | 24 ++++++++++++--------
1 file changed, 15 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
[05/50] [abbrv] incubator-carbondata git commit: [CARBONDATA-964]
Undate FAQs docment This closes #823
Posted by ja...@apache.org.
[CARBONDATA-964] Undate FAQs docment This closes #823
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/462e29ab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/462e29ab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/462e29ab
Branch: refs/heads/12-dev
Commit: 462e29abece3ae9eb000ef4f98f05416d606fa27
Parents: 0a11c2a 9d70dff
Author: chenliang613 <ch...@huawei.com>
Authored: Sat Apr 22 10:52:50 2017 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Sat Apr 22 10:52:50 2017 +0800
----------------------------------------------------------------------
docs/faq.md | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 55 insertions(+)
----------------------------------------------------------------------
[40/50] [abbrv] incubator-carbondata git commit: Fix
CarbonSessionExample
Posted by ja...@apache.org.
Fix CarbonSessionExample
Fix CarbonSessionExample
Fix CarbonSessionExample
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/b731123c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/b731123c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/b731123c
Branch: refs/heads/12-dev
Commit: b731123c25dc2800e5264381f353d1409cdf7b8f
Parents: 3252e7c
Author: erlu <er...@erludeMacBook-Pro.local>
Authored: Wed Apr 26 11:34:37 2017 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Mon May 1 21:23:56 2017 +0530
----------------------------------------------------------------------
.../java/org/apache/carbondata/core/locks/LocalFileLock.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/b731123c/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
index 4cbfc7a..d357a11 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
@@ -26,6 +26,7 @@ import java.nio.channels.OverlappingFileLockException;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.util.CarbonProperties;
@@ -149,8 +150,9 @@ public class LocalFileLock extends AbstractCarbonLock {
try {
fileOutputStream.close();
// deleting the lock file after releasing the lock.
- if (FileFactory.getCarbonFile(lockFilePath, FileFactory.getFileType(lockFilePath))
- .delete()) {
+ CarbonFile lockFile = FileFactory
+ .getCarbonFile(lockFilePath, FileFactory.getFileType(lockFilePath));
+ if (!lockFile.exists() || lockFile.delete()) {
LOGGER.info("Successfully deleted the lock file " + lockFilePath);
} else {
LOGGER.error("Not able to delete the lock file " + lockFilePath);
[35/50] [abbrv] incubator-carbondata git commit: [CARBONDATA-934]
Cast Filter Expression Push Down to Carbon Optimization. This closes #805
Posted by ja...@apache.org.
[CARBONDATA-934] Cast Filter Expression Push Down to Carbon Optimization. This closes #805
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/b99c2c1a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/b99c2c1a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/b99c2c1a
Branch: refs/heads/12-dev
Commit: b99c2c1a1144ab272ac4d118e388d575d7b6b168
Parents: 472f9c9 96e5b5c
Author: ravipesala <ra...@gmail.com>
Authored: Mon May 1 06:37:37 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Mon May 1 06:37:37 2017 +0530
----------------------------------------------------------------------
.../detailquery/CastColumnTestCase.scala | 973 +++++++++++++++++++
.../RangeFilterAllDataTypesTestCases.scala | 131 ++-
.../detailquery/RangeFilterTestCase.scala | 4 +-
.../apache/carbondata/spark/CarbonFilters.scala | 370 -------
.../apache/spark/sql/CarbonBoundReference.scala | 7 +-
.../spark/sql/optimizer/CarbonFilters.scala | 373 +++++++
.../apache/spark/sql/CarbonBoundReference.scala | 51 +
.../scala/org/apache/spark/sql/CarbonScan.scala | 14 +-
.../spark/sql/SparkUnknownExpression.scala | 27 +-
.../execution/CarbonLateDecodeStrategy.scala | 41 +-
.../execution/CastExpressionOptimization.scala | 373 +++++++
.../spark/sql/optimizer/CarbonFilters.scala | 386 ++++++++
.../carbondata/query/TestNotEqualToFilter.scala | 3 +
13 files changed, 2340 insertions(+), 413 deletions(-)
----------------------------------------------------------------------
[32/50] [abbrv] incubator-carbondata git commit: Update examples for
code style This closes #866
Posted by ja...@apache.org.
Update examples for code style This closes #866
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/472f9c9d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/472f9c9d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/472f9c9d
Branch: refs/heads/12-dev
Commit: 472f9c9d51943be611ad6a7fe92fbf6f90feae4d
Parents: b728b8f 2e8ef57
Author: chenliang613 <ch...@huawei.com>
Authored: Fri Apr 28 17:13:43 2017 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Fri Apr 28 17:13:43 2017 +0800
----------------------------------------------------------------------
.../carbondata/examples/AlterTableExample.scala | 32 ++--
.../examples/CarbonDataFrameExample.scala | 4 +-
.../examples/CarbonSessionExample.scala | 114 +++++++-------
.../examples/SparkSessionExample.scala | 156 +++++++++----------
4 files changed, 158 insertions(+), 148 deletions(-)
----------------------------------------------------------------------
[18/50] [abbrv] incubator-carbondata git commit: [CARBONDATA-985]
Removed unnecessary .show method call in test cases This closes #847
Posted by ja...@apache.org.
[CARBONDATA-985] Removed unnecessary .show method call in test cases This closes #847
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/e579694c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/e579694c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/e579694c
Branch: refs/heads/12-dev
Commit: e579694ccf2e4e79d55cdf6b79cd93cafa98ca28
Parents: 598bbd2 533835c
Author: jackylk <ja...@huawei.com>
Authored: Wed Apr 26 10:38:23 2017 -0700
Committer: jackylk <ja...@huawei.com>
Committed: Wed Apr 26 10:38:23 2017 -0700
----------------------------------------------------------------------
...ncrementalColumnDictionaryGeneratorTest.java | 2 -
.../ServerDictionaryGeneratorTest.java | 1 -
.../hadoop/test/util/StoreCreator.java | 1 -
.../HighCardinalityDataTypesTestCase.scala | 5 +-
.../detailquery/RangeFilterTestCase.scala | 9 -
...TypeDirectDictionaryWithNoDictTestCase.scala | 6 +-
.../carbondata/spark/util/CarbonScalaUtil.scala | 4 -
.../execution/command/carbonTableSchema.scala | 2 +-
.../complexType/TestComplexTypeQuery.scala | 28 +--
.../TestLoadDataWithSingleQuotechar.scala | 2 +-
.../createtable/TestCreateTableSyntax.scala | 6 +-
.../datacompaction/DataCompactionTest.scala | 2 +-
.../iud/DeleteCarbonTableTestCase.scala | 14 +-
.../testsuite/iud/IUDCompactionTestCases.scala | 146 +++++++---------
.../iud/UpdateCarbonTableTestCase.scala | 169 ++++++++-----------
.../AutoHighCardinalityIdentifyTestCase.scala | 4 +-
.../execution/command/carbonTableSchema.scala | 17 +-
.../carbondata/CarbonDataSourceSuite.scala | 2 +-
.../carbon/datastore/BlockIndexStoreTest.java | 2 -
.../carbondata/processing/StoreCreator.java | 12 --
20 files changed, 178 insertions(+), 256 deletions(-)
----------------------------------------------------------------------
[34/50] [abbrv] incubator-carbondata git commit: Cast Push Down
Optimization
Posted by ja...@apache.org.
Cast Push Down Optimization
Furthur Optimize Cast Expression with Early Evaluation
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/96e5b5cc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/96e5b5cc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/96e5b5cc
Branch: refs/heads/12-dev
Commit: 96e5b5ccd9d712a577e4e4b291e1486293383be8
Parents: 472f9c9
Author: sounakr <so...@gmail.com>
Authored: Tue Apr 4 15:16:46 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Mon May 1 06:36:53 2017 +0530
----------------------------------------------------------------------
.../detailquery/CastColumnTestCase.scala | 973 +++++++++++++++++++
.../RangeFilterAllDataTypesTestCases.scala | 131 ++-
.../detailquery/RangeFilterTestCase.scala | 4 +-
.../apache/carbondata/spark/CarbonFilters.scala | 370 -------
.../apache/spark/sql/CarbonBoundReference.scala | 7 +-
.../spark/sql/optimizer/CarbonFilters.scala | 373 +++++++
.../apache/spark/sql/CarbonBoundReference.scala | 51 +
.../scala/org/apache/spark/sql/CarbonScan.scala | 14 +-
.../spark/sql/SparkUnknownExpression.scala | 27 +-
.../execution/CarbonLateDecodeStrategy.scala | 41 +-
.../execution/CastExpressionOptimization.scala | 373 +++++++
.../spark/sql/optimizer/CarbonFilters.scala | 386 ++++++++
.../carbondata/query/TestNotEqualToFilter.scala | 3 +
13 files changed, 2340 insertions(+), 413 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96e5b5cc/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/CastColumnTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/CastColumnTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/CastColumnTestCase.scala
new file mode 100644
index 0000000..bc94350
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/CastColumnTestCase.scala
@@ -0,0 +1,973 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.detailquery
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+/**
+ * Test Class for Range Filters.
+ */
+class CastColumnTestCase extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll {
+ //For the Hive table creation and data loading
+ sql("drop table if exists DICTIONARY_CARBON_1")
+ sql("drop table if exists DICTIONARY_HIVE_1")
+ sql("drop table if exists NO_DICTIONARY_CARBON_2")
+ sql("drop table if exists NO_DICTIONARY_HIVE_2")
+
+ //For Carbon cube creation.
+ sql("CREATE TABLE DICTIONARY_CARBON_1 (empno string, " +
+ "doj Timestamp, workgroupcategory Int, empname String,workgroupcategoryname String, " +
+ "deptno Int, deptname String, projectcode Int, projectjoindate Timestamp, " +
+ "projectenddate Timestamp, designation String,attendance Int,utilization " +
+ "Int,salary Int) STORED BY 'org.apache.carbondata.format' " +
+ "TBLPROPERTIES('DICTIONARY_INCLUDE'='empname,workgroupcategory, designation')"
+ )
+ sql(
+ s"LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE DICTIONARY_CARBON_1 " +
+ "OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+ )
+
+ sql(
+ "create table DICTIONARY_HIVE_1(empno string,empname string,designation string,doj " +
+ "Timestamp,workgroupcategory int, " +
+ "workgroupcategoryname string,deptno int, deptname string, projectcode int, " +
+ "projectjoindate Timestamp,projectenddate Timestamp,attendance int, "
+ + "utilization int,salary int) row format delimited fields terminated by ',' " +
+ "tblproperties(\"skip.header.line.count\"=\"1\") " +
+ ""
+ )
+
+ sql(
+ s"load data local inpath '$resourcesPath/datawithoutheader.csv' into table " +
+ "DICTIONARY_HIVE_1"
+ )
+
+
+ sql("CREATE TABLE NO_DICTIONARY_CARBON_2 (empno string, " +
+ "doj Timestamp, workgroupcategory Int, empname String,workgroupcategoryname String, " +
+ "deptno Int, deptname String, projectcode Int, projectjoindate Timestamp, " +
+ "projectenddate Timestamp, designation String,attendance Int,utilization " +
+ "Int,salary Int) STORED BY 'org.apache.carbondata.format' " +
+ "TBLPROPERTIES('DICTIONARY_INCLUDE'='workgroupcategory', 'DICTIONARY_EXCLUDE'='empno, designation')"
+ )
+ sql(
+ s"LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE NO_DICTIONARY_CARBON_2 " +
+ "OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+ )
+
+ sql(
+ "create table NO_DICTIONARY_HIVE_2(empno string,empname string,designation string,doj " +
+ "Timestamp,workgroupcategory int, " +
+ "workgroupcategoryname string,deptno int, deptname string, projectcode int, " +
+ "projectjoindate Timestamp,projectenddate Timestamp,attendance int, "
+ + "utilization int,salary int) row format delimited fields terminated by ',' " +
+ "tblproperties(\"skip.header.line.count\"=\"1\") " +
+ ""
+ )
+
+ sql(
+ s"load data local inpath '$resourcesPath/datawithoutheader.csv' into table " +
+ "NO_DICTIONARY_HIVE_2"
+ );
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss")
+ }
+ test("Dictionary String ") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno = 11"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno = 11")
+ )
+ }
+
+ test("Dictionary String OR") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno = '11' or empno = '15'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno = '11' or empno = '15'")
+ )
+ }
+
+ test("Dictionary String OR Implicit Cast to int") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno = 11 or empno = 15"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno = 11 or empno = 15")
+ )
+ }
+
+ test("Dictionary String OR explicit Cast to int") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(empno as int) = 11 or cast(empno as int) = 15"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(empno as int) = 11 or cast (empno as int) = 15")
+ )
+ }
+
+ test("Dictionary INT OR to implicit String") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory = '1' or workgroupcategory = '2'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory = '1' or workgroupcategory = '2'")
+ )
+ }
+
+ test("Dictionary INT OR to excplicit String") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(workgroupcategory as string)= '1' or cast(workgroupcategory as string) = '2'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as string) = '1' or cast(workgroupcategory as string) = '2'")
+ )
+ }
+
+ test("Dictionary INT OR to explicit double") {
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(workgroupcategory as double)= 1.0 or cast(workgroupcategory as double) = 2.0").show(2000, false)
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) = 1.0 or cast(workgroupcategory as double) = 2.0 ").show(2000, false)
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(workgroupcategory as double)= 1.0 or cast(workgroupcategory as double) = 2.0"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) = 1.0 or cast(workgroupcategory as double) = 2.0 ")
+ )
+ }
+
+ test("Dictionary String Not") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno != '11'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno != '11'")
+ )
+ }
+
+ test("Dictionary String Not Implicit Cast to int") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno != 11"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno != 11")
+ )
+ }
+
+ test("Dictionary String Not explicit Cast to int") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(empno as int) != 11"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(empno as int) != 11")
+ )
+ }
+
+ test("Dictionary INT Not to implicit String") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory != '1'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory != '1'")
+ )
+ }
+
+ test("Dictionary INT Not to excplicit String") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(workgroupcategory as string) != '1'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as string) != '1'")
+ )
+ }
+
+ test("Dictionary INT Not to explicit double") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(workgroupcategory as double) != 1.0"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) != 1.0")
+ )
+ }
+
+ test("Dictionary String In") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno in ('11', '15')"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno in ('11', '15')")
+ )
+ }
+
+ test("Dictionary String In Implicit ") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno in (11, 15)"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno in (11, 15)")
+ )
+ }
+
+ test("Dictionary String In Explicit ") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (empno as int) in (11, 15)"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast (empno as int) in (11, 15)")
+ )
+ }
+
+ test("Dictionary String In Explicit double") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (empno as double) in (11.0, 15.0)"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast (empno as double) in (11.0, 15.0)")
+ )
+ }
+
+ test("Dictionary String Not in Explicit double 1") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (empno as double) not in (11.0, 15.0)"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast (empno as double) not in (11.0, 15.0)")
+ )
+ }
+
+ test("Dictionary INT In to implicit Int") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory in ('1', '2')"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory in ('1', '2')")
+ )
+ }
+
+ test("Dictionary INT In to excplicit String") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(workgroupcategory as string) in ('1', '2')"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as string) in ('1', '2')")
+ )
+ }
+
+ test("Dictionary INT In to explicit double") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(workgroupcategory as double) in (1.0, 2.0)"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) in (1.0, 2.0)")
+ )
+ }
+
+ test("Dictionary String Greater Than") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno > '11'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno > '11'")
+ )
+ }
+
+ test("Dictionary String Greater Than Implicit") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno > 11"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno > 11")
+ )
+ }
+
+ test("Dictionary String Greater Than explicit 1") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(empno as int) > '11'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast (empno as int) > '11'")
+ )
+ }
+
+
+ test("Dictionary String Greater Than explicit 2") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(empno as int) > 11"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast (empno as int) > 11")
+ )
+ }
+
+ test("Dictionary INT Greater Than") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory > 1"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory > 1")
+ )
+ }
+
+
+ test("Dictionary INT Greater Than implicit ") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory > '1'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory > '1'")
+ )
+ }
+
+
+ test("Dictionary INT Greater Than double") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (workgroupcategory as double) > 1.0"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) > 1.0")
+ )
+ }
+
+
+ test("Dictionary INT Greater Than String ") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (workgroupcategory as double) > '1.0'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) > '1.0'")
+ )
+ }
+
+
+ test("Dictionary String Greater Than equal") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno >= '11'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno >= '11'")
+ )
+ }
+
+ test("Dictionary String Greater Than Implicit equal") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno >= 11"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno >= 11")
+ )
+ }
+
+ test("Dictionary String Greater Than equal explicit 1") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(empno as int) >= '11'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast (empno as int) >= '11'")
+ )
+ }
+
+
+ test("Dictionary String Greater Than equal explicit 2") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(empno as int) >= 11"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast (empno as int) >= 11")
+ )
+ }
+
+ test("Dictionary INT Greater Than equal ") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory >= 1"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory >= 1")
+ )
+ }
+
+
+ test("Dictionary INT Greater Than equal implicit ") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory >= '1'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory >= '1'")
+ )
+ }
+
+
+ test("Dictionary INT Greater Than equal 2") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (workgroupcategory as double) >= 1.0"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) >= 1.0")
+ )
+ }
+
+
+ test("Dictionary INT Greater equal Than") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (workgroupcategory as double) >= '1.0'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) >= '1.0'")
+ )
+ }
+
+ test("Dictionary String less Than") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno < '20'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno < '20'")
+ )
+ }
+
+ test("Dictionary String Less Than Implicit") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno < 20"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno < 20")
+ )
+ }
+
+ test("Dictionary String Less Than explicit 1") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(empno as int) < '20'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast (empno as int) < '20'")
+ )
+ }
+
+
+ test("Dictionary String Less Than explicit 2") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(empno as int) < 20"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast (empno as int) < 20")
+ )
+ }
+
+ test("Dictionary INT Less Than") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory < 2"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory < 2")
+ )
+ }
+
+
+ test("Dictionary INT Less Than implicit ") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory < '2'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory < '2'")
+ )
+ }
+
+
+ test("Dictionary INT Less Than double") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (workgroupcategory as double) < 2.0"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) < 2.0")
+ )
+ }
+
+
+ test("Dictionary INT Less Than String ") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (workgroupcategory as double) < '2.0'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) < '2.0'")
+ )
+ }
+
+
+ test("Dictionary String Less Than equal") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno <= '20'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno <= '20'")
+ )
+ }
+
+ test("Dictionary String Less Than Implicit equal") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno <= 20"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno <= 20")
+ )
+ }
+
+ test("Dictionary String Less Than equal explicit 1") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(empno as int) <= '20'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast (empno as int) <= '20'")
+ )
+ }
+
+
+ test("Dictionary String Less Than equal explicit 2") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(empno as int) <= 20"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast (empno as int) <= 20")
+ )
+ }
+
+ test("Dictionary INT Less Than equal ") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory <= 2"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory <= 2")
+ )
+ }
+
+
+ test("Dictionary INT Less Than equal implicit ") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory <= '2'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory <= '2'")
+ )
+ }
+
+
+ test("Dictionary INT Less Than equal 2") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (workgroupcategory as double) <= 2.0"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) <= 2.0")
+ )
+ }
+
+
+ test("Dictionary INT Less equal Than") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (workgroupcategory as double) <= '2.0'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) <= '2.0'")
+ )
+ }
+
+
+ test("Dictionary INT greater less Than explicit double") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (workgroupcategory as double) <= '2.0' and cast (workgroupcategory as double) >= '1.0'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) <= '2.0' and cast (workgroupcategory as double) >= '1.0'")
+ )
+ }
+
+
+ test("Dictionary INT greater less Than explicit string") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (workgroupcategory as string) <= '2.0' and cast (workgroupcategory as string) >= '1.0'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as string) <= '2.0' and cast (workgroupcategory as string) >= '1.0'")
+ )
+ }
+
+
+ test("Dictionary INT greater less Than implicit string") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory <= '2.0' and workgroupcategory >= '1.0'"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory <= '2.0' and workgroupcategory >= '1.0'")
+ )
+ }
+
+
+ test("Dictionary INT greater less Than implicit int") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory <= 2 and workgroupcategory >= 1"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory <= 2 and workgroupcategory >= 1")
+ )
+ }
+
+
+ test("Dictionary INT greater less Than implicit double") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory <= 2.0 and workgroupcategory >= 1.0"),
+ sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory <= 2.0 and workgroupcategory >= 1.0")
+ )
+ }
+
+
+ // No dictionary cases
+ test("NO Dictionary String ") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno = 11"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno = 11")
+ )
+ }
+
+ test("NO Dictionary String OR") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno = '11' or empno = '15'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno = '11' or empno = '15'")
+ )
+ }
+
+ test("NO Dictionary String OR Implicit Cast to int") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno = 11 or empno = 15"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno = 11 or empno = 15")
+ )
+ }
+
+ test("NO Dictionary String OR explicit Cast to int") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(empno as int) = 11 or cast(empno as int) = 15"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(empno as int) = 11 or cast (empno as int) = 15")
+ )
+ }
+
+ test("NO Dictionary INT OR to implicit String") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory = '1' or workgroupcategory = '2'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory = '1' or workgroupcategory = '2'")
+ )
+ }
+
+ test("NO Dictionary INT OR to excplicit String") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(workgroupcategory as string)= '1' or cast(workgroupcategory as string) = '2'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as string) = '1' or cast(workgroupcategory as string) = '2'")
+ )
+ }
+
+ test("NO Dictionary INT OR to explicit double") {
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(workgroupcategory as double)= 1.0 or cast(workgroupcategory as double) = 2.0").show(2000, false)
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) = 1.0 or cast(workgroupcategory as double) = 2.0 ").show(2000, false)
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(workgroupcategory as double)= 1.0 or cast(workgroupcategory as double) = 2.0"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) = 1.0 or cast(workgroupcategory as double) = 2.0 ")
+ )
+ }
+
+ test("NO Dictionary String Not") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno != '11'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno != '11'")
+ )
+ }
+
+ test("NO Dictionary String Not Implicit Cast to int") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno != 11"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno != 11")
+ )
+ }
+
+ test("NO Dictionary String Not explicit Cast to int") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(empno as int) != 11"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(empno as int) != 11")
+ )
+ }
+
+ test("NO Dictionary INT Not to implicit String") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory != '1'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory != '1'")
+ )
+ }
+
+ test("NO Dictionary INT Not to excplicit String") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(workgroupcategory as string) != '1'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as string) != '1'")
+ )
+ }
+
+ test("NO Dictionary INT Not to explicit double") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(workgroupcategory as double) != 1.0"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) != 1.0")
+ )
+ }
+
+ test("NO Dictionary String In") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno in ('11', '15')"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno in ('11', '15')")
+ )
+ }
+
+ test("NO Dictionary String In Implicit ") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno in (11, 15)"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno in (11, 15)")
+ )
+ }
+
+ test("NO Dictionary String In Explicit ") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (empno as int) in (11, 15)"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast (empno as int) in (11, 15)")
+ )
+ }
+
+ test("NO Dictionary String In Explicit double") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (empno as double) in (11.0, 15.0)"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast (empno as double) in (11.0, 15.0)")
+ )
+ }
+
+ test("NO Dictionary String Not in Explicit double 1") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (empno as double) not in (11.0, 15.0)"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast (empno as double) not in (11.0, 15.0)")
+ )
+ }
+
+ test("NO Dictionary INT In to implicit Int") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory in ('1', '2')"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory in ('1', '2')")
+ )
+ }
+
+ test("NO Dictionary INT In to excplicit String") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(workgroupcategory as string) in ('1', '2')"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as string) in ('1', '2')")
+ )
+ }
+
+ test("NO Dictionary INT In to explicit double") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(workgroupcategory as double) in (1.0, 2.0)"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) in (1.0, 2.0)")
+ )
+ }
+
+ test("NO Dictionary String Greater Than") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno > '11'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno > '11'")
+ )
+ }
+
+ test("NO Dictionary String Greater Than Implicit") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno > 11"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno > 11")
+ )
+ }
+
+ test("NO Dictionary String Greater Than explicit 1") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(empno as int) > '11'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast (empno as int) > '11'")
+ )
+ }
+
+
+ test("NO Dictionary String Greater Than explicit 2") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(empno as int) > 11"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast (empno as int) > 11")
+ )
+ }
+
+ test("NO Dictionary INT Greater Than") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory > 1"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory > 1")
+ )
+ }
+
+
+ test("NO Dictionary INT Greater Than implicit ") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory > '1'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory > '1'")
+ )
+ }
+
+
+ test("NO Dictionary INT Greater Than double") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (workgroupcategory as double) > 1.0"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) > 1.0")
+ )
+ }
+
+
+ test("NO Dictionary INT Greater Than String ") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (workgroupcategory as double) > '1.0'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) > '1.0'")
+ )
+ }
+
+
+ test("NO Dictionary String Greater Than equal") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno >= '11'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno >= '11'")
+ )
+ }
+
+ test("NO Dictionary String Greater Than Implicit equal") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno >= 11"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno >= 11")
+ )
+ }
+
+ test("NO Dictionary String Greater Than equal explicit 1") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(empno as int) >= '11'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast (empno as int) >= '11'")
+ )
+ }
+
+
+ test("NO Dictionary String Greater Than equal explicit 2") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(empno as int) >= 11"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast (empno as int) >= 11")
+ )
+ }
+
+ test("NO Dictionary INT Greater Than equal ") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory >= 1"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory >= 1")
+ )
+ }
+
+
+ test("NO Dictionary INT Greater Than equal implicit ") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory >= '1'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory >= '1'")
+ )
+ }
+
+
+ test("NO Dictionary INT Greater Than equal 2") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (workgroupcategory as double) >= 1.0"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) >= 1.0")
+ )
+ }
+
+
+ test("NO Dictionary INT Greater equal Than") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (workgroupcategory as double) >= '1.0'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) >= '1.0'")
+ )
+ }
+
+ test("NO Dictionary String less Than") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno < '20'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno < '20'")
+ )
+ }
+
+ test("NO Dictionary String Less Than Implicit") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno < 20"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno < 20")
+ )
+ }
+
+ test("NO Dictionary String Less Than explicit 1") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(empno as int) < '20'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast (empno as int) < '20'")
+ )
+ }
+
+
+ test("NO Dictionary String Less Than explicit 2") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(empno as int) < 20"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast (empno as int) < 20")
+ )
+ }
+
+ test("NO Dictionary INT Less Than") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory < 2"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory < 2")
+ )
+ }
+
+
+ test("NO Dictionary INT Less Than implicit ") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory < '2'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory < '2'")
+ )
+ }
+
+
+ test("NO Dictionary INT Less Than double") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (workgroupcategory as double) < 2.0"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) < 2.0")
+ )
+ }
+
+
+ test("NO Dictionary INT Less Than String ") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (workgroupcategory as double) < '2.0'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) < '2.0'")
+ )
+ }
+
+
+ test("NO Dictionary String Less Than equal") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno <= '20'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno <= '20'")
+ )
+ }
+
+ test("NO Dictionary String Less Than Implicit equal") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno <= 20"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno <= 20")
+ )
+ }
+
+ test("NO Dictionary String Less Than equal explicit 1") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(empno as int) <= '20'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast (empno as int) <= '20'")
+ )
+ }
+
+
+ test("NO Dictionary String Less Than equal explicit 2") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(empno as int) <= 20"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast (empno as int) <= 20")
+ )
+ }
+
+ test("NO Dictionary INT Less Than equal ") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory <= 2"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory <= 2")
+ )
+ }
+
+
+ test("NO Dictionary INT Less Than equal implicit ") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory <= '2'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory <= '2'")
+ )
+ }
+
+
+ test("NO Dictionary INT Less Than equal 2") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (workgroupcategory as double) <= 2.0"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) <= 2.0")
+ )
+ }
+
+
+ test("NO Dictionary INT Less equal Than") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (workgroupcategory as double) <= '2.0'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) <= '2.0'")
+ )
+ }
+
+
+ test("NO Dictionary INT greater less Than explicit double") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (workgroupcategory as double) <= '2.0' and cast (workgroupcategory as double) >= '1.0'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) <= '2.0' and cast (workgroupcategory as double) >= '1.0'")
+ )
+ }
+
+
+ test("NO Dictionary INT greater less Than explicit string") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (workgroupcategory as string) <= '2.0' and cast (workgroupcategory as string) >= '1.0'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as string) <= '2.0' and cast (workgroupcategory as string) >= '1.0'")
+ )
+ }
+
+
+ test("NO Dictionary INT greater less Than implicit string") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory <= '2.0' and workgroupcategory >= '1.0'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory <= '2.0' and workgroupcategory >= '1.0'")
+ )
+ }
+
+
+ test("NO Dictionary INT greater less Than implicit int") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory <= 2 and workgroupcategory >= 1"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory <= 2 and workgroupcategory >= 1")
+ )
+ }
+
+
+ test("NO Dictionary INT greater less Than implicit double") {
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory <= 2.0 and workgroupcategory >= 1.0"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory <= 2.0 and workgroupcategory >= 1.0")
+ )
+ }
+
+ // Direct Dictionary and Timestamp Column.
+
+
+
+
+ override def afterAll {
+ sql("drop table if exists DICTIONARY_CARBON_1")
+ sql("drop table if exists DICTIONARY_HIVE_1")
+ sql("drop table if exists NO_DICTIONARY_CARBON_2")
+ sql("drop table if exists NO_DICTIONARY_HIVE_2")
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96e5b5cc/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
index 68b08a3..3f42d56 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
@@ -242,6 +242,7 @@ class RangeFilterMyTests extends QueryTest with BeforeAndAfterAll {
}
test("test range filter for direct dictionary"){
+ sql("select doj from directDictionaryTable where doj > cast('2016-03-14 15:00:17' as timestamp)").show(2000,false)
checkAnswer(
sql("select doj from directDictionaryTable where doj > '2016-03-14 15:00:17'"),
sql("select doj from directDictionaryTable_hive where doj > '2016-03-14 15:00:17'")
@@ -256,6 +257,110 @@ class RangeFilterMyTests extends QueryTest with BeforeAndAfterAll {
)
}
+ test("test range filter for direct dictionary equality"){
+ checkAnswer(
+ sql("select doj from directDictionaryTable where doj = '2016-03-14 15:00:16'"),
+ Seq(Row(Timestamp.valueOf("2016-03-14 15:00:16.0"))
+ )
+ )
+ }
+
+ test("test range filter for direct dictionary not equality"){
+ sql("select doj from directDictionaryTable where doj != '2016-03-14 15:00:16'").show(2000, false)
+ checkAnswer(
+ sql("select doj from directDictionaryTable where doj != '2016-03-14 15:00:16'"),
+ Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:10.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:11.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:12.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:13.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:14.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:15.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:17.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:18.0")))
+ )
+ }
+
+ test("test range filter for direct dictionary and with explicit casts"){
+ sql("select doj from directDictionaryTable where doj > cast ('2016-03-14 15:00:16' as timestamp) and doj < cast ('2016-03-14 15:00:18' as timestamp)").show(2000, false)
+ checkAnswer(
+ sql("select doj from directDictionaryTable where doj > cast ('2016-03-14 15:00:16' as timestamp) and doj < cast ('2016-03-14 15:00:18' as timestamp)"),
+ Seq(Row(Timestamp.valueOf("2016-03-14 15:00:17.0"))
+ )
+ )
+ }
+
+ /*
+ Commented this test case
+ test("test range filter for direct dictionary and with DirectVals as long") {
+ checkAnswer(
+ sql(
+ "select doj from directDictionaryTable where doj > cast (1457992816l as timestamp) and doj < cast (1457992818l as timestamp)"),
+ Seq(Row(Timestamp.valueOf("2016-03-14 15:00:17.0"))
+ )
+ )
+ }
+ */
+
+
+ // Test of Cast Optimization
+ test("test range filter for different Timestamp formats"){
+ checkAnswer(
+ sql("select count(*) from directDictionaryTable where doj = '2016-03-14 15:00:180000000'"),
+ Seq(Row(0)
+ )
+ )
+ }
+
+ test("test range filter for different Timestamp formats1"){
+ checkAnswer(
+ sql("select count(*) from directDictionaryTable where doj = '03-03-14 15:00:18'"),
+ Seq(Row(0)
+ )
+ )
+ }
+
+ test("test range filter for different Timestamp formats2"){
+ checkAnswer(
+ sql("select count(*) from directDictionaryTable where doj = '2016-03-14'"),
+ Seq(Row(0)
+ )
+ )
+ }
+
+ test("test range filter for different Timestamp formats3"){
+ checkAnswer(
+ sql("select count(*) from directDictionaryTable where doj > '2016-03-14 15:00:18.000'"),
+ sql("select count(*) from directDictionaryTable_hive where doj > '2016-03-14 15:00:18.000'")
+ )
+ }
+
+ test("test range filter for different Timestamp In format "){
+ sql("select doj from directDictionaryTable").show(200,false)
+ sql("select * from directDictionaryTable where doj in ('2016-03-14 15:00:18', '2016-03-14 15:00:17' )").show(200, false)
+ checkAnswer(
+ sql("select doj from directDictionaryTable where doj in ('2016-03-14 15:00:18', '2016-03-14 15:00:17')"),
+ Seq(Row(Timestamp.valueOf("2016-03-14 15:00:17.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:18.0")))
+ )
+ }
+
+ /*
+ test("test range filter for different Timestamp Not In format 5"){
+ sql("select doj from directDictionaryTable where doj not in (null, '2016-03-14 15:00:18', '2016-03-14 15:00:17','2016-03-14 15:00:11', '2016-03-14 15:00:12')").show(200, false)
+ checkAnswer(
+ sql("select doj from directDictionaryTable where doj Not in (null, '2016-03-14 15:00:18', '2016-03-14 15:00:17','2016-03-14 15:00:11', '2016-03-14 15:00:12')"),
+ Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:10.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:13.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:14.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:15.0")),
+ Row(Timestamp.valueOf("2016-03-14 15:00:16.0")))
+ )
+ }
+ */
+
+
test("test range filter for direct dictionary and boundary"){
checkAnswer(
sql("select count(*) from directDictionaryTable where doj > '2016-03-14 15:00:18.0' and doj < '2016-03-14 15:00:09.0'"),
@@ -510,6 +615,13 @@ class RangeFilterMyTests extends QueryTest with BeforeAndAfterAll {
)
}
+ test("test range filter No Dictionary Range"){
+ checkAnswer(
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_8 where empno > '13' and empno < '17'"),
+ sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_8 where empno > '13' and empno < '17'")
+ )
+ }
+
test("test range filter for more columns conditions"){
checkAnswer(
sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_8 where empno > '13' and workgroupcategory < 3 and deptno > 12 and empno < '17'"),
@@ -531,25 +643,6 @@ class RangeFilterMyTests extends QueryTest with BeforeAndAfterAll {
)
}
- /*
- test("test range filter with complex datatypes 1"){
- sql("select proddate.productionDate from complexcarbontable where proddate.productionDate > '30-11-2015' and proddate.productionDate < '31-01-2016' ").show()
- checkAnswer(
- sql("select proddate.productionDate from complexcarbontable where proddate.productionDate > '30-11-2015' and proddate.productionDate < '31-01-2016'"),
- Seq(Row("30-12-2015"))
- )
- }
- */
-
- /*
- test("test range filter with complex datatypes 2"){
- sql("select ROMSize,mobile.imei from complexcarbontable where proddate.productionDate > '30-11-2015' and proddate.productionDate < '31-01-2016' ").show()
- checkAnswer(
- sql("select ROMSize,mobile.imei from complexcarbontable where proddate.productionDate > '30-11-2015' and proddate.productionDate < '31-01-2016'"),
- Seq(Row("8ROM size",""))
- )
- }
- */
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96e5b5cc/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterTestCase.scala
index 3dd01cc..46b514a 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterTestCase.scala
@@ -89,7 +89,7 @@ class RangeFilterTestCase extends QueryTest with BeforeAndAfterAll {
"deptno Int, deptname String, projectcode Int, projectjoindate Timestamp, " +
"projectenddate Timestamp, designation String,attendance Int,utilization " +
"Int,salary Int) STORED BY 'org.apache.carbondata.format' " +
- "TBLPROPERTIES('DICTIONARY_EXCLUDE'='empno, empname,designation')"
+ "TBLPROPERTIES('DICTIONARY_INCLUDE'='workgroupcategory','DICTIONARY_EXCLUDE'='empno, empname,designation')"
)
sql(
s"LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE NO_DICTIONARY_CARBON_6 " +
@@ -102,7 +102,7 @@ class RangeFilterTestCase extends QueryTest with BeforeAndAfterAll {
"deptno Int, deptname String, projectcode Int, projectjoindate Timestamp, " +
"projectenddate Timestamp, designation String,attendance Int,utilization " +
"Int,salary Int) STORED BY 'org.apache.carbondata.format' " +
- "TBLPROPERTIES('DICTIONARY_EXCLUDE'='empname,designation')"
+ "TBLPROPERTIES('DICTIONARY_INCLUDE'='workgroupcategory','DICTIONARY_EXCLUDE'='empname,designation')"
)
sql(
s"LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE DICTIONARY_CARBON_6 " +
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96e5b5cc/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala
deleted file mode 100644
index 5698dab..0000000
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala
+++ /dev/null
@@ -1,370 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark
-
-import scala.collection.mutable.ArrayBuffer
-
-import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.optimizer.AttributeReferenceWrapper
-import org.apache.spark.sql.sources
-import org.apache.spark.sql.types.StructType
-
-import org.apache.carbondata.core.metadata.datatype.DataType
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn
-import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonColumnExpression, Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression}
-import org.apache.carbondata.core.scan.expression.conditional._
-import org.apache.carbondata.core.scan.expression.logical.{AndExpression, FalseExpression, OrExpression}
-import org.apache.carbondata.spark.util.CarbonScalaUtil
-
-/**
- * All filter conversions are done here.
- */
-object CarbonFilters {
-
- /**
- * Converts data sources filters to carbon filter predicates.
- */
- def createCarbonFilter(schema: StructType,
- predicate: sources.Filter): Option[CarbonExpression] = {
- val dataTypeOf = schema.map(f => f.name -> f.dataType).toMap
-
- def createFilter(predicate: sources.Filter): Option[CarbonExpression] = {
- predicate match {
-
- case sources.EqualTo(name, value) =>
- Some(new EqualToExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
- case sources.Not(sources.EqualTo(name, value)) =>
- Some(new NotEqualsExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
-
- case sources.EqualNullSafe(name, value) =>
- Some(new EqualToExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
- case sources.Not(sources.EqualNullSafe(name, value)) =>
- Some(new NotEqualsExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
-
- case sources.GreaterThan(name, value) =>
- Some(new GreaterThanExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
- case sources.LessThan(name, value) =>
- Some(new LessThanExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
- case sources.GreaterThanOrEqual(name, value) =>
- Some(new GreaterThanEqualToExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
- case sources.LessThanOrEqual(name, value) =>
- Some(new LessThanEqualToExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, value)))
-
- case sources.In(name, values) =>
- Some(new InExpression(getCarbonExpression(name),
- new ListExpression(
- convertToJavaList(values.map(f => getCarbonLiteralExpression(name, f)).toList))))
- case sources.Not(sources.In(name, values)) =>
- Some(new NotInExpression(getCarbonExpression(name),
- new ListExpression(
- convertToJavaList(values.map(f => getCarbonLiteralExpression(name, f)).toList))))
-
- case sources.IsNull(name) =>
- Some(new EqualToExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, null), true))
- case sources.IsNotNull(name) =>
- Some(new NotEqualsExpression(getCarbonExpression(name),
- getCarbonLiteralExpression(name, null), true))
-
- case sources.And(lhs, rhs) =>
- (createFilter(lhs) ++ createFilter(rhs)).reduceOption(new AndExpression(_, _))
-
- case sources.Or(lhs, rhs) =>
- for {
- lhsFilter <- createFilter(lhs)
- rhsFilter <- createFilter(rhs)
- } yield {
- new OrExpression(lhsFilter, rhsFilter)
- }
-
- case _ => None
- }
- }
-
- def getCarbonExpression(name: String) = {
- new CarbonColumnExpression(name,
- CarbonScalaUtil.convertSparkToCarbonDataType(dataTypeOf(name)))
- }
-
- def getCarbonLiteralExpression(name: String, value: Any): CarbonExpression = {
- val dataTypeOfAttribute = CarbonScalaUtil.convertSparkToCarbonDataType(dataTypeOf(name))
- val dataType = if (Option(value).isDefined
- && dataTypeOfAttribute == DataType.STRING
- && value.isInstanceOf[Double]) {
- DataType.DOUBLE
- } else {
- dataTypeOfAttribute
- }
- new CarbonLiteralExpression(value, dataType)
- }
-
- createFilter(predicate)
- }
-
-
- // Check out which filters can be pushed down to carbon, remaining can be handled in spark layer.
- // Mostly dimension filters are only pushed down since it is faster in carbon.
- def selectFilters(filters: Seq[Expression],
- attrList: java.util.HashSet[AttributeReferenceWrapper],
- aliasMap: CarbonAliasDecoderRelation): Unit = {
- def translate(expr: Expression, or: Boolean = false): Option[sources.Filter] = {
- expr match {
- case or@ Or(left, right) =>
-
- val leftFilter = translate(left, or = true)
- val rightFilter = translate(right, or = true)
- if (leftFilter.isDefined && rightFilter.isDefined) {
- Some( sources.Or(leftFilter.get, rightFilter.get))
- } else {
- or.collect {
- case attr: AttributeReference =>
- attrList.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
- }
- None
- }
-
- case And(left, right) =>
- val leftFilter = translate(left, or)
- val rightFilter = translate(right, or)
- if (or) {
- if (leftFilter.isDefined && rightFilter.isDefined) {
- (leftFilter ++ rightFilter).reduceOption(sources.And)
- } else {
- None
- }
- } else {
- (leftFilter ++ rightFilter).reduceOption(sources.And)
- }
-
- case EqualTo(a: Attribute, Literal(v, t)) =>
- Some(sources.EqualTo(a.name, v))
- case EqualTo(l@Literal(v, t), a: Attribute) =>
- Some(sources.EqualTo(a.name, v))
- case Not(EqualTo(a: Attribute, Literal(v, t))) =>
- Some(sources.Not(sources.EqualTo(a.name, v)))
- case Not(EqualTo(Literal(v, t), a: Attribute)) =>
- Some(sources.Not(sources.EqualTo(a.name, v)))
- case IsNotNull(a: Attribute) =>
- Some(sources.IsNotNull(a.name))
- case IsNull(a: Attribute) =>
- Some(sources.IsNull(a.name))
- case Not(In(a: Attribute, list)) if !list.exists(!_.isInstanceOf[Literal]) =>
- val hSet = list.map(e => e.eval(EmptyRow))
- Some(sources.Not(sources.In(a.name, hSet.toArray)))
- case In(a: Attribute, list) if !list.exists(!_.isInstanceOf[Literal]) =>
- val hSet = list.map(e => e.eval(EmptyRow))
- Some(sources.In(a.name, hSet.toArray))
- case GreaterThan(a: Attribute, Literal(v, t)) =>
- Some(sources.GreaterThan(a.name, v))
- case GreaterThan(Literal(v, t), a: Attribute) =>
- Some(sources.LessThan(a.name, v))
- case LessThan(a: Attribute, Literal(v, t)) =>
- Some(sources.LessThan(a.name, v))
- case LessThan(Literal(v, t), a: Attribute) =>
- Some(sources.GreaterThan(a.name, v))
- case GreaterThanOrEqual(a: Attribute, Literal(v, t)) =>
- Some(sources.GreaterThanOrEqual(a.name, v))
- case GreaterThanOrEqual(Literal(v, t), a: Attribute) =>
- Some(sources.LessThanOrEqual(a.name, v))
- case LessThanOrEqual(a: Attribute, Literal(v, t)) =>
- Some(sources.LessThanOrEqual(a.name, v))
- case LessThanOrEqual(Literal(v, t), a: Attribute) =>
- Some(sources.GreaterThanOrEqual(a.name, v))
-
- case others =>
- if (!or) {
- others.collect {
- case attr: AttributeReference =>
- attrList.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
- }
- }
- None
- }
- }
- filters.flatMap(translate(_, false)).toArray
- }
-
- def processExpression(exprs: Seq[Expression],
- attributesNeedToDecode: java.util.HashSet[AttributeReference],
- unprocessedExprs: ArrayBuffer[Expression],
- carbonTable: CarbonTable): Option[CarbonExpression] = {
- def transformExpression(expr: Expression, or: Boolean = false): Option[CarbonExpression] = {
- expr match {
- case orFilter@ Or(left, right) =>
- val leftFilter = transformExpression(left, or = true)
- val rightFilter = transformExpression(right, or = true)
- if (leftFilter.isDefined && rightFilter.isDefined) {
- Some(new OrExpression(leftFilter.get, rightFilter.get))
- } else {
- if (!or) {
- orFilter.collect {
- case attr: AttributeReference => attributesNeedToDecode.add(attr)
- }
- unprocessedExprs += orFilter
- }
- None
- }
-
- case And(left, right) =>
- val leftFilter = transformExpression(left, or)
- val rightFilter = transformExpression(right, or)
- if (or) {
- if (leftFilter.isDefined && rightFilter.isDefined) {
- (leftFilter ++ rightFilter).reduceOption(new AndExpression(_, _))
- } else {
- None
- }
- } else {
- (leftFilter ++ rightFilter).reduceOption(new AndExpression(_, _))
- }
-
-
- case EqualTo(a: Attribute, l@Literal(v, t)) =>
- Some(
- new EqualToExpression(
- transformExpression(a).get,
- transformExpression(l).get
- )
- )
- case EqualTo(l@Literal(v, t), a: Attribute) =>
- Some(
- new EqualToExpression(
- transformExpression(a).get,
- transformExpression(l).get
- )
- )
-
- case Not(EqualTo(a: Attribute, l@Literal(v, t))) =>
- Some(
- new NotEqualsExpression(
- transformExpression(a).get,
- transformExpression(l).get
- )
- )
- case Not(EqualTo(l@Literal(v, t), a: Attribute)) =>
- Some(
- new NotEqualsExpression(
- transformExpression(a).get,
- transformExpression(l).get
- )
- )
- case IsNotNull(child: Attribute) =>
- Some(new NotEqualsExpression(transformExpression(child).get,
- transformExpression(Literal(null)).get, true))
- case IsNull(child: Attribute) =>
- Some(new EqualToExpression(transformExpression(child).get,
- transformExpression(Literal(null)).get, true))
- case Not(In(a: Attribute, list))
- if !list.exists(!_.isInstanceOf[Literal]) =>
- if (list.exists(x => isNullLiteral(x.asInstanceOf[Literal]))) {
- Some(new FalseExpression(transformExpression(a).get))
- } else {
- Some(new NotInExpression(transformExpression(a).get,
- new ListExpression(convertToJavaList(list.map(transformExpression(_).get)))))
- }
- case In(a: Attribute, list) if !list.exists(!_.isInstanceOf[Literal]) =>
- Some(new InExpression(transformExpression(a).get,
- new ListExpression(convertToJavaList(list.map(transformExpression(_).get)))))
-
- case GreaterThan(a: Attribute, l@Literal(v, t)) =>
- Some(new GreaterThanExpression(transformExpression(a).get, transformExpression(l).get))
- case GreaterThan(l@Literal(v, t), a: Attribute) =>
- Some(new LessThanExpression(transformExpression(a).get, transformExpression(l).get))
-
- case LessThan(a: Attribute, l@Literal(v, t)) =>
- Some(new LessThanExpression(transformExpression(a).get, transformExpression(l).get))
- case LessThan(l@Literal(v, t), a: Attribute) =>
- Some(new GreaterThanExpression(transformExpression(a).get, transformExpression(l).get))
-
- case GreaterThanOrEqual(a: Attribute, l@Literal(v, t)) =>
- Some(new GreaterThanEqualToExpression(transformExpression(a).get,
- transformExpression(l).get))
- case GreaterThanOrEqual(l@Literal(v, t), a: Attribute) =>
- Some(new LessThanEqualToExpression(transformExpression(a).get,
- transformExpression(l).get))
-
- case LessThanOrEqual(a: Attribute, l@Literal(v, t)) =>
- Some(new LessThanEqualToExpression(transformExpression(a).get,
- transformExpression(l).get))
- case LessThanOrEqual(l@Literal(v, t), a: Attribute) =>
- Some(new GreaterThanEqualToExpression(transformExpression(a).get,
- transformExpression(l).get))
-
- case AttributeReference(name, dataType, _, _) =>
- Some(new CarbonColumnExpression(name,
- CarbonScalaUtil.convertSparkToCarbonDataType(
- getActualCarbonDataType(name, carbonTable))))
- case Literal(name, dataType) => Some(new
- CarbonLiteralExpression(name, CarbonScalaUtil.convertSparkToCarbonDataType(dataType)))
- case others =>
- if (!or) {
- others.collect {
- case attr: AttributeReference => attributesNeedToDecode.add(attr)
- }
- unprocessedExprs += others
- }
- None
- }
- }
- exprs.flatMap(transformExpression(_, false)).reduceOption(new AndExpression(_, _))
- }
- private def isNullLiteral(exp: Expression): Boolean = {
- if (null != exp
- && exp.isInstanceOf[Literal]
- && (exp.asInstanceOf[Literal].dataType == org.apache.spark.sql.types.DataTypes.NullType)
- || (exp.asInstanceOf[Literal].value == null)) {
- true
- } else {
- false
- }
- }
- private def getActualCarbonDataType(column: String, carbonTable: CarbonTable) = {
- var carbonColumn: CarbonColumn =
- carbonTable.getDimensionByName(carbonTable.getFactTableName, column)
- val dataType = if (carbonColumn != null) {
- carbonColumn.getDataType
- } else {
- carbonColumn = carbonTable.getMeasureByName(carbonTable.getFactTableName, column)
- carbonColumn.getDataType match {
- case DataType.INT => DataType.LONG
- case DataType.LONG => DataType.LONG
- case DataType.DECIMAL => DataType.DECIMAL
- case _ => DataType.DOUBLE
- }
- }
- CarbonScalaUtil.convertCarbonToSparkDataType(dataType)
- }
-
- // Convert scala list to java list, Cannot use scalaList.asJava as while deserializing it is
- // not able find the classes inside scala list and gives ClassNotFoundException.
- private def convertToJavaList(
- scalaList: Seq[CarbonExpression]): java.util.List[CarbonExpression] = {
- val javaList = new java.util.ArrayList[CarbonExpression]()
- scalaList.foreach(javaList.add)
- javaList
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96e5b5cc/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
index 3bb1efd..a6c28a9 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
@@ -18,12 +18,14 @@
package org.apache.spark.sql
import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.{Attribute, ExprId, LeafExpression, NamedExpression}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, ExprId, LeafExpression, NamedExpression}
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
+import org.apache.spark.sql.sources.Filter
import org.apache.spark.sql.types.DataType
import org.apache.carbondata.core.scan.expression.ColumnExpression
+case class CastExpr(expr: Expression) extends Filter
case class CarbonBoundReference(colExp: ColumnExpression, dataType: DataType, nullable: Boolean)
extends LeafExpression with NamedExpression with CodegenFallback {
@@ -38,8 +40,7 @@ case class CarbonBoundReference(colExp: ColumnExpression, dataType: DataType, nu
override def toAttribute: Attribute = throw new UnsupportedOperationException
- override def qualifiers: Seq[String] = throw new UnsupportedOperationException
-
override def exprId: ExprId = throw new UnsupportedOperationException
+ override def qualifiers: Seq[String] = throw new UnsupportedOperationException
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96e5b5cc/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
new file mode 100644
index 0000000..cf8689c
--- /dev/null
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
@@ -0,0 +1,373 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark
+
+import scala.collection.mutable.ArrayBuffer
+
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.optimizer.AttributeReferenceWrapper
+import org.apache.spark.sql.sources
+import org.apache.spark.sql.types.StructType
+
+import org.apache.carbondata.core.metadata.datatype.DataType
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn
+import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonColumnExpression, Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression}
+import org.apache.carbondata.core.scan.expression.conditional._
+import org.apache.carbondata.core.scan.expression.logical.{AndExpression, FalseExpression, OrExpression}
+import org.apache.carbondata.spark.util.CarbonScalaUtil
+
+/**
+ * All filter conversions are done here.
+ */
+object CarbonFilters {
+
+ /**
+ * Converts data sources filters to carbon filter predicates.
+ */
+ def createCarbonFilter(schema: StructType,
+ predicate: sources.Filter): Option[CarbonExpression] = {
+ val dataTypeOf = schema.map(f => f.name -> f.dataType).toMap
+
+ def createFilter(predicate: sources.Filter): Option[CarbonExpression] = {
+ predicate match {
+
+ case sources.EqualTo(name, value) =>
+ Some(new EqualToExpression(getCarbonExpression(name),
+ getCarbonLiteralExpression(name, value)))
+ case sources.Not(sources.EqualTo(name, value)) =>
+ Some(new NotEqualsExpression(getCarbonExpression(name),
+ getCarbonLiteralExpression(name, value)))
+
+ case sources.EqualNullSafe(name, value) =>
+ Some(new EqualToExpression(getCarbonExpression(name),
+ getCarbonLiteralExpression(name, value)))
+ case sources.Not(sources.EqualNullSafe(name, value)) =>
+ Some(new NotEqualsExpression(getCarbonExpression(name),
+ getCarbonLiteralExpression(name, value)))
+
+ case sources.GreaterThan(name, value) =>
+ Some(new GreaterThanExpression(getCarbonExpression(name),
+ getCarbonLiteralExpression(name, value)))
+ case sources.LessThan(name, value) =>
+ Some(new LessThanExpression(getCarbonExpression(name),
+ getCarbonLiteralExpression(name, value)))
+ case sources.GreaterThanOrEqual(name, value) =>
+ Some(new GreaterThanEqualToExpression(getCarbonExpression(name),
+ getCarbonLiteralExpression(name, value)))
+ case sources.LessThanOrEqual(name, value) =>
+ Some(new LessThanEqualToExpression(getCarbonExpression(name),
+ getCarbonLiteralExpression(name, value)))
+
+ case sources.In(name, values) =>
+ Some(new InExpression(getCarbonExpression(name),
+ new ListExpression(
+ convertToJavaList(values.map(f => getCarbonLiteralExpression(name, f)).toList))))
+ case sources.Not(sources.In(name, values)) =>
+ Some(new NotInExpression(getCarbonExpression(name),
+ new ListExpression(
+ convertToJavaList(values.map(f => getCarbonLiteralExpression(name, f)).toList))))
+
+ case sources.IsNull(name) =>
+ Some(new EqualToExpression(getCarbonExpression(name),
+ getCarbonLiteralExpression(name, null), true))
+ case sources.IsNotNull(name) =>
+ Some(new NotEqualsExpression(getCarbonExpression(name),
+ getCarbonLiteralExpression(name, null), true))
+
+ case sources.And(lhs, rhs) =>
+ (createFilter(lhs) ++ createFilter(rhs)).reduceOption(new AndExpression(_, _))
+
+ case sources.Or(lhs, rhs) =>
+ for {
+ lhsFilter <- createFilter(lhs)
+ rhsFilter <- createFilter(rhs)
+ } yield {
+ new OrExpression(lhsFilter, rhsFilter)
+ }
+
+ case _ => None
+ }
+ }
+
+ def getCarbonExpression(name: String) = {
+ new CarbonColumnExpression(name,
+ CarbonScalaUtil.convertSparkToCarbonDataType(dataTypeOf(name)))
+ }
+
+ def getCarbonLiteralExpression(name: String, value: Any): CarbonExpression = {
+ val dataTypeOfAttribute = CarbonScalaUtil.convertSparkToCarbonDataType(dataTypeOf(name))
+ val dataType = if (Option(value).isDefined
+ && dataTypeOfAttribute == DataType.STRING
+ && value.isInstanceOf[Double]) {
+ DataType.DOUBLE
+ } else {
+ dataTypeOfAttribute
+ }
+ new CarbonLiteralExpression(value, dataType)
+ }
+
+ createFilter(predicate)
+ }
+
+
+ // Check out which filters can be pushed down to carbon, remaining can be handled in spark layer.
+ // Mostly dimension filters are only pushed down since it is faster in carbon.
+ // TODO - The Filters are first converted Intermediate sources filters expression and then these
+ // expressions are again converted back to CarbonExpression. Instead of two step process of
+ // evaluating the filters it can be merged into a single one.
+ def selectFilters(filters: Seq[Expression],
+ attrList: java.util.HashSet[AttributeReferenceWrapper],
+ aliasMap: CarbonAliasDecoderRelation): Unit = {
+ def translate(expr: Expression, or: Boolean = false): Option[sources.Filter] = {
+ expr match {
+ case or@ Or(left, right) =>
+
+ val leftFilter = translate(left, or = true)
+ val rightFilter = translate(right, or = true)
+ if (leftFilter.isDefined && rightFilter.isDefined) {
+ Some( sources.Or(leftFilter.get, rightFilter.get))
+ } else {
+ or.collect {
+ case attr: AttributeReference =>
+ attrList.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
+ }
+ None
+ }
+
+ case And(left, right) =>
+ val leftFilter = translate(left, or)
+ val rightFilter = translate(right, or)
+ if (or) {
+ if (leftFilter.isDefined && rightFilter.isDefined) {
+ (leftFilter ++ rightFilter).reduceOption(sources.And)
+ } else {
+ None
+ }
+ } else {
+ (leftFilter ++ rightFilter).reduceOption(sources.And)
+ }
+
+ case EqualTo(a: Attribute, Literal(v, t)) =>
+ Some(sources.EqualTo(a.name, v))
+ case EqualTo(l@Literal(v, t), a: Attribute) =>
+ Some(sources.EqualTo(a.name, v))
+ case Not(EqualTo(a: Attribute, Literal(v, t))) =>
+ Some(sources.Not(sources.EqualTo(a.name, v)))
+ case Not(EqualTo(Literal(v, t), a: Attribute)) =>
+ Some(sources.Not(sources.EqualTo(a.name, v)))
+ case IsNotNull(a: Attribute) =>
+ Some(sources.IsNotNull(a.name))
+ case IsNull(a: Attribute) =>
+ Some(sources.IsNull(a.name))
+ case Not(In(a: Attribute, list)) if !list.exists(!_.isInstanceOf[Literal]) =>
+ val hSet = list.map(e => e.eval(EmptyRow))
+ Some(sources.Not(sources.In(a.name, hSet.toArray)))
+ case In(a: Attribute, list) if !list.exists(!_.isInstanceOf[Literal]) =>
+ val hSet = list.map(e => e.eval(EmptyRow))
+ Some(sources.In(a.name, hSet.toArray))
+ case GreaterThan(a: Attribute, Literal(v, t)) =>
+ Some(sources.GreaterThan(a.name, v))
+ case GreaterThan(Literal(v, t), a: Attribute) =>
+ Some(sources.LessThan(a.name, v))
+ case LessThan(a: Attribute, Literal(v, t)) =>
+ Some(sources.LessThan(a.name, v))
+ case LessThan(Literal(v, t), a: Attribute) =>
+ Some(sources.GreaterThan(a.name, v))
+ case GreaterThanOrEqual(a: Attribute, Literal(v, t)) =>
+ Some(sources.GreaterThanOrEqual(a.name, v))
+ case GreaterThanOrEqual(Literal(v, t), a: Attribute) =>
+ Some(sources.LessThanOrEqual(a.name, v))
+ case LessThanOrEqual(a: Attribute, Literal(v, t)) =>
+ Some(sources.LessThanOrEqual(a.name, v))
+ case LessThanOrEqual(Literal(v, t), a: Attribute) =>
+ Some(sources.GreaterThanOrEqual(a.name, v))
+
+ case others =>
+ if (!or) {
+ others.collect {
+ case attr: AttributeReference =>
+ attrList.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
+ }
+ }
+ None
+ }
+ }
+ filters.flatMap(translate(_, false)).toArray
+ }
+
+ def processExpression(exprs: Seq[Expression],
+ attributesNeedToDecode: java.util.HashSet[AttributeReference],
+ unprocessedExprs: ArrayBuffer[Expression],
+ carbonTable: CarbonTable): Option[CarbonExpression] = {
+ def transformExpression(expr: Expression, or: Boolean = false): Option[CarbonExpression] = {
+ expr match {
+ case orFilter@ Or(left, right) =>
+ val leftFilter = transformExpression(left, or = true)
+ val rightFilter = transformExpression(right, or = true)
+ if (leftFilter.isDefined && rightFilter.isDefined) {
+ Some(new OrExpression(leftFilter.get, rightFilter.get))
+ } else {
+ if (!or) {
+ orFilter.collect {
+ case attr: AttributeReference => attributesNeedToDecode.add(attr)
+ }
+ unprocessedExprs += orFilter
+ }
+ None
+ }
+
+ case And(left, right) =>
+ val leftFilter = transformExpression(left, or)
+ val rightFilter = transformExpression(right, or)
+ if (or) {
+ if (leftFilter.isDefined && rightFilter.isDefined) {
+ (leftFilter ++ rightFilter).reduceOption(new AndExpression(_, _))
+ } else {
+ None
+ }
+ } else {
+ (leftFilter ++ rightFilter).reduceOption(new AndExpression(_, _))
+ }
+
+
+ case EqualTo(a: Attribute, l@Literal(v, t)) =>
+ Some(
+ new EqualToExpression(
+ transformExpression(a).get,
+ transformExpression(l).get
+ )
+ )
+ case EqualTo(l@Literal(v, t), a: Attribute) =>
+ Some(
+ new EqualToExpression(
+ transformExpression(a).get,
+ transformExpression(l).get
+ )
+ )
+
+ case Not(EqualTo(a: Attribute, l@Literal(v, t))) =>
+ Some(
+ new NotEqualsExpression(
+ transformExpression(a).get,
+ transformExpression(l).get
+ )
+ )
+ case Not(EqualTo(l@Literal(v, t), a: Attribute)) =>
+ Some(
+ new NotEqualsExpression(
+ transformExpression(a).get,
+ transformExpression(l).get
+ )
+ )
+ case IsNotNull(child: Attribute) =>
+ Some(new NotEqualsExpression(transformExpression(child).get,
+ transformExpression(Literal(null)).get, true))
+ case IsNull(child: Attribute) =>
+ Some(new EqualToExpression(transformExpression(child).get,
+ transformExpression(Literal(null)).get, true))
+ case Not(In(a: Attribute, list))
+ if !list.exists(!_.isInstanceOf[Literal]) =>
+ if (list.exists(x => isNullLiteral(x.asInstanceOf[Literal]))) {
+ Some(new FalseExpression(transformExpression(a).get))
+ } else {
+ Some(new NotInExpression(transformExpression(a).get,
+ new ListExpression(convertToJavaList(list.map(transformExpression(_).get)))))
+ }
+ case In(a: Attribute, list) if !list.exists(!_.isInstanceOf[Literal]) =>
+ Some(new InExpression(transformExpression(a).get,
+ new ListExpression(convertToJavaList(list.map(transformExpression(_).get)))))
+
+ case GreaterThan(a: Attribute, l@Literal(v, t)) =>
+ Some(new GreaterThanExpression(transformExpression(a).get, transformExpression(l).get))
+ case GreaterThan(l@Literal(v, t), a: Attribute) =>
+ Some(new LessThanExpression(transformExpression(a).get, transformExpression(l).get))
+
+ case LessThan(a: Attribute, l@Literal(v, t)) =>
+ Some(new LessThanExpression(transformExpression(a).get, transformExpression(l).get))
+ case LessThan(l@Literal(v, t), a: Attribute) =>
+ Some(new GreaterThanExpression(transformExpression(a).get, transformExpression(l).get))
+
+ case GreaterThanOrEqual(a: Attribute, l@Literal(v, t)) =>
+ Some(new GreaterThanEqualToExpression(transformExpression(a).get,
+ transformExpression(l).get))
+ case GreaterThanOrEqual(l@Literal(v, t), a: Attribute) =>
+ Some(new LessThanEqualToExpression(transformExpression(a).get,
+ transformExpression(l).get))
+
+ case LessThanOrEqual(a: Attribute, l@Literal(v, t)) =>
+ Some(new LessThanEqualToExpression(transformExpression(a).get,
+ transformExpression(l).get))
+ case LessThanOrEqual(l@Literal(v, t), a: Attribute) =>
+ Some(new GreaterThanEqualToExpression(transformExpression(a).get,
+ transformExpression(l).get))
+
+ case AttributeReference(name, dataType, _, _) =>
+ Some(new CarbonColumnExpression(name,
+ CarbonScalaUtil.convertSparkToCarbonDataType(
+ getActualCarbonDataType(name, carbonTable))))
+ case Literal(name, dataType) => Some(new
+ CarbonLiteralExpression(name, CarbonScalaUtil.convertSparkToCarbonDataType(dataType)))
+ case others =>
+ if (!or) {
+ others.collect {
+ case attr: AttributeReference => attributesNeedToDecode.add(attr)
+ }
+ unprocessedExprs += others
+ }
+ None
+ }
+ }
+ exprs.flatMap(transformExpression(_, false)).reduceOption(new AndExpression(_, _))
+ }
+ private def isNullLiteral(exp: Expression): Boolean = {
+ if (null != exp
+ && exp.isInstanceOf[Literal]
+ && (exp.asInstanceOf[Literal].dataType == org.apache.spark.sql.types.DataTypes.NullType)
+ || (exp.asInstanceOf[Literal].value == null)) {
+ true
+ } else {
+ false
+ }
+ }
+ private def getActualCarbonDataType(column: String, carbonTable: CarbonTable) = {
+ var carbonColumn: CarbonColumn =
+ carbonTable.getDimensionByName(carbonTable.getFactTableName, column)
+ val dataType = if (carbonColumn != null) {
+ carbonColumn.getDataType
+ } else {
+ carbonColumn = carbonTable.getMeasureByName(carbonTable.getFactTableName, column)
+ carbonColumn.getDataType match {
+ case DataType.INT => DataType.LONG
+ case DataType.LONG => DataType.LONG
+ case DataType.DECIMAL => DataType.DECIMAL
+ case _ => DataType.DOUBLE
+ }
+ }
+ CarbonScalaUtil.convertCarbonToSparkDataType(dataType)
+ }
+
+ // Convert scala list to java list, Cannot use scalaList.asJava as while deserializing it is
+ // not able find the classes inside scala list and gives ClassNotFoundException.
+ private def convertToJavaList(
+ scalaList: Seq[CarbonExpression]): java.util.List[CarbonExpression] = {
+ val javaList = new java.util.ArrayList[CarbonExpression]()
+ scalaList.foreach(javaList.add)
+ javaList
+ }
+}
[17/50] [abbrv] incubator-carbondata git commit: removed unnecessary
.show call in test cases
Posted by ja...@apache.org.
removed unnecessary .show call in test cases
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/533835ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/533835ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/533835ca
Branch: refs/heads/12-dev
Commit: 533835ca04b07b3b7f340ffb7ff5c5571fad0b76
Parents: 598bbd2
Author: kunal642 <ku...@knoldus.in>
Authored: Thu Apr 20 19:28:48 2017 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Wed Apr 26 10:18:32 2017 -0700
----------------------------------------------------------------------
...ncrementalColumnDictionaryGeneratorTest.java | 2 -
.../ServerDictionaryGeneratorTest.java | 1 -
.../hadoop/test/util/StoreCreator.java | 1 -
.../HighCardinalityDataTypesTestCase.scala | 5 +-
.../detailquery/RangeFilterTestCase.scala | 9 -
...TypeDirectDictionaryWithNoDictTestCase.scala | 6 +-
.../carbondata/spark/util/CarbonScalaUtil.scala | 4 -
.../execution/command/carbonTableSchema.scala | 2 +-
.../complexType/TestComplexTypeQuery.scala | 28 +--
.../TestLoadDataWithSingleQuotechar.scala | 2 +-
.../createtable/TestCreateTableSyntax.scala | 6 +-
.../datacompaction/DataCompactionTest.scala | 2 +-
.../iud/DeleteCarbonTableTestCase.scala | 14 +-
.../testsuite/iud/IUDCompactionTestCases.scala | 146 +++++++---------
.../iud/UpdateCarbonTableTestCase.scala | 169 ++++++++-----------
.../AutoHighCardinalityIdentifyTestCase.scala | 4 +-
.../execution/command/carbonTableSchema.scala | 17 +-
.../carbondata/CarbonDataSourceSuite.scala | 2 +-
.../carbon/datastore/BlockIndexStoreTest.java | 2 -
.../carbondata/processing/StoreCreator.java | 12 --
20 files changed, 178 insertions(+), 256 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/533835ca/core/src/test/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGeneratorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGeneratorTest.java b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGeneratorTest.java
index 5179305..49db6ce 100644
--- a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGeneratorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGeneratorTest.java
@@ -150,8 +150,6 @@ public class IncrementalColumnDictionaryGeneratorTest {
// Create the generator and add the keys to dictionary
IncrementalColumnDictionaryGenerator generator =
new IncrementalColumnDictionaryGenerator(carbonDimension, 10);
- Integer generatedKey = generator.generateKey("First");
- Integer obtainedKey = generator.getOrGenerateKey("First");
// Create a table schema for saving the dictionary
TableSchema tableSchema = new TableSchema();
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/533835ca/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java
index 1fe31a5..b66331f 100644
--- a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java
@@ -178,7 +178,6 @@ public class ServerDictionaryGeneratorTest {
firstKey.setColumnName(empColumnSchema.getColumnName());
firstKey.setData("FirstKey");
serverDictionaryGenerator.initializeGeneratorForTable(firstKey);
- Integer value = serverDictionaryGenerator.generateKey(firstKey);
//Update generator with a new dimension
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/533835ca/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
index 2997e94..c90b34c 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
@@ -266,7 +266,6 @@ public class StoreCreator {
tableInfo.setLastUpdatedTime(System.currentTimeMillis());
tableInfo.setFactTable(tableSchema);
tableInfo.setAggregateTableList(new ArrayList<TableSchema>());
-
CarbonTablePath carbonTablePath = CarbonStorePath
.getCarbonTablePath(absoluteTableIdentifier.getStorePath(),
absoluteTableIdentifier.getCarbonTableIdentifier());
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/533835ca/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/HighCardinalityDataTypesTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/HighCardinalityDataTypesTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/HighCardinalityDataTypesTestCase.scala
index 5aa30b1..639eab1 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/HighCardinalityDataTypesTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/HighCardinalityDataTypesTestCase.scala
@@ -83,13 +83,12 @@ class NO_DICTIONARY_COL_TestCase extends QueryTest with BeforeAndAfterAll {
s"LOAD DATA LOCAL INPATH '$resourcesPath/data2.csv' INTO TABLE filtertestTable OPTIONS"+
s"('DELIMITER'= ',', " +
s"'FILEHEADER'= '')"
- );
+ )
}
test("Count (*) with filter") {
- // sql("select * from NO_DICTIONARY_CARBON_6 where empno != '16'").show()
- sql("select * from NO_DICTIONARY_CARBON_6 where empno > '11' and empno <= '30'").show()
+ sql("select * from NO_DICTIONARY_CARBON_6 where empno > '11' and empno <= '30'")
checkAnswer(
sql("select count(*) from NO_DICTIONARY_CARBON_6 where empno='11'"),
Seq(Row(1))
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/533835ca/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterTestCase.scala
index 67860e9..89b4db9 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterTestCase.scala
@@ -136,8 +136,6 @@ class RangeFilterTestCase extends QueryTest with BeforeAndAfterAll {
}
test("Range filter No Dictionary 1") {
- sql("select * from NO_DICTIONARY_CARBON_6").show()
- sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_6 where empno > '11' and empno < '15'").show()
checkAnswer(
sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_6 where empno > '11' and empno < '15'"),
sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_6 where empno > '11' and empno < '15'")
@@ -198,7 +196,6 @@ class RangeFilterTestCase extends QueryTest with BeforeAndAfterAll {
}
test("Range filter No Dictionary outside Boundary 5") {
- sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_6 where empno < '11' and empno > '20'").show()
checkAnswer(
sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_6 where empno < '11' and empno > '20'"),
sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_6 where empno < '11' and empno > '20'")
@@ -249,7 +246,6 @@ class RangeFilterTestCase extends QueryTest with BeforeAndAfterAll {
}
test("Range filter No Dictionary Inside Boundary 12") {
- sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_6 where empno >= '11' and empno > '12' and empno <= '20' and empno <= '15'").show()
checkAnswer(
sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_6 where empno >= '11' and empno > '12' and empno <= '20' and empno <= '15'"),
sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_6 where empno >= '11' and empno > '12' and empno <= '20' and empno <= '15'")
@@ -271,7 +267,6 @@ class RangeFilterTestCase extends QueryTest with BeforeAndAfterAll {
}
test("Range filter No Dictionary Inside Boundary 15") {
- sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_6 where empno >= '11' or empno > '12' and empno <= '20'").show()
checkAnswer(
sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_6 where empno >= '11' or empno > '12' and empno <= '20'"),
sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_6 where empno >= '11' or empno > '12' and empno <= '20'")
@@ -279,7 +274,6 @@ class RangeFilterTestCase extends QueryTest with BeforeAndAfterAll {
}
test("Range filter No Dictionary Inside Boundary 16") {
- sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_6 where empno >= '11' and empno > '12' or empno <= '20'").show()
checkAnswer(
sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_6 where empno >= '11' and empno > '12' or empno <= '20'"),
sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_6 where empno >= '11' and empno > '12' or empno <= '20'")
@@ -337,9 +331,6 @@ class RangeFilterTestCase extends QueryTest with BeforeAndAfterAll {
}
test("Range filter Dictionary 1") {
- sql("select * from DICTIONARY_CARBON_6").show()
- // sql("select * from NO_DICTIONARY_CARBON_6 where empno != '16'").show()
- sql("select * from DICTIONARY_CARBON_6 where empno > '11' and workgroupcategory = '1' or workgroupcategoryname = 'developer' and empno < '15'").show()
checkAnswer(
sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_6 where empno > '11' and empno < '15'"),
sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_6 where empno > '11' and empno < '15'")
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/533835ca/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryWithNoDictTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryWithNoDictTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryWithNoDictTestCase.scala
index cba7c99..c0b5389 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryWithNoDictTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryWithNoDictTestCase.scala
@@ -62,7 +62,7 @@ class DateDataTypeDirectDictionaryWithNoDictTestCase extends QueryTest with Befo
}
test("select doj from directDictionaryTable") {
- sql("select doj from directDictionaryTable").show()
+ sql("select doj from directDictionaryTable")
checkAnswer(
sql("select doj from directDictionaryTable"),
Seq(Row(Date.valueOf("2016-03-14")),
@@ -74,7 +74,7 @@ class DateDataTypeDirectDictionaryWithNoDictTestCase extends QueryTest with Befo
test("select doj from directDictionaryTable with equals filter") {
- sql("select doj from directDictionaryTable where doj='2016-03-14 15:00:09'").show()
+ sql("select doj from directDictionaryTable where doj='2016-03-14 15:00:09'")
checkAnswer(
sql("select doj from directDictionaryTable where doj='2016-03-14'"),
Seq(Row(Date.valueOf("2016-03-14")))
@@ -83,7 +83,7 @@ class DateDataTypeDirectDictionaryWithNoDictTestCase extends QueryTest with Befo
}
test("select doj from directDictionaryTable with greater than filter") {
- sql("select doj from directDictionaryTable where doj>'2016-03-14 15:00:09'").show()
+ sql("select doj from directDictionaryTable where doj>'2016-03-14 15:00:09'")
checkAnswer(
sql("select doj from directDictionaryTable where doj>'2016-03-14 15:00:09'"),
Seq(Row(Date.valueOf("2016-04-14")))
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/533835ca/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index c88f7a4..50ebdcb 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -17,19 +17,15 @@
package org.apache.carbondata.spark.util
-import java.io.File
import java.text.SimpleDateFormat
import org.apache.spark.sql._
import org.apache.spark.sql.execution.command.DataTypeInfo
import org.apache.spark.sql.types._
-import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.datatype.{DataType => CarbonDataType}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn
-import org.apache.carbondata.core.util.CarbonProperties
object CarbonScalaUtil {
def convertSparkToCarbonDataType(
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/533835ca/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index a824bde..ba2576a 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -145,7 +145,7 @@ case class CreateTable(cm: TableModel) extends RunnableCommand {
val tableInfo: TableInfo = TableNewProcessor(cm)
- if (tableInfo.getFactTable.getListOfColumns.size <= 0) {
+ if (tableInfo.getFactTable.getListOfColumns.isEmpty) {
sys.error("No Dimensions found. Table should have at least one dimesnion !")
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/533835ca/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
index 05e0483..c2c15eb 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
@@ -28,13 +28,13 @@ import org.scalatest.BeforeAndAfterAll
class TestComplexTypeQuery extends QueryTest with BeforeAndAfterAll {
override def beforeAll: Unit = {
- sql("drop table if exists complexcarbontable").show
- sql("drop table if exists complexhivetable").show
- sql("drop table if exists complex_filter").show
- sql("drop table if exists structusingstructCarbon").show
- sql("drop table if exists structusingstructHive").show
- sql("drop table if exists structusingarraycarbon").show
- sql("drop table if exists structusingarrayhive").show
+ sql("drop table if exists complexcarbontable")
+ sql("drop table if exists complexhivetable")
+ sql("drop table if exists complex_filter")
+ sql("drop table if exists structusingstructCarbon")
+ sql("drop table if exists structusingstructHive")
+ sql("drop table if exists structusingarraycarbon")
+ sql("drop table if exists structusingarrayhive")
sql(
"create table complexcarbontable(deviceInformationId int, channelsId string, ROMSize " +
"string, ROMName String, purchasedate string, mobile struct<imei:string, imsi:string>, MAC " +
@@ -67,7 +67,7 @@ class TestComplexTypeQuery extends QueryTest with BeforeAndAfterAll {
"/array1.csv' INTO TABLE complex_filter options ('DELIMITER'=',', 'QUOTECHAR'='\"', " +
"'COMPLEX_DELIMITER_LEVEL_1'='$', 'FILEHEADER'= 'test1,test2,test3,test4,test5,test6," +
"test7')")
- .show()
+ ()
sql(
"create table structusingarraycarbon (MAC struct<MAC1:array<string>," +
@@ -278,12 +278,12 @@ class TestComplexTypeQuery extends QueryTest with BeforeAndAfterAll {
}
override def afterAll {
- sql("drop table if exists complexcarbontable").show
- sql("drop table if exists complexhivetable").show
- sql("drop table if exists structusingstructCarbon").show
- sql("drop table if exists structusingstructHive").show
- sql("drop table if exists structusingarraycarbon").show
- sql("drop table if exists structusingarrayhive").show
+ sql("drop table if exists complexcarbontable")
+ sql("drop table if exists complexhivetable")
+ sql("drop table if exists structusingstructCarbon")
+ sql("drop table if exists structusingstructHive")
+ sql("drop table if exists structusingarraycarbon")
+ sql("drop table if exists structusingarrayhive")
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/533835ca/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala
index cc6e697..8aade63 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala
@@ -36,7 +36,7 @@ class TestLoadDataWithSingleQuotechar extends QueryTest with BeforeAndAfterAll {
sql(
s"LOAD DATA LOCAL INPATH '$resourcesPath/dataWithSingleQuote.csv' INTO TABLE " +
"carbontable OPTIONS('DELIMITER'= ',')")
- sql("SELECT * from carbontable").show(100, false)
+ sql("SELECT * from carbontable")
checkAnswer(
sql("SELECT * from carbontable"),
Seq(Row(1,"Tom"),
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/533835ca/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala
index fc1aa66..d0e9639 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala
@@ -37,7 +37,7 @@ class TestCreateTableSyntax extends QueryTest with BeforeAndAfterAll {
sql("create table carbontable(id int, username struct<sur_name:string," +
"actual_name:struct<first_name:string,last_name:string>>, country string, salary double)" +
"STORED BY 'org.apache.carbondata.format'")
- sql("describe carbontable").show
+ sql("describe carbontable")
}
test("Test table rename operation on carbon table and on hive table") {
@@ -128,10 +128,10 @@ class TestCreateTableSyntax extends QueryTest with BeforeAndAfterAll {
sql("create table carbontable(id int, username struct<sur_name:string," +
"actual_name:struct<first_name:string,last_name:string>>, country string, salary double)" +
"STORED BY 'org.apache.carbondata.format'")
- sql("describe formatted carbontable").show(50)
+ sql("describe formatted carbontable")
sql("create table hivetable(id int, username struct<sur_name:string," +
"actual_name:struct<first_name:string,last_name:string>>, country string, salary double)")
- sql("describe formatted hivetable").show(50)
+ sql("describe formatted hivetable")
}
test("describe command carbon table for decimal scale and precision test") {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/533835ca/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala
index 76e327c..e9118ba 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala
@@ -190,7 +190,7 @@ class DataCompactionTest extends QueryTest with BeforeAndAfterAll {
sql("update cardinalityUpdateTest set (FirstName) = ('FirstTwentyOne') where ID = 2").show()
// alter table.
- sql("alter table cardinalityUpdateTest compact 'major'").show()
+ sql("alter table cardinalityUpdateTest compact 'major'")
// Verify the new updated value in compacted segment.
// now check the answers it should be same.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/533835ca/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
index 40590cb..93da343 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
@@ -32,7 +32,7 @@ class DeleteCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
sql("use iud_db")
}
test("delete data from carbon table with alias [where clause ]") {
- sql("""create table iud_db.dest (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""create table iud_db.dest (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud_db.dest""")
sql("""delete from iud_db.dest d where d.c1 = 'a'""").show
checkAnswer(
@@ -42,7 +42,7 @@ class DeleteCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
}
test("delete data from carbon table[where clause ]") {
sql("""drop table if exists iud_db.dest""")
- sql("""create table iud_db.dest (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""create table iud_db.dest (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud_db.dest""")
sql("""delete from dest where c2 = 2""").show
checkAnswer(
@@ -52,7 +52,7 @@ class DeleteCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
}
test("delete data from carbon table[where IN ]") {
sql("""drop table if exists iud_db.dest""")
- sql("""create table iud_db.dest (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""create table iud_db.dest (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud_db.dest""")
sql("""delete from dest where c1 IN ('d', 'e')""").show
checkAnswer(
@@ -63,7 +63,7 @@ class DeleteCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
test("delete data from carbon table[with alias No where clause]") {
sql("""drop table if exists iud_db.dest""")
- sql("""create table iud_db.dest (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""create table iud_db.dest (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud_db.dest""")
sql("""delete from iud_db.dest a""").show
checkAnswer(
@@ -73,7 +73,7 @@ class DeleteCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
}
test("delete data from carbon table[No alias No where clause]") {
sql("""drop table if exists iud_db.dest""")
- sql("""create table iud_db.dest (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""create table iud_db.dest (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud_db.dest""")
sql("""delete from dest""").show()
checkAnswer(
@@ -84,7 +84,7 @@ class DeleteCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
test("delete data from carbon table[ JOIN with another table ]") {
sql("""drop table if exists iud_db.dest""")
- sql("""create table iud_db.dest (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""create table iud_db.dest (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud_db.dest""")
sql(""" DELETE FROM dest t1 INNER JOIN source2 t2 ON t1.c1 = t2.c11""").show(truncate = false)
checkAnswer(
@@ -115,7 +115,7 @@ class DeleteCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
// }
test("delete data from carbon table[where numeric condition ]") {
sql("""drop table if exists iud_db.dest""")
- sql("""create table iud_db.dest (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""create table iud_db.dest (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud_db.dest""")
sql("""delete from iud_db.dest where c2 >= 4""").show()
checkAnswer(
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/533835ca/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/iud/IUDCompactionTestCases.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/iud/IUDCompactionTestCases.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/iud/IUDCompactionTestCases.scala
index fd8c6cf..9da3913 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/iud/IUDCompactionTestCases.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/iud/IUDCompactionTestCases.scala
@@ -33,8 +33,6 @@ class HorizontalCompactionTestCase extends QueryTest with BeforeAndAfterAll {
sql("""use iud4""")
sql(
"""create table iud4.dest (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
-
- .show()
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/comp1.csv' INTO table iud4.dest""")
sql(
"""create table iud4.source2 (c11 string,c22 int,c33 string,c55 string, c66 int) STORED BY 'org.apache.carbondata.format'""")
@@ -43,7 +41,6 @@ class HorizontalCompactionTestCase extends QueryTest with BeforeAndAfterAll {
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/other.csv' INTO table iud4.other""")
sql(
"""create table iud4.hdest (c1 string,c2 int,c3 string,c5 string) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' LINES TERMINATED BY '\n' STORED AS TEXTFILE""")
- .show()
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/comp1.csv' INTO table iud4.hdest""")
sql(
"""CREATE TABLE iud4.update_01(imei string,age int,task bigint,num double,level decimal(10,3),name string)STORED BY 'org.apache.carbondata.format' """)
@@ -56,20 +53,19 @@ class HorizontalCompactionTestCase extends QueryTest with BeforeAndAfterAll {
test("test IUD Horizontal Compaction Update Alter Clean") {
- sql("""drop database if exists iud4 cascade""").show()
+ sql("""drop database if exists iud4 cascade""")
sql("""create database iud4""")
- sql("""use iud4""").show()
+ sql("""use iud4""")
sql(
"""create table dest2 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
- .show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp1.csv' INTO table dest2""").show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp2.csv' INTO table dest2""").show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp3.csv' INTO table dest2""").show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp4.csv' INTO table dest2""").show()
+
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp1.csv' INTO table dest2""")
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp2.csv' INTO table dest2""")
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp3.csv' INTO table dest2""")
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp4.csv' INTO table dest2""")
sql(
"""create table source2 (c11 string,c22 int,c33 string,c55 string, c66 int) STORED BY 'org.apache.carbondata.format'""")
- .show()
- sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/source3.csv' INTO table source2""").show()
+ sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/source3.csv' INTO table source2""")
sql(
"""update dest2 d set (d.c3, d.c5 ) = (select s.c33,s.c55 from source2 s where d.c1 = s.c11 and s.c22 < 3 or (s.c22 > 10 and s.c22 < 13) or (s.c22 > 20 and s.c22 < 23) or (s.c22 > 30 and s.c22 < 33))""")
.show()
@@ -79,9 +75,8 @@ class HorizontalCompactionTestCase extends QueryTest with BeforeAndAfterAll {
sql(
"""update dest2 d set (d.c3, d.c5 ) = (select s.c33,s.c55 from source2 s where d.c1 = s.c11 and (s.c22 > 5 and c22 < 8) or (s.c22 > 15 and s.c22 < 18 ) or (s.c22 > 25 and c22 < 28) or (s.c22 > 35 and c22 < 38))""")
.show()
- sql("""alter table dest2 compact 'minor'""").show()
- sql("""clean files for table dest2""").show()
- sql("""select c1,c2,c3,c5 from dest2 order by c2""").show(100)
+ sql("""alter table dest2 compact 'minor'""")
+ sql("""clean files for table dest2""")
checkAnswer(
sql("""select c1,c2,c3,c5 from dest2 order by c2"""),
Seq(Row("a", 1, "MGM", "Disco"),
@@ -125,33 +120,31 @@ class HorizontalCompactionTestCase extends QueryTest with BeforeAndAfterAll {
Row("i", 39, "ii", "iii"),
Row("j", 40, "jj", "jjj"))
)
- sql("""drop table dest2""").show()
+ sql("""drop table dest2""")
}
test("test IUD Horizontal Compaction Delete") {
- sql("""drop database if exists iud4 cascade""").show()
+ sql("""drop database if exists iud4 cascade""")
sql("""create database iud4""")
- sql("""use iud4""").show()
+ sql("""use iud4""")
sql(
"""create table dest2 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
- .show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp1.csv' INTO table dest2""").show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp2.csv' INTO table dest2""").show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp3.csv' INTO table dest2""").show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp4.csv' INTO table dest2""").show()
- sql("""select * from dest2""").show()
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp1.csv' INTO table dest2""")
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp2.csv' INTO table dest2""")
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp3.csv' INTO table dest2""")
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp4.csv' INTO table dest2""")
+ sql("""select * from dest2""")
sql(
"""create table source2 (c11 string,c22 int,c33 string,c55 string, c66 int) STORED BY 'org.apache.carbondata.format'""")
- .show()
- sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/source3.csv' INTO table source2""").show()
- sql("""select * from source2""").show()
+ sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/source3.csv' INTO table source2""")
+ sql("""select * from source2""")
sql("""delete from dest2 where (c2 < 3) or (c2 > 10 and c2 < 13) or (c2 > 20 and c2 < 23) or (c2 > 30 and c2 < 33)""").show()
- sql("""select * from dest2 order by 2""").show()
+ sql("""select * from dest2 order by 2""")
sql("""delete from dest2 where (c2 > 3 and c2 < 5) or (c2 > 13 and c2 < 15) or (c2 > 23 and c2 < 25) or (c2 > 33 and c2 < 35)""").show()
- sql("""select * from dest2 order by 2""").show()
+ sql("""select * from dest2 order by 2""")
sql("""delete from dest2 where (c2 > 5 and c2 < 8) or (c2 > 15 and c2 < 18 ) or (c2 > 25 and c2 < 28) or (c2 > 35 and c2 < 38)""").show()
- sql("""clean files for table dest2""").show()
+ sql("""clean files for table dest2""")
checkAnswer(
sql("""select c1,c2,c3,c5 from dest2 order by c2"""),
Seq(Row("c", 3, "cc", "ccc"),
@@ -175,32 +168,30 @@ class HorizontalCompactionTestCase extends QueryTest with BeforeAndAfterAll {
Row("i", 39, "ii", "iii"),
Row("j", 40, "jj", "jjj"))
)
- sql("""drop table dest2""").show()
+ sql("""drop table dest2""")
}
test("test IUD Horizontal Compaction Multiple Update Vertical Compaction and Clean") {
- sql("""drop database if exists iud4 cascade""").show()
+ sql("""drop database if exists iud4 cascade""")
sql("""create database iud4""")
- sql("""use iud4""").show()
+ sql("""use iud4""")
sql(
"""create table dest2 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
- .show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp1.csv' INTO table dest2""").show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp2.csv' INTO table dest2""").show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp3.csv' INTO table dest2""").show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp4.csv' INTO table dest2""").show()
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp1.csv' INTO table dest2""")
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp2.csv' INTO table dest2""")
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp3.csv' INTO table dest2""")
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp4.csv' INTO table dest2""")
sql(
"""create table source2 (c11 string,c22 int,c33 string,c55 string, c66 int) STORED BY 'org.apache.carbondata.format'""")
- .show()
- sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/source3.csv' INTO table source2""").show()
+ sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/source3.csv' INTO table source2""")
sql("""update dest2 d set (d.c3, d.c5 ) = (select s.c33,s.c55 from source2 s where d.c1 = s.c11 and s.c22 < 3 or (s.c22 > 10 and s.c22 < 13) or (s.c22 > 20 and s.c22 < 23) or (s.c22 > 30 and s.c22 < 33))""").show()
sql("""update dest2 d set (d.c3, d.c5 ) = (select s.c11,s.c66 from source2 s where d.c1 = s.c11 and s.c22 < 3 or (s.c22 > 10 and s.c22 < 13) or (s.c22 > 20 and s.c22 < 23) or (s.c22 > 30 and s.c22 < 33))""").show()
sql("""update dest2 d set (d.c3, d.c5 ) = (select s.c33,s.c55 from source2 s where d.c1 = s.c11 and (s.c22 > 3 and s.c22 < 5) or (s.c22 > 13 and s.c22 < 15) or (s.c22 > 23 and s.c22 < 25) or (s.c22 > 33 and s.c22 < 35))""").show()
sql("""update dest2 d set (d.c3, d.c5 ) = (select s.c11,s.c66 from source2 s where d.c1 = s.c11 and (s.c22 > 3 and s.c22 < 5) or (s.c22 > 13 and s.c22 < 15) or (s.c22 > 23 and s.c22 < 25) or (s.c22 > 33 and s.c22 < 35))""").show()
sql("""update dest2 d set (d.c3, d.c5 ) = (select s.c33,s.c55 from source2 s where d.c1 = s.c11 and (s.c22 > 5 and c22 < 8) or (s.c22 > 15 and s.c22 < 18 ) or (s.c22 > 25 and c22 < 28) or (s.c22 > 35 and c22 < 38))""").show()
sql("""update dest2 d set (d.c3, d.c5 ) = (select s.c11,s.c66 from source2 s where d.c1 = s.c11 and (s.c22 > 5 and c22 < 8) or (s.c22 > 15 and s.c22 < 18 ) or (s.c22 > 25 and c22 < 28) or (s.c22 > 35 and c22 < 38))""").show()
- sql("""alter table dest2 compact 'major'""").show()
- sql("""clean files for table dest2""").show()
+ sql("""alter table dest2 compact 'major'""")
+ sql("""clean files for table dest2""")
checkAnswer(
sql("""select c1,c2,c3,c5 from dest2 order by c2"""),
Seq(Row("a", 1, "a", "10"),
@@ -244,29 +235,27 @@ class HorizontalCompactionTestCase extends QueryTest with BeforeAndAfterAll {
Row("i", 39, "ii", "iii"),
Row("j", 40, "jj", "jjj"))
)
- sql("""drop table dest2""").show()
+ sql("""drop table dest2""")
}
test("test IUD Horizontal Compaction Update Delete and Clean") {
- sql("""drop database if exists iud4 cascade""").show()
+ sql("""drop database if exists iud4 cascade""")
sql("""create database iud4""")
- sql("""use iud4""").show()
+ sql("""use iud4""")
sql(
"""create table dest2 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
- .show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp1.csv' INTO table dest2""").show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp2.csv' INTO table dest2""").show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp3.csv' INTO table dest2""").show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp4.csv' INTO table dest2""").show()
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp1.csv' INTO table dest2""")
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp2.csv' INTO table dest2""")
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp3.csv' INTO table dest2""")
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp4.csv' INTO table dest2""")
sql(
"""create table source2 (c11 string,c22 int,c33 string,c55 string, c66 int) STORED BY 'org.apache.carbondata.format'""")
- .show()
- sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/source3.csv' INTO table source2""").show()
+ sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/source3.csv' INTO table source2""")
sql("""update dest2 d set (d.c3, d.c5 ) = (select s.c33,s.c55 from source2 s where d.c1 = s.c11 and s.c22 < 3 or (s.c22 > 10 and s.c22 < 13) or (s.c22 > 20 and s.c22 < 23) or (s.c22 > 30 and s.c22 < 33))""").show()
sql("""delete from dest2 where (c2 < 2) or (c2 > 10 and c2 < 13) or (c2 > 20 and c2 < 23) or (c2 > 30 and c2 < 33)""").show()
sql("""delete from dest2 where (c2 > 3 and c2 < 5) or (c2 > 13 and c2 < 15) or (c2 > 23 and c2 < 25) or (c2 > 33 and c2 < 35)""").show()
sql("""delete from dest2 where (c2 > 5 and c2 < 8) or (c2 > 15 and c2 < 18 ) or (c2 > 25 and c2 < 28) or (c2 > 35 and c2 < 38)""").show()
- sql("""clean files for table dest2""").show()
+ sql("""clean files for table dest2""")
checkAnswer(
sql("""select c1,c2,c3,c5 from dest2 order by c2"""),
Seq(Row("b", 2, "RGK", "Music"),
@@ -291,17 +280,16 @@ class HorizontalCompactionTestCase extends QueryTest with BeforeAndAfterAll {
Row("i", 39, "ii", "iii"),
Row("j", 40, "jj", "jjj"))
)
- sql("""drop table dest2""").show()
+ sql("""drop table dest2""")
}
test("test IUD Horizontal Compaction Check Column Cardinality") {
- sql("""drop database if exists iud4 cascade""").show()
+ sql("""drop database if exists iud4 cascade""")
sql("""create database iud4""")
- sql("""use iud4""").show()
+ sql("""use iud4""")
sql(
"""create table T_Carbn01(Active_status String,Item_type_cd INT,Qty_day_avg INT,Qty_total INT,Sell_price BIGINT,Sell_pricep DOUBLE,Discount_price DOUBLE,Profit DECIMAL(3,2),Item_code String,Item_name String,Outlet_name String,Update_time TIMESTAMP,Create_date String)STORED BY 'org.apache.carbondata.format'""")
- .show()
- sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/T_Hive1.csv' INTO table t_carbn01 options ('BAD_RECORDS_LOGGER_ENABLE' = 'FALSE', 'BAD_RECORDS_ACTION' = 'FORCE','DELIMITER'=',', 'QUOTECHAR'='\', 'FILEHEADER'='Active_status,Item_type_cd,Qty_day_avg,Qty_total,Sell_price,Sell_pricep,Discount_price,Profit,Item_code,Item_name,Outlet_name,Update_time,Create_date')""").show()
+ sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/T_Hive1.csv' INTO table t_carbn01 options ('BAD_RECORDS_LOGGER_ENABLE' = 'FALSE', 'BAD_RECORDS_ACTION' = 'FORCE','DELIMITER'=',', 'QUOTECHAR'='\', 'FILEHEADER'='Active_status,Item_type_cd,Qty_day_avg,Qty_total,Sell_price,Sell_pricep,Discount_price,Profit,Item_code,Item_name,Outlet_name,Update_time,Create_date')""")
sql("""update t_carbn01 set (item_code) = ('Orange') where item_type_cd = 14""").show()
sql("""update t_carbn01 set (item_code) = ('Banana') where item_type_cd = 2""").show()
sql("""delete from t_carbn01 where item_code in ('RE3423ee','Orange','Banana')""").show()
@@ -314,27 +302,24 @@ class HorizontalCompactionTestCase extends QueryTest with BeforeAndAfterAll {
Row("SE3423ee"),
Row("SE3423ee"))
)
- sql("""drop table t_carbn01""").show()
+ sql("""drop table t_carbn01""")
}
test("test IUD Horizontal Compaction Segment Delete Test Case") {
- sql("""drop database if exists iud4 cascade""").show()
+ sql("""drop database if exists iud4 cascade""")
sql("""create database iud4""")
- sql("""use iud4""").show()
+ sql("""use iud4""")
sql(
"""create table dest2 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
- .show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp1.csv' INTO table dest2""").show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp2.csv' INTO table dest2""").show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp3.csv' INTO table dest2""").show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp4.csv' INTO table dest2""").show()
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp1.csv' INTO table dest2""")
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp2.csv' INTO table dest2""")
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp3.csv' INTO table dest2""")
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp4.csv' INTO table dest2""")
sql(
- """delete from dest2 where (c2 < 3) or (c2 > 10 and c2 < 13) or (c2 > 20 and c2 < 23) or (c2 > 30 and c2 < 33)""")
-
- .show()
- sql("""DELETE SEGMENT 0 FROM TABLE dest2""").show()
- sql("""clean files for table dest2""").show()
+ """delete from dest2 where (c2 < 3) or (c2 > 10 and c2 < 13) or (c2 > 20 and c2 < 23) or (c2 > 30 and c2 < 33)""").show()
+ sql("""DELETE SEGMENT 0 FROM TABLE dest2""")
+ sql("""clean files for table dest2""")
sql(
"""update dest2 set (c5) = ('8RAM size') where (c2 > 3 and c2 < 5) or (c2 > 13 and c2 < 15) or (c2 > 23 and c2 < 25) or (c2 > 33 and c2 < 35)""")
.show()
@@ -342,27 +327,26 @@ class HorizontalCompactionTestCase extends QueryTest with BeforeAndAfterAll {
sql("""select count(*) from dest2"""),
Seq(Row(24))
)
- sql("""drop table dest2""").show()
+ sql("""drop table dest2""")
}
test("test case full table delete") {
- sql("""drop database if exists iud4 cascade""").show()
+ sql("""drop database if exists iud4 cascade""")
sql("""create database iud4""")
- sql("""use iud4""").show()
+ sql("""use iud4""")
sql(
"""create table dest2 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
- .show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp1.csv' INTO table dest2""").show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp2.csv' INTO table dest2""").show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp3.csv' INTO table dest2""").show()
- sql(s"""load data local inpath '$resourcesPath/IUD/comp4.csv' INTO table dest2""").show()
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp1.csv' INTO table dest2""")
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp2.csv' INTO table dest2""")
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp3.csv' INTO table dest2""")
+ sql(s"""load data local inpath '$resourcesPath/IUD/comp4.csv' INTO table dest2""")
sql("""delete from dest2 where c2 < 41""").show()
- sql("""alter table dest2 compact 'major'""").show()
+ sql("""alter table dest2 compact 'major'""")
checkAnswer(
sql("""select count(*) from dest2"""),
Seq(Row(0))
)
- sql("""drop table dest2""").show()
+ sql("""drop table dest2""")
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/533835ca/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/iud/UpdateCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/iud/UpdateCarbonTableTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/iud/UpdateCarbonTableTestCase.scala
index 4145a8c..8e9efe5 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/iud/UpdateCarbonTableTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/iud/UpdateCarbonTableTestCase.scala
@@ -29,7 +29,7 @@ class UpdateCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
sql("drop database if exists iud cascade")
sql("create database iud")
sql("use iud")
- sql("""create table iud.dest (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""create table iud.dest (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest""")
sql("""create table iud.source2 (c11 string,c22 int,c33 string,c55 string, c66 int) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/source2.csv' INTO table iud.source2""")
@@ -46,7 +46,7 @@ class UpdateCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
test("test update operation with 0 rows updation.") {
sql("""drop table iud.zerorows""").show
- sql("""create table iud.zerorows (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""create table iud.zerorows (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.zerorows""")
sql("""update zerorows d set (d.c2) = (d.c2 + 1) where d.c1 = 'a'""").show()
sql("""update zerorows d set (d.c2) = (d.c2 + 1) where d.c1 = 'xxx'""").show()
@@ -62,7 +62,7 @@ class UpdateCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
test("update carbon table[select from source table with where and exist]") {
sql("""drop table iud.dest11""").show
- sql("""create table iud.dest11 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""create table iud.dest11 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest11""")
sql("""update iud.dest11 d set (d.c3, d.c5 ) = (select s.c33,s.c55 from iud.source2 s where d.c1 = s.c11) where 1 = 1""").show()
checkAnswer(
@@ -73,8 +73,8 @@ class UpdateCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
}
test("update carbon table[using destination table columns with where and exist]") {
- sql("""drop table iud.dest22""").show
- sql("""create table iud.dest22 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""drop table iud.dest22""")
+ sql("""create table iud.dest22 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest22""")
checkAnswer(
sql("""select c2 from iud.dest22 where c1='a'"""),
@@ -85,24 +85,24 @@ class UpdateCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
sql("""select c2 from iud.dest22 where c1='a'"""),
Seq(Row(2))
)
- sql("""drop table iud.dest22""").show
+ sql("""drop table iud.dest22""")
}
test("update carbon table without alias in set columns") {
- sql("""drop table iud.dest33""").show
- sql("""create table iud.dest33 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""drop table iud.dest33""")
+ sql("""create table iud.dest33 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest33""")
sql("""update iud.dest33 d set (c3,c5 ) = (select s.c33 ,s.c55 from iud.source2 s where d.c1 = s.c11) where d.c1 = 'a'""").show()
checkAnswer(
sql("""select c3,c5 from iud.dest33 where c1='a'"""),
Seq(Row("MGM","Disco"))
)
- sql("""drop table iud.dest33""").show
+ sql("""drop table iud.dest33""")
}
test("update carbon table without alias in set columns with mulitple loads") {
- sql("""drop table iud.dest33""").show
- sql("""create table iud.dest33 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""drop table iud.dest33""")
+ sql("""create table iud.dest33 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest33""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest33""")
sql("""update iud.dest33 d set (c3,c5 ) = (select s.c33 ,s.c55 from iud.source2 s where d.c1 = s.c11) where d.c1 = 'a'""").show()
@@ -110,151 +110,151 @@ class UpdateCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
sql("""select c3,c5 from iud.dest33 where c1='a'"""),
Seq(Row("MGM","Disco"),Row("MGM","Disco"))
)
- sql("""drop table iud.dest33""").show
+ sql("""drop table iud.dest33""")
}
test("update carbon table without alias in set three columns") {
- sql("""drop table iud.dest44""").show
- sql("""create table iud.dest44 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""drop table iud.dest44""")
+ sql("""create table iud.dest44 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest44""")
sql("""update iud.dest44 d set (c1,c3,c5 ) = (select s.c11, s.c33 ,s.c55 from iud.source2 s where d.c1 = s.c11) where d.c1 = 'a'""").show()
checkAnswer(
sql("""select c1,c3,c5 from iud.dest44 where c1='a'"""),
Seq(Row("a","MGM","Disco"))
)
- sql("""drop table iud.dest44""").show
+ sql("""drop table iud.dest44""")
}
test("update carbon table[single column select from source with where and exist]") {
- sql("""drop table iud.dest55""").show
- sql("""create table iud.dest55 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""drop table iud.dest55""")
+ sql("""create table iud.dest55 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest55""")
sql("""update iud.dest55 d set (c3) = (select s.c33 from iud.source2 s where d.c1 = s.c11) where 1 = 1""").show()
checkAnswer(
sql("""select c1,c3 from iud.dest55 """),
Seq(Row("a","MGM"),Row("b","RGK"),Row("c","cc"),Row("d","dd"),Row("e","ee"))
)
- sql("""drop table iud.dest55""").show
+ sql("""drop table iud.dest55""")
}
test("update carbon table[single column SELECT from source with where and exist]") {
- sql("""drop table iud.dest55""").show
- sql("""create table iud.dest55 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""drop table iud.dest55""")
+ sql("""create table iud.dest55 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest55""")
sql("""update iud.dest55 d set (c3) = (SELECT s.c33 from iud.source2 s where d.c1 = s.c11) where 1 = 1""").show()
checkAnswer(
sql("""select c1,c3 from iud.dest55 """),
Seq(Row("a","MGM"),Row("b","RGK"),Row("c","cc"),Row("d","dd"),Row("e","ee"))
)
- sql("""drop table iud.dest55""").show
+ sql("""drop table iud.dest55""")
}
test("update carbon table[using destination table columns without where clause]") {
- sql("""drop table iud.dest66""").show
- sql("""create table iud.dest66 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""drop table iud.dest66""")
+ sql("""create table iud.dest66 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest66""")
sql("""update iud.dest66 d set (c2, c5 ) = (c2 + 1, concat(c5 , "z"))""").show()
checkAnswer(
sql("""select c2,c5 from iud.dest66 """),
Seq(Row(2,"aaaz"),Row(3,"bbbz"),Row(4,"cccz"),Row(5,"dddz"),Row(6,"eeez"))
)
- sql("""drop table iud.dest66""").show
+ sql("""drop table iud.dest66""")
}
test("update carbon table[using destination table columns with where clause]") {
- sql("""drop table iud.dest77""").show
- sql("""create table iud.dest77 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""drop table iud.dest77""")
+ sql("""create table iud.dest77 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest77""")
sql("""update iud.dest77 d set (c2, c5 ) = (c2 + 1, concat(c5 , "z")) where d.c3 = 'dd'""").show()
checkAnswer(
sql("""select c2,c5 from iud.dest77 where c3 = 'dd'"""),
Seq(Row(5,"dddz"))
)
- sql("""drop table iud.dest77""").show
+ sql("""drop table iud.dest77""")
}
test("update carbon table[using destination table( no alias) columns without where clause]") {
- sql("""drop table iud.dest88""").show
- sql("""create table iud.dest88 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""drop table iud.dest88""")
+ sql("""create table iud.dest88 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest88""")
sql("""update iud.dest88 set (c2, c5 ) = (c2 + 1, concat(c5 , "y" ))""").show()
checkAnswer(
sql("""select c2,c5 from iud.dest88 """),
Seq(Row(2,"aaay"),Row(3,"bbby"),Row(4,"cccy"),Row(5,"dddy"),Row(6,"eeey"))
)
- sql("""drop table iud.dest88""").show
+ sql("""drop table iud.dest88""")
}
test("update carbon table[using destination table columns with hard coded value ]") {
- sql("""drop table iud.dest99""").show
- sql("""create table iud.dest99 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""drop table iud.dest99""")
+ sql("""create table iud.dest99 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest99""")
sql("""update iud.dest99 d set (c2, c5 ) = (c2 + 1, "xyx")""").show()
checkAnswer(
sql("""select c2,c5 from iud.dest99 """),
Seq(Row(2,"xyx"),Row(3,"xyx"),Row(4,"xyx"),Row(5,"xyx"),Row(6,"xyx"))
)
- sql("""drop table iud.dest99""").show
+ sql("""drop table iud.dest99""")
}
test("update carbon tableusing destination table columns with hard coded value and where condition]") {
- sql("""drop table iud.dest110""").show
- sql("""create table iud.dest110 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""drop table iud.dest110""")
+ sql("""create table iud.dest110 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest110""")
sql("""update iud.dest110 d set (c2, c5 ) = (c2 + 1, "xyx") where d.c1 = 'e'""").show()
checkAnswer(
sql("""select c2,c5 from iud.dest110 where c1 = 'e' """),
Seq(Row(6,"xyx"))
)
- sql("""drop table iud.dest110""").show
+ sql("""drop table iud.dest110""")
}
test("update carbon table[using source table columns with where and exist and no destination table condition]") {
- sql("""drop table iud.dest120""").show
- sql("""create table iud.dest120 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""drop table iud.dest120""")
+ sql("""create table iud.dest120 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest120""")
sql("""update iud.dest120 d set (c3, c5 ) = (select s.c33 ,s.c55 from iud.source2 s where d.c1 = s.c11)""").show()
checkAnswer(
sql("""select c3,c5 from iud.dest120 """),
Seq(Row("MGM","Disco"),Row("RGK","Music"),Row("cc","ccc"),Row("dd","ddd"),Row("ee","eee"))
)
- sql("""drop table iud.dest120""").show
+ sql("""drop table iud.dest120""")
}
test("update carbon table[using destination table where and exist]") {
- sql("""drop table iud.dest130""").show
- sql("""create table iud.dest130 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""drop table iud.dest130""")
+ sql("""create table iud.dest130 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest130""")
sql("""update iud.dest130 dd set (c2, c5 ) = (c2 + 1, "xyx") where dd.c1 = 'a'""").show()
checkAnswer(
sql("""select c2,c5 from iud.dest130 where c1 = 'a' """),
Seq(Row(2,"xyx"))
)
- sql("""drop table iud.dest130""").show
+ sql("""drop table iud.dest130""")
}
test("update carbon table[using destination table (concat) where and exist]") {
- sql("""drop table iud.dest140""").show
- sql("""create table iud.dest140 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""drop table iud.dest140""")
+ sql("""create table iud.dest140 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest140""")
sql("""update iud.dest140 d set (c2, c5 ) = (c2 + 1, concat(c5 , "z")) where d.c1 = 'a'""").show()
checkAnswer(
sql("""select c2,c5 from iud.dest140 where c1 = 'a'"""),
Seq(Row(2,"aaaz"))
)
- sql("""drop table iud.dest140""").show
+ sql("""drop table iud.dest140""")
}
test("update carbon table[using destination table (concat) with where") {
- sql("""drop table iud.dest150""").show
- sql("""create table iud.dest150 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""drop table iud.dest150""")
+ sql("""create table iud.dest150 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest150""")
sql("""update iud.dest150 d set (c5) = (concat(c5 , "z")) where d.c1 = 'b'""").show()
checkAnswer(
sql("""select c5 from iud.dest150 where c1 = 'b' """),
Seq(Row("bbbz"))
)
- sql("""drop table iud.dest150""").show
+ sql("""drop table iud.dest150""")
}
test("update table with data for datatype mismatch with column ") {
@@ -266,45 +266,32 @@ class UpdateCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
}
test("update carbon table-error[more columns in source table not allowed") {
- try {
+ val exception = intercept[Exception] {
sql("""update iud.dest d set (c2, c5 ) = (c2 + 1, concat(c5 , "z"), "abc")""").show()
- assert(false)
- } catch {
- case ex: Exception =>
- assertResult("Number of source and destination columns are not matching")(ex.getMessage)
}
+ assertResult("Number of source and destination columns are not matching")(exception.getMessage)
}
+
test("update carbon table-error[no set columns") {
- try {
+ intercept[Exception] {
sql("""update iud.dest d set () = ()""").show()
- assert(false)
- } catch {
- case ex: org.apache.spark.sql.AnalysisException =>
- assert(true)
}
}
+
test("update carbon table-error[no set columns with updated column") {
- try {
+ intercept[Exception] {
sql("""update iud.dest d set = (c1+1)""").show()
- assert(false)
- } catch {
- case ex: org.apache.spark.sql.AnalysisException =>
- assert(true)
}
}
test("update carbon table-error[one set column with two updated column") {
- try {
+ intercept[Exception] {
sql("""update iud.dest set c2 = (c2 + 1, concat(c5 , "z") )""").show()
- assert(false)
- } catch {
- case ex: org.apache.spark.sql.AnalysisException =>
- assert(true)
}
}
test("""update carbon [special characters in value- test parsing logic ]""") {
- sql("""drop table iud.dest160""").show
- sql("""create table iud.dest160 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""drop table iud.dest160""")
+ sql("""create table iud.dest160 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest160""")
sql("""update iud.dest160 set(c1) = ("ab\')$*)(&^)")""").show()
sql("""update iud.dest160 set(c1) = ('abd$asjdh$adasj$l;sdf$*)$*)(&^')""").show()
@@ -316,70 +303,58 @@ class UpdateCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
sql("""update iud.dest160 d set (c3,c5) = (select s.c33,'a\\a\\' from iud.source2 s where d.c1 = s.c11 and d.c2 = s.c22) where d.c2 between 1 and 3""").show()
sql("""update iud.dest160 d set (c3,c5) =(select s.c33,'a\'a\\' from iud.source2 s where d.c1 = s.c11 and d.c2 = s.c22) where d.c2 between 1 and 3""").show()
sql("""update iud.dest160 d set (c3,c5)=(select s.c33,'\\a\'a\"' from iud.source2 s where d.c1 = s.c11 and d.c2 = s.c22) where d.c2 between 1 and 3""").show()
- sql("""drop table iud.dest160""").show
+ sql("""drop table iud.dest160""")
}
test("""update carbon [sub query, between and existing in outer condition.(Customer query ) ]""") {
- sql("""drop table iud.dest170""").show
- sql("""create table iud.dest170 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""drop table iud.dest170""")
+ sql("""create table iud.dest170 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest170""")
sql("""update iud.dest170 d set (c3)=(select s.c33 from iud.source2 s where d.c1 = s.c11 and d.c2 = s.c22) where d.c2 between 1 and 3""").show()
checkAnswer(
sql("""select c3 from iud.dest170 as d where d.c2 between 1 and 3"""),
Seq(Row("MGM"), Row("RGK"), Row("cc"))
)
- sql("""drop table iud.dest170""").show
+ sql("""drop table iud.dest170""")
}
test("""update carbon [self join select query ]""") {
- sql("""drop table iud.dest171""").show
- sql("""create table iud.dest171 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""drop table iud.dest171""")
+ sql("""create table iud.dest171 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest171""")
sql("""update iud.dest171 d set (c3)=(select concat(s.c3 , "z") from iud.dest171 s where d.c2 = s.c2)""").show
- sql("""drop table iud.dest172""").show
- sql("""create table iud.dest172 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""").show()
+ sql("""drop table iud.dest172""")
+ sql("""create table iud.dest172 (c1 string,c2 int,c3 string,c5 string) STORED BY 'org.apache.carbondata.format'""")
sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/IUD/dest.csv' INTO table iud.dest172""")
sql("""update iud.dest172 d set (c3)=( concat(c3 , "z"))""").show
checkAnswer(
sql("""select c3 from iud.dest171"""),
sql("""select c3 from iud.dest172""")
)
- sql("""drop table iud.dest171""").show
- sql("""drop table iud.dest172""").show
+ sql("""drop table iud.dest171""")
+ sql("""drop table iud.dest172""")
}
test("update carbon table-error[closing bracket missed") {
- try {
+ intercept[Exception] {
sql("""update iud.dest d set (c2) = (194""").show()
- assert(false)
- } catch {
- case ex: Exception =>
- assert(true)
}
}
test("update carbon table-error[starting bracket missed") {
- try {
+ intercept[Exception] {
sql("""update iud.dest d set (c2) = 194)""").show()
- assert(false)
- } catch {
- case ex: Exception =>
- assert(true)
}
}
test("update carbon table-error[missing starting and closing bracket") {
- try {
+ intercept[Exception] {
sql("""update iud.dest d set (c2) = 194""").show()
- assert(false)
- } catch {
- case ex: Exception =>
- assert(true)
}
}
test("test create table with column name as tupleID"){
- try {
+ intercept[Exception] {
sql("CREATE table carbontable (empno int, tupleID String, " +
"designation String, doj Timestamp, workgroupcategory int, " +
"workgroupcategoryname String, deptno int, deptname String, projectcode int, " +
@@ -387,8 +362,6 @@ class UpdateCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
"utilization int,salary int) STORED BY 'org.apache.carbondata.format' " +
"TBLPROPERTIES('DICTIONARY_INCLUDE'='empno,workgroupcategory,deptno,projectcode'," +
"'DICTIONARY_EXCLUDE'='empname')")
- }catch {
- case ex: Exception => assert(true)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/533835ca/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
index 376b352..8841918 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
@@ -148,7 +148,7 @@ class AutoHighCardinalityIdentifyTestCase extends QueryTest with BeforeAndAfterA
val oldTable = relation("highcard").tableMeta.carbonTable
sql(s"LOAD DATA LOCAL INPATH '$filePath' into table highcard")
val newTable = relation("highcard").tableMeta.carbonTable
- sql(s"select count(hc1) from highcard").show
+ sql(s"select count(hc1) from highcard")
// check dictionary file
checkDictFile(newTable)
@@ -160,7 +160,7 @@ class AutoHighCardinalityIdentifyTestCase extends QueryTest with BeforeAndAfterA
val oldTable = relation("colgrp_highcard").tableMeta.carbonTable
sql(s"LOAD DATA LOCAL INPATH '$filePath' into table colgrp_highcard")
val newTable = relation("colgrp_highcard").tableMeta.carbonTable
- sql(s"select hc1 from colgrp_highcard").show
+ sql(s"select hc1 from colgrp_highcard")
// check dictionary file
val tableIdentifier = new CarbonTableIdentifier(newTable.getDatabaseName,
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/533835ca/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 06ab47d..bb797ec 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -268,7 +268,7 @@ object LoadTable {
val catalog = CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore
- // upate the schema modified time
+ // update the schema modified time
catalog.updateSchemasUpdatedTime(catalog.touchSchemaFileSystemTime(
carbonLoadModel.getDatabaseName,
carbonLoadModel.getTableName))
@@ -362,7 +362,7 @@ case class LoadTable(
)
try {
// take lock only in case of normal data load.
- if (!updateModel.isDefined) {
+ if (updateModel.isEmpty) {
if (carbonLock.lockWithRetries()) {
LOGGER.info("Successfully able to get the table metadata file lock")
} else {
@@ -399,7 +399,7 @@ case class LoadTable(
val quoteChar = options.getOrElse("quotechar", "\"")
val fileHeader = options.getOrElse("fileheader", "")
val escapeChar = options.getOrElse("escapechar", "\\")
- val commentchar = options.getOrElse("commentchar", "#")
+ val commentChar = options.getOrElse("commentchar", "#")
val columnDict = options.getOrElse("columndict", null)
val serializationNullFormat = options.getOrElse("serialization_null_format", "\\N")
val badRecordsLoggerEnable = options.getOrElse("bad_records_logger_enable", "false")
@@ -416,7 +416,7 @@ case class LoadTable(
val maxColumns = options.getOrElse("maxcolumns", null)
carbonLoadModel.setEscapeChar(checkDefaultValue(escapeChar, "\\"))
carbonLoadModel.setQuoteChar(checkDefaultValue(quoteChar, "\""))
- carbonLoadModel.setCommentChar(checkDefaultValue(commentchar, "#"))
+ carbonLoadModel.setCommentChar(checkDefaultValue(commentChar, "#"))
carbonLoadModel.setDateFormat(dateFormat)
carbonLoadModel.setDefaultTimestampFormat(CarbonProperties.getInstance().getProperty(
CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
@@ -735,9 +735,7 @@ case class CarbonDropTableCommand(ifExistsSet: Boolean,
sys.error("Table is locked for deletion. Please try after some time")
}
LOGGER.audit(s"Deleting table [$tableName] under database [$dbName]")
- val carbonTable = catalog
- .getTableFromMetadata(dbName, tableName)
- .map(_.carbonTable).getOrElse(null)
+ val carbonTable = catalog.getTableFromMetadata(dbName, tableName).map(_.carbonTable).orNull
if (null != carbonTable) {
// clear driver B-tree and dictionary cache
ManageDictionaryAndBTree.clearBTreeAndDictionaryLRUCache(carbonTable)
@@ -787,9 +785,8 @@ private[sql] case class DescribeCommandFormatted(
val fieldName = field.name.toLowerCase
val comment = if (dims.contains(fieldName)) {
val dimension = relation.metaData.carbonTable.getDimensionByName(
- relation.tableMeta.carbonTableIdentifier.getTableName,
- fieldName)
- if (null != dimension.getColumnProperties && dimension.getColumnProperties.size() > 0) {
+ relation.tableMeta.carbonTableIdentifier.getTableName, fieldName)
+ if (null != dimension.getColumnProperties && !dimension.getColumnProperties.isEmpty) {
colProps.append(fieldName).append(".")
.append(mapper.writeValueAsString(dimension.getColumnProperties))
.append(",")
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/533835ca/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
index ac7dce3..bd70d85 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
@@ -120,7 +120,7 @@ class CarbonDataSourceSuite extends QueryTest with BeforeAndAfterAll {
.mode(SaveMode.Overwrite)
.save()
sql(s"select city, sum(m1) from testBigData " +
- s"where country='country12' group by city order by city").show()
+ s"where country='country12' group by city order by city")
checkAnswer(
sql(s"select city, sum(m1) from testBigData " +
s"where country='country12' group by city order by city"),
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/533835ca/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java b/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
index 156c57e..23f7230 100644
--- a/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
+++ b/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
@@ -78,8 +78,6 @@ public class BlockIndexStoreTest extends TestCase {
@Test public void testLoadAndGetTaskIdToSegmentsMapForSingleSegment()
throws IOException {
- String canonicalPath =
- new File(this.getClass().getResource("/").getPath() + "/../../").getCanonicalPath();
File file = getPartFile();
TableBlockInfo info =
new TableBlockInfo(file.getAbsolutePath(), 0, "0", new String[] { "loclhost" },
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/533835ca/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java b/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
index 87f1190..12dcd15 100644
--- a/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
+++ b/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
@@ -120,9 +120,7 @@ public class StoreCreator {
* Create store without any restructure
*/
public static void createCarbonStore() {
-
try {
-
String factFilePath = new File("../hadoop/src/test/resources/data.csv").getCanonicalPath();
File storeDir = new File(absoluteTableIdentifier.getStorePath());
CarbonUtil.deleteFoldersAndFiles(storeDir);
@@ -133,7 +131,6 @@ public class StoreCreator {
writeDictionary(factFilePath, table);
CarbonDataLoadSchema schema = new CarbonDataLoadSchema(table);
CarbonLoadModel loadModel = new CarbonLoadModel();
- String partitionId = "0";
loadModel.setCarbonDataLoadSchema(schema);
loadModel.setDatabaseName(absoluteTableIdentifier.getCarbonTableIdentifier().getDatabaseName());
loadModel.setTableName(absoluteTableIdentifier.getCarbonTableIdentifier().getTableName());
@@ -173,7 +170,6 @@ public class StoreCreator {
} catch (Exception e) {
e.printStackTrace();
}
-
}
private static CarbonTable createTable() throws IOException {
@@ -378,8 +374,6 @@ public class StoreCreator {
CarbonProperties.getInstance().addProperty("is.compressed.keyblock", "false");
CarbonProperties.getInstance().addProperty("carbon.leaf.node.size", "120000");
- String fileNamePrefix = "";
-
String graphPath =
outPutLoc + File.separator + loadModel.getDatabaseName() + File.separator + tableName
+ File.separator + 0 + File.separator + 1 + File.separator + tableName + ".ktr";
@@ -417,11 +411,6 @@ public class StoreCreator {
info.setDatabaseName(databaseName);
info.setTableName(tableName);
-// DataGraphExecuter graphExecuter = new DataGraphExecuter(dataProcessTaskStatus);
-// graphExecuter
-// .executeGraph(graphPath, info, loadModel.getSchema());
- // LoadMetadataDetails[] loadDetails =
- // CarbonUtil.readLoadMetadata(loadModel.schema.getCarbonTable().getMetaDataFilepath());
writeLoadMetadata(loadModel.getCarbonDataLoadSchema(), loadModel.getTableName(), loadModel.getTableName(),
new ArrayList<LoadMetadataDetails>());
@@ -444,7 +433,6 @@ public class StoreCreator {
break;
}
}
- // Files.copy(factFile.toPath(), file.toPath(), REPLACE_EXISTING);
factFile.renameTo(new File(segLocation + "/" + factFile.getName()));
CarbonUtil.deleteFoldersAndFiles(folder);
}
[14/50] [abbrv] incubator-carbondata git commit: [CARBONDATA-990] fix
instruction error This closes #857
Posted by ja...@apache.org.
[CARBONDATA-990] fix instruction error This closes #857
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/4b615d8c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/4b615d8c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/4b615d8c
Branch: refs/heads/12-dev
Commit: 4b615d8cdb10542d99c339077d814b7b7b004e6d
Parents: 373f4e7 64d2a4b
Author: chenliang613 <ch...@huawei.com>
Authored: Wed Apr 26 19:08:34 2017 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Wed Apr 26 19:08:34 2017 +0800
----------------------------------------------------------------------
docs/installation-guide.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[29/50] [abbrv] incubator-carbondata git commit: Use map to store
SegmentProperties and reuse SegmentProperties.
Posted by ja...@apache.org.
Use map to store SegmentProperties and reuse SegmentProperties.
Add comment.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/33a3858a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/33a3858a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/33a3858a
Branch: refs/heads/12-dev
Commit: 33a3858a8c28551dd37f4a530cb52d97d888f810
Parents: 58be73f
Author: Yadong Qi <qi...@gmail.com>
Authored: Fri Apr 21 16:21:04 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Thu Apr 27 22:15:44 2017 -0700
----------------------------------------------------------------------
.../core/datastore/SegmentTaskIndexStore.java | 61 +++++++++++++++++++-
.../core/datastore/block/SegmentTaskIndex.java | 10 ++--
.../schema/table/column/ColumnSchema.java | 10 +++-
.../datastore/block/SegmentTaskIndexTest.java | 4 +-
4 files changed, 76 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/33a3858a/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
index 862455e..b1eaa84 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
@@ -19,6 +19,7 @@ package org.apache.carbondata.core.datastore;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -31,12 +32,14 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.CarbonLRUCache;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.block.SegmentTaskIndex;
import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.exception.IndexBuilderException;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.mutate.UpdateVO;
import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -68,6 +71,9 @@ public class SegmentTaskIndexStore
*/
private Map<String, Object> segmentLockMap;
+ private Map<SegmentPropertiesWrapper, SegmentProperties> segmentProperties =
+ new HashMap<SegmentPropertiesWrapper, SegmentProperties>();
+
/**
* constructor to initialize the SegmentTaskIndexStore
*
@@ -345,7 +351,26 @@ public class SegmentTaskIndexStore
List<DataFileFooter> footerList = CarbonUtil
.readCarbonIndexFile(taskBucketHolder.taskNo, taskBucketHolder.bucketNumber,
tableBlockInfoList, tableIdentifier);
- AbstractIndex segment = new SegmentTaskIndex();
+
+ // Reuse SegmentProperties object if tableIdentifier, columnsInTable and columnCardinality are
+ // the same.
+ List<ColumnSchema> columnsInTable = footerList.get(0).getColumnInTable();
+ int[] columnCardinality = footerList.get(0).getSegmentInfo().getColumnCardinality();
+ SegmentPropertiesWrapper segmentPropertiesWrapper =
+ new SegmentPropertiesWrapper(tableIdentifier, columnsInTable, columnCardinality);
+ SegmentProperties segmentProperties;
+ if (this.segmentProperties.containsKey(segmentPropertiesWrapper)) {
+ segmentProperties = this.segmentProperties.get(segmentPropertiesWrapper);
+ } else {
+ // create a metadata details
+ // this will be useful in query handling
+ // all the data file metadata will have common segment properties we
+ // can use first one to get create the segment properties
+ segmentProperties = new SegmentProperties(columnsInTable, columnCardinality);
+ this.segmentProperties.put(segmentPropertiesWrapper, segmentProperties);
+ }
+
+ AbstractIndex segment = new SegmentTaskIndex(segmentProperties);
// file path of only first block is passed as it all table block info path of
// same task id will be same
segment.buildIndex(footerList);
@@ -396,4 +421,38 @@ public class SegmentTaskIndexStore
return result;
}
}
+
+ /**
+ * This class wraps tableIdentifier, columnsInTable and columnCardinality as a key to determine
+ * whether the SegmentProperties object can be reused.
+ */
+ public static class SegmentPropertiesWrapper {
+ private AbsoluteTableIdentifier tableIdentifier;
+ private List<ColumnSchema> columnsInTable;
+ private int[] columnCardinality;
+
+ public SegmentPropertiesWrapper(AbsoluteTableIdentifier tableIdentifier,
+ List<ColumnSchema> columnsInTable, int[] columnCardinality) {
+ this.tableIdentifier = tableIdentifier;
+ this.columnsInTable = columnsInTable;
+ this.columnCardinality = columnCardinality;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof SegmentPropertiesWrapper)) {
+ return false;
+ }
+ SegmentPropertiesWrapper other = (SegmentPropertiesWrapper) obj;
+ return tableIdentifier.equals(other.tableIdentifier)
+ && columnsInTable.equals(other.columnsInTable)
+ && Arrays.equals(columnCardinality, other.columnCardinality);
+ }
+
+ @Override
+ public int hashCode() {
+ return tableIdentifier.hashCode()
+ + columnsInTable.hashCode() + Arrays.hashCode(columnCardinality);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/33a3858a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java
index 7293323..f577ede 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndex.java
@@ -29,17 +29,15 @@ import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
*/
public class SegmentTaskIndex extends AbstractIndex {
+ public SegmentTaskIndex(SegmentProperties segmentProperties) {
+ this.segmentProperties = segmentProperties;
+ }
+
/**
* Below method is store the blocks in some data structure
*
*/
public void buildIndex(List<DataFileFooter> footerList) {
- // create a metadata details
- // this will be useful in query handling
- // all the data file metadata will have common segment properties we
- // can use first one to get create the segment properties
- segmentProperties = new SegmentProperties(footerList.get(0).getColumnInTable(),
- footerList.get(0).getSegmentInfo().getColumnCardinality());
// create a segment builder info
// in case of segment create we do not need any file path and each column value size
// as Btree will be build as per min max and start key
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/33a3858a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
index 7aa0900..abe065c 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
@@ -263,7 +263,8 @@ public class ColumnSchema implements Serializable {
@Override public int hashCode() {
final int prime = 31;
int result = 1;
- result = prime * result + ((columnName == null) ? 0 : columnName.hashCode());
+ result = prime * result + ((columnName == null) ? 0 : columnName.hashCode()) +
+ ((dataType == null) ? 0 : dataType.hashCode());
return result;
}
@@ -288,6 +289,13 @@ public class ColumnSchema implements Serializable {
} else if (!columnName.equals(other.columnName)) {
return false;
}
+ if (dataType == null) {
+ if (other.dataType != null) {
+ return false;
+ }
+ } else if (!dataType.equals(other.dataType)) {
+ return false;
+ }
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/33a3858a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexTest.java
index ac2cfaa..4c08c6b 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexTest.java
@@ -58,7 +58,6 @@ public class SegmentTaskIndexTest {
@Mock public void build(BTreeBuilderInfo segmentBuilderInfos) {}
};
long numberOfRows = 100;
- SegmentTaskIndex segmentTaskIndex = new SegmentTaskIndex();
columnSchema.setColumnName("employeeName");
columnSchemaList.add(new ColumnSchema());
@@ -68,6 +67,9 @@ public class SegmentTaskIndexTest {
footer.setNumberOfRows(numberOfRows);
footerList.add(footer);
+ SegmentProperties properties = new SegmentProperties(footerList.get(0).getColumnInTable(),
+ footerList.get(0).getSegmentInfo().getColumnCardinality());
+ SegmentTaskIndex segmentTaskIndex = new SegmentTaskIndex(properties);
segmentTaskIndex.buildIndex(footerList);
assertEquals(footerList.get(0).getNumberOfRows(), numberOfRows);
[16/50] [abbrv] incubator-carbondata git commit: [CARBONDATA-986] Add
alter table example This closes #850
Posted by ja...@apache.org.
[CARBONDATA-986] Add alter table example This closes #850
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/598bbd2b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/598bbd2b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/598bbd2b
Branch: refs/heads/12-dev
Commit: 598bbd2b8a41ac0547799d0576f07e6b0da9888a
Parents: 4b615d8 1f9721f
Author: chenliang613 <ch...@huawei.com>
Authored: Wed Apr 26 23:53:20 2017 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Wed Apr 26 23:53:20 2017 +0800
----------------------------------------------------------------------
.../carbondata/examples/AlterTableExample.scala | 97 ++++++++++++++++++++
1 file changed, 97 insertions(+)
----------------------------------------------------------------------
[09/50] [abbrv] incubator-carbondata git commit:
docs-for-optimizing-mass-data-loading
Posted by ja...@apache.org.
docs-for-optimizing-mass-data-loading
docs-for-optimizing-mass-data-loading
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/7f8fd8e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/7f8fd8e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/7f8fd8e8
Branch: refs/heads/12-dev
Commit: 7f8fd8e84cc2c39d355ef3101bda73f03711a851
Parents: 060b455
Author: WilliamZhu <al...@gmail.com>
Authored: Tue Apr 25 10:11:46 2017 +0800
Committer: WilliamZhu <al...@gmail.com>
Committed: Tue Apr 25 10:17:58 2017 +0800
----------------------------------------------------------------------
docs/useful-tips-on-carbondata.md | 85 +++++++++++++++++++++++-----------
1 file changed, 59 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7f8fd8e8/docs/useful-tips-on-carbondata.md
----------------------------------------------------------------------
diff --git a/docs/useful-tips-on-carbondata.md b/docs/useful-tips-on-carbondata.md
index b1ff903..bfddf29 100644
--- a/docs/useful-tips-on-carbondata.md
+++ b/docs/useful-tips-on-carbondata.md
@@ -22,17 +22,19 @@ This tutorial guides you to create CarbonData Tables and optimize performance.
The following sections will elaborate on the above topics :
* [Suggestions to create CarbonData Table](#suggestions-to-create-carbondata-table)
-* [Configurations For Optimizing CarbonData Performance](#configurations-for-optimizing-carbondata-performance)
+* [Configuration for Optimizing Data Loading performance for Massive Data](#configuration-for-optimizing-data-loading-performance-for-massive-data)
+* [Optimizing Mass Data Loading](#optimizing-mass-data-loading)
+
## Suggestions to Create CarbonData Table
Recently CarbonData was used to analyze performance of Telecommunication field.
The results of the analysis for table creation with dimensions ranging from
-10 thousand to 10 billion rows and 100 to 300 columns have been summarized below.
+10 thousand to 10 billion rows and 100 to 300 columns have been summarized below.
The following table describes some of the columns from the table used.
-
-
+
+
**Table Column Description**
| Column Name | Data Type | Cardinality | Attribution |
@@ -51,29 +53,29 @@ CarbonData has more than 50 test cases, on the basis of these we have following
* **Put the frequently-used column filter in the beginning**
- For example, MSISDN filter is used in most of the query then we must put the MSISDN in the first column.
+ For example, MSISDN filter is used in most of the query then we must put the MSISDN in the first column.
The create table command can be modified as suggested below :
```
create table carbondata_table(
msisdn String,
...
- )STORED BY 'org.apache.carbondata.format'
+ )STORED BY 'org.apache.carbondata.format'
TBLPROPERTIES ( 'DICTIONARY_EXCLUDE'='MSISDN,..',
'DICTIONARY_INCLUDE'='...');
```
-
+
Now the query with MSISDN in the filter will be more efficient.
* **Put the frequently-used columns in the order of low to high cardinality**
-
+
If the table in the specified query has multiple columns which are frequently used to filter the results, it is suggested to put
- the columns in the order of cardinality low to high. This ordering of frequently used columns improves the compression ratio and
+ the columns in the order of cardinality low to high. This ordering of frequently used columns improves the compression ratio and
enhances the performance of queries with filter on these columns.
-
- For example if MSISDN, HOST and Dime_1 are frequently-used columns, then the column order of table is suggested as
- Dime_1>HOST>MSISDN as Dime_1 has the lowest cardinality.
+
+ For example if MSISDN, HOST and Dime_1 are frequently-used columns, then the column order of table is suggested as
+ Dime_1>HOST>MSISDN as Dime_1 has the lowest cardinality.
The create table command can be modified as suggested below :
```
@@ -82,7 +84,7 @@ The create table command can be modified as suggested below :
HOST String,
MSISDN String,
...
- )STORED BY 'org.apache.carbondata.format'
+ )STORED BY 'org.apache.carbondata.format'
TBLPROPERTIES ( 'DICTIONARY_EXCLUDE'='MSISDN,HOST..',
'DICTIONARY_INCLUDE'='Dime_1..');
```
@@ -100,7 +102,7 @@ The create table command can be modified as below :
HOST String,
MSISDN String,
...
- )STORED BY 'org.apache.carbondata.format'
+ )STORED BY 'org.apache.carbondata.format'
TBLPROPERTIES ( 'DICTIONARY_EXCLUDE'='MSISDN,HOST,IMSI..',
'DICTIONARY_INCLUDE'='Dime_1,END_TIME,BEGIN_TIME..');
```
@@ -108,7 +110,7 @@ The create table command can be modified as below :
* **For measure type columns with non high accuracy, replace Numeric(20,0) data type with Double data type**
- For columns of measure type, not requiring high accuracy, it is suggested to replace Numeric data type with Double to enhance
+ For columns of measure type, not requiring high accuracy, it is suggested to replace Numeric data type with Double to enhance
query performance. The create table command can be modified as below :
```
@@ -121,17 +123,17 @@ query performance. The create table command can be modified as below :
counter_2 double,
...
counter_100 double
- )STORED BY 'org.apache.carbondata.format'
+ )STORED BY 'org.apache.carbondata.format'
TBLPROPERTIES ( 'DICTIONARY_EXCLUDE'='MSISDN,HOST,IMSI',
'DICTIONARY_INCLUDE'='Dime_1,END_TIME,BEGIN_TIME');
```
The result of performance analysis of test-case shows reduction in query execution time from 15 to 3 seconds, thereby improving performance by nearly 5 times.
-
+
* **Columns of incremental character should be re-arranged at the end of dimensions**
Consider the following scenario where data is loaded each day and the start_time is incremental for each load, it is
-suggested to put start_time at the end of dimensions.
+suggested to put start_time at the end of dimensions.
Incremental values are efficient in using min/max index. The create table command can be modified as below :
@@ -145,25 +147,56 @@ suggested to put start_time at the end of dimensions.
BEGIN_TIME bigint,
...
counter_100 double
- )STORED BY 'org.apache.carbondata.format'
+ )STORED BY 'org.apache.carbondata.format'
TBLPROPERTIES ( 'DICTIONARY_EXCLUDE'='MSISDN,HOST,IMSI',
- 'DICTIONARY_INCLUDE'='Dime_1,END_TIME,BEGIN_TIME');
+ 'DICTIONARY_INCLUDE'='Dime_1,END_TIME,BEGIN_TIME');
```
* **Avoid adding high cardinality columns to dictionary**
- If the system has low memory configuration, then it is suggested to exclude high cardinality columns from the dictionary to
-enhance load performance. Creation of dictionary for high cardinality columns at time of load will degrade load performance due to
-excessive memory usage.
+ If the system has low memory configuration, then it is suggested to exclude high cardinality columns from the dictionary to
+enhance load performance. Creation of dictionary for high cardinality columns at time of load will degrade load performance due to
+excessive memory usage.
By default CarbonData determines the cardinality at the first data load and allows for dictionary creation only if the cardinality is less than
1 million.
+
+## Configuration for Optimizing Data Loading performance for Massive Data
+
+
+ CarbonData supports large data load, in this process sorting data while loading consumes a lot of memory and disk IO and
+ this can result sometimes in "Out Of Memory" exception.
+ If you do not have much memory to use, then you may prefer to slow the speed of data loading instead of data load failure.
+ You can configure CarbonData by tuning following properties in carbon.properties file to get a better performance.:
+
+| Parameter | Default Value | Description/Tuning |
+|-----------|-------------|--------|
+|carbon.number.of.cores.while.loading|Default: 2.This value should be >= 2|Specifies the number of cores used for data processing during data loading in CarbonData. |
+|carbon.sort.size|Data loading|Default: 100000. The value should be >= 100.|Threshhold to write local file in sort step when loading data|
+|carbon.sort.file.write.buffer.size|Default: 50000.|DataOutputStream buffer. |
+|carbon.number.of.cores.block.sort|Default: 7 | If you have huge memory and cpus, increase it as you will|
+|carbon.merge.sort.reader.thread|Default: 3 |Specifies the number of cores used for temp file merging during data loading in CarbonData.|
+|carbon.merge.sort.prefetch|Default: true | You may want set this value to false if you have not enough memory|
+
+
+For example, if there are 10 million records ,and i have only 16 cores ,64GB memory, will be loaded to CarbonData table.
+Using the default configuration always fail in sort step. Modify carbon.properties as suggested below
+
+
+```
+carbon.number.of.cores.block.sort=1
+carbon.merge.sort.reader.thread=1
+carbon.sort.size=5000
+carbon.sort.file.write.buffer.size=5000
+carbon.merge.sort.prefetch=false
+```
+
## Configurations for Optimizing CarbonData Performance
-Recently we did some performance POC on CarbonData for Finance and telecommunication Field. It involved detailed queries and aggregation
+Recently we did some performance POC on CarbonData for Finance and telecommunication Field. It involved detailed queries and aggregation
scenarios. After the completion of POC, some of the configurations impacting the performance have been identified and tabulated below :
| Parameter | Location | Used For | Description | Tuning |
@@ -176,5 +209,5 @@ scenarios. After the completion of POC, some of the configurations impacting the
| carbon.detail.batch.size | spark/carbonlib/carbon.properties | Data loading | The buffer size to store records, returned from the block scan. | In limit scenario this parameter is very important. For example your query limit is 1000. But if we set this value to 3000 that means we get 3000 records from scan but spark will only take 1000 rows. So the 2000 remaining are useless. In one Finance test case after we set it to 100, in the limit 1000 scenario the performance increase about 2 times in comparison to if we set this value to 12000. |
| carbon.use.local.dir | spark/carbonlib/carbon.properties | Data loading | Whether use YARN local directories for multi-table load disk load balance | If this is set it to true CarbonData will use YARN local directories for multi-table load disk load balance, that will improve the data load performance. |
-
-
\ No newline at end of file
+Note: If your CarbonData instance is provided only for query, you may specify the conf 'spark.speculation=true' which is conf
+ in spark.
\ No newline at end of file
[10/50] [abbrv] incubator-carbondata git commit:
docs-for-optimizing-massive-data-loading This closes #845
Posted by ja...@apache.org.
docs-for-optimizing-massive-data-loading This closes #845
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/d695a8af
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/d695a8af
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/d695a8af
Branch: refs/heads/12-dev
Commit: d695a8af443612167cc047ad734a8cd15ef36361
Parents: 060b455 7f8fd8e
Author: chenliang613 <ch...@huawei.com>
Authored: Tue Apr 25 15:48:13 2017 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Tue Apr 25 15:48:13 2017 +0800
----------------------------------------------------------------------
docs/useful-tips-on-carbondata.md | 85 +++++++++++++++++++++++-----------
1 file changed, 59 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
[06/50] [abbrv] incubator-carbondata git commit: update CompareTest
Posted by ja...@apache.org.
update CompareTest
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/d3abcd91
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/d3abcd91
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/d3abcd91
Branch: refs/heads/12-dev
Commit: d3abcd91b4fa07c2ee09ea1e811ca008d10e3e1c
Parents: 462e29a
Author: jackylk <ja...@huawei.com>
Authored: Sun Apr 23 12:51:34 2017 -0600
Committer: jackylk <ja...@huawei.com>
Committed: Sun Apr 23 12:51:34 2017 -0600
----------------------------------------------------------------------
.../main/scala/org/apache/carbondata/examples/CompareTest.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d3abcd91/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
index 41a7850..1d24e36 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
@@ -70,7 +70,7 @@ object CompareTest {
val rdd = spark.sparkContext
.parallelize(1 to 10 * 1000 * 1000, 4)
.map { x =>
- ("city" + x % 8, "country" + x % 1103, "planet" + x % 10007, x.toString,
+ ("city" + x % 8, "country" + x % 1103, "planet" + x % 10007, "IDENTIFIER" + x.toString,
(x % 16).toShort, x / 2, (x << 1).toLong, x.toDouble / 13, x.toDouble / 11)
}.map { x =>
Row(x._1, x._2, x._3, x._4, x._5, x._6, x._7, x._8, x._9)
@@ -279,6 +279,7 @@ object CompareTest {
val loadParquetTime = loadParquetTable(spark, df)
val loadCarbonV3Time = loadCarbonTable(spark, df, version = "3")
println(s"load completed, time: $loadParquetTime, $loadCarbonV3Time")
+ df.unpersist()
spark.read.parquet(parquetTableName).registerTempTable(parquetTableName)
}
@@ -326,7 +327,6 @@ object CompareTest {
s""""fetched":${parquetResult(index)._2}, """ +
s""""type":"${query.queryType}", """ +
s""""desc":"${query.desc}", """ +
- s""""timestamp": "$timestamp" """ +
s""""date": "${formatter.format(date)}" """ +
"}"
)
[39/50] [abbrv] incubator-carbondata git commit: [CARBONDATA-981] Fix
metaStorePath with hive-site.xml. This closes #867
Posted by ja...@apache.org.
[CARBONDATA-981] Fix metaStorePath with hive-site.xml. This closes #867
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/3252e7ce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/3252e7ce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/3252e7ce
Branch: refs/heads/12-dev
Commit: 3252e7ce2b963a04313365ec59b1d1fea36f15b6
Parents: 33bfb49 0f3b6e4
Author: ravipesala <ra...@gmail.com>
Authored: Mon May 1 15:54:43 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Mon May 1 15:54:43 2017 +0530
----------------------------------------------------------------------
.../org/apache/spark/sql/CarbonSession.scala | 22 +++++++++++++-------
1 file changed, 14 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
[13/50] [abbrv] incubator-carbondata git commit: [CARBONDATA-990] fix
instruction error
Posted by ja...@apache.org.
[CARBONDATA-990] fix instruction error
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/64d2a4b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/64d2a4b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/64d2a4b1
Branch: refs/heads/12-dev
Commit: 64d2a4b14f07e4fde264da221dda336092c28e7e
Parents: 373f4e7
Author: ranmx <ra...@fosun.com>
Authored: Wed Apr 26 17:22:44 2017 +0800
Committer: ranmx <ra...@fosun.com>
Committed: Wed Apr 26 17:22:44 2017 +0800
----------------------------------------------------------------------
docs/installation-guide.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/64d2a4b1/docs/installation-guide.md
----------------------------------------------------------------------
diff --git a/docs/installation-guide.md b/docs/installation-guide.md
index 2fb6766..48de5b9 100644
--- a/docs/installation-guide.md
+++ b/docs/installation-guide.md
@@ -113,7 +113,7 @@ To get started with CarbonData : [Quick Start](quick-start-guide.md), [DDL Opera
| spark.yarn.dist.archives | Comma-separated list of archives to be extracted into the working directory of each executor. |`$SPARK_HOME/carbonlib/carbondata.tar.gz` |
| spark.executor.extraJavaOptions | A string of extra JVM options to pass to executors. For instance **NOTE**: You can enter multiple values separated by space. |`-Dcarbon.properties.filepath=carbon.properties` |
| spark.executor.extraClassPath | Extra classpath entries to prepend to the classpath of executors. **NOTE**: If SPARK_CLASSPATH is defined in spark-env.sh, then comment it and append the values in below parameter spark.driver.extraClassPath |`carbondata.tar.gz/carbonlib/*` |
- | spark.driver.extraClassPath | Extra classpath entries to prepend to the classpath of the driver. **NOTE**: If SPARK_CLASSPATH is defined in spark-env.sh, then comment it and append the value in below parameter spark.driver.extraClassPath. |`$SPARK_HOME/carbonlib/carbonlib/*` |
+ | spark.driver.extraClassPath | Extra classpath entries to prepend to the classpath of the driver. **NOTE**: If SPARK_CLASSPATH is defined in spark-env.sh, then comment it and append the value in below parameter spark.driver.extraClassPath. |`$SPARK_HOME/carbonlib/*` |
| spark.driver.extraJavaOptions | A string of extra JVM options to pass to the driver. For instance, GC settings or other logging. |`-Dcarbon.properties.filepath=$SPARK_HOME/conf/carbon.properties` |
[27/50] [abbrv] incubator-carbondata git commit: insertInto read from
OneRowRelation
Posted by ja...@apache.org.
insertInto read from OneRowRelation
fix comments
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/2a4ab4b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/2a4ab4b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/2a4ab4b1
Branch: refs/heads/12-dev
Commit: 2a4ab4b17c0ba24e98707589514d9887714c7685
Parents: 61d3f3c
Author: QiangCai <qi...@qq.com>
Authored: Thu Apr 13 18:39:12 2017 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Thu Apr 27 22:08:21 2017 -0700
----------------------------------------------------------------------
.../spark/sql/hive/CarbonAnalysisRules.scala | 17 ++++++++-
.../InsertIntoCarbonTableTestCase.scala | 40 ++++++++++++++++++++
2 files changed, 56 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2a4ab4b1/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonAnalysisRules.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonAnalysisRules.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonAnalysisRules.scala
index 45accac..6061e3e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonAnalysisRules.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonAnalysisRules.scala
@@ -18,6 +18,7 @@
package org.apache.spark.sql.hive
import org.apache.spark.sql._
+import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, Cast, NamedExpression}
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules._
import org.apache.spark.sql.execution.datasources.LogicalRelation
@@ -52,7 +53,21 @@ object CarbonPreInsertionCasts extends Rule[LogicalPlan] {
)
}
if (child.output.size >= relation.carbonRelation.output.size) {
- InsertIntoCarbonTable(relation, p.partition, p.child, p.overwrite, p.ifNotExists)
+ val newChildOutput = child.output.zipWithIndex.map { columnWithIndex =>
+ columnWithIndex._1 match {
+ case attr: Alias =>
+ Alias(attr.child, s"col${ columnWithIndex._2 }")(attr.exprId)
+ case attr: Attribute =>
+ Alias(attr, s"col${ columnWithIndex._2 }")(NamedExpression.newExprId)
+ case attr => attr
+ }
+ }
+ val newChild: LogicalPlan = if (newChildOutput == child.output) {
+ p.child
+ } else {
+ Project(newChildOutput, child)
+ }
+ InsertIntoCarbonTable(relation, p.partition, newChild, p.overwrite, p.ifNotExists)
} else {
sys.error("Cannot insert into target table because column number are different")
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2a4ab4b1/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
new file mode 100644
index 0000000..0020568
--- /dev/null
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.allqueries
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
+ override def beforeAll: Unit = {
+ sql("drop table if exists OneRowTable")
+ }
+
+ test("insert select one row") {
+ sql("create table OneRowTable(col1 string, col2 string, col3 int, col4 double) stored by 'carbondata'")
+ sql("insert into OneRowTable select '0.1', 'a.b', 1, 1.2")
+ checkAnswer(sql("select * from OneRowTable"), Seq(Row("0.1", "a.b", 1, 1.2)))
+ }
+
+ override def afterAll {
+ sql("drop table if exists OneRowTable")
+ }
+}
[47/50] [abbrv] incubator-carbondata git commit: Added options to
include and exclude dictionary columns in dataframe
Posted by ja...@apache.org.
Added options to include and exclude dictionary columns in dataframe
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/1814048e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/1814048e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/1814048e
Branch: refs/heads/12-dev
Commit: 1814048ee9edbe64f483a3212258a39efb9a58b0
Parents: 0287e1e
Author: Sanoj MG <sa...@gmail.com>
Authored: Tue Apr 18 17:22:14 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Tue May 2 09:43:27 2017 +0800
----------------------------------------------------------------------
.../testsuite/dataload/TestLoadDataFrame.scala | 48 +++++++++++++++++++-
.../spark/CarbonDataFrameWriter.scala | 5 ++
.../spark/sql/CarbonDataFrameWriter.scala | 29 ++----------
3 files changed, 57 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/1814048e/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
index 3b0fd4a..6e79a10 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataFrame.scala
@@ -27,6 +27,7 @@ import org.scalatest.BeforeAndAfterAll
class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll {
var df: DataFrame = _
var dataFrame: DataFrame = _
+ var df2: DataFrame = _
def buildTestData() = {
@@ -45,6 +46,9 @@ class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll {
StructField("string", StringType, nullable = false) :: Nil)
dataFrame = sqlContext.createDataFrame(rdd, schema)
+ df2 = sqlContext.sparkContext.parallelize(1 to 1000)
+ .map(x => ("key_" + x, "str_" + x, x, x * 2, x * 3))
+ .toDF("c1", "c2", "c3", "c4", "c5")
}
def dropTable() = {
@@ -52,7 +56,9 @@ class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS carbon2")
sql("DROP TABLE IF EXISTS carbon3")
sql("DROP TABLE IF EXISTS carbon4")
-
+ sql("DROP TABLE IF EXISTS carbon5")
+ sql("DROP TABLE IF EXISTS carbon6")
+ sql("DROP TABLE IF EXISTS carbon7")
}
@@ -114,6 +120,46 @@ class TestLoadDataFrame extends QueryTest with BeforeAndAfterAll {
sql("SELECT decimal FROM carbon4"),Seq(Row(BigDecimal.valueOf(10000.00)),Row(BigDecimal.valueOf(1234.44))))
}
+ test("test load dataframe with integer columns included in the dictionary"){
+ df2.write
+ .format("carbondata")
+ .option("tableName", "carbon5")
+ .option("compress", "true")
+ .option("dictionary_include","c3,c4")
+ .mode(SaveMode.Overwrite)
+ .save()
+ checkAnswer(
+ sql("select count(*) from carbon5 where c3 > 300"), Row(700)
+ )
+ }
+
+ test("test load dataframe with string column excluded from the dictionary"){
+ df2.write
+ .format("carbondata")
+ .option("tableName", "carbon6")
+ .option("compress", "true")
+ .option("dictionary_exclude","c2")
+ .mode(SaveMode.Overwrite)
+ .save()
+ checkAnswer(
+ sql("select count(*) from carbon6 where c3 > 300"), Row(700)
+ )
+ }
+
+ test("test load dataframe with both dictionary include and exclude specified"){
+ df2.write
+ .format("carbondata")
+ .option("tableName", "carbon7")
+ .option("compress", "true")
+ .option("dictionary_include","c3,c4")
+ .option("dictionary_exclude","c2")
+ .mode(SaveMode.Overwrite)
+ .save()
+ checkAnswer(
+ sql("select count(*) from carbon7 where c3 > 300"), Row(700)
+ )
+ }
+
override def afterAll {
dropTable
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/1814048e/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
index 0e2e4dd..9f813a8 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
@@ -175,6 +175,10 @@ class CarbonDataFrameWriter(val dataFrame: DataFrame) {
}
private def makeCreateTableString(schema: StructType, options: CarbonOption): String = {
+ val properties = Map(
+ "DICTIONARY_INCLUDE" -> options.dictionaryInclude,
+ "DICTIONARY_EXCLUDE" -> options.dictionaryExclude
+ ).filter(_._2.isDefined).map(p => s"'${p._1}' = '${p._2.get}'").mkString(",")
val carbonSchema = schema.map { field =>
s"${ field.name } ${ convertToCarbonType(field.dataType) }"
}
@@ -182,6 +186,7 @@ class CarbonDataFrameWriter(val dataFrame: DataFrame) {
CREATE TABLE IF NOT EXISTS ${options.dbName}.${options.tableName}
(${ carbonSchema.mkString(", ") })
STORED BY '${ CarbonContext.datasourceName }'
+ ${ if (properties.nonEmpty) " TBLPROPERTIES (" + properties + ")" else ""}
"""
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/1814048e/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
index 465fad0..7cd5af6 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
@@ -162,30 +162,11 @@ class CarbonDataFrameWriter(sqlContext: SQLContext, val dataFrame: DataFrame) {
val carbonSchema = schema.map { field =>
s"${ field.name } ${ convertToCarbonType(field.dataType) }"
}
- var property = new StringBuilder
- property.append(
- if (options.dictionaryInclude.isDefined) {
- s"'DICTIONARY_INCLUDE' = '${options.dictionaryInclude.get}' ,"
- } else {
- ""
- }
- ).append(
- if (options.dictionaryExclude.isDefined) {
- s"'DICTIONARY_EXCLUDE' = '${options.dictionaryExclude.get}' ,"
- } else {
- ""
- }
- ).append(
- if (options.tableBlockSize.isDefined) {
- s"'table_blocksize' = '${options.tableBlockSize.get}'"
- } else {
- ""
- }
- )
- if (property.nonEmpty && property.charAt(property.length-1) == ',') {
- property = property.replace(property.length - 1, property.length, "")
- }
-
+ val property = Map(
+ "DICTIONARY_INCLUDE" -> options.dictionaryInclude,
+ "DICTIONARY_EXCLUDE" -> options.dictionaryExclude,
+ "TABLE_BLOCKSIZE" -> options.tableBlockSize
+ ).filter(_._2.isDefined).map(p => s"'${p._1}' = '${p._2.get}'").mkString(",")
s"""
| CREATE TABLE IF NOT EXISTS ${options.dbName}.${options.tableName}
| (${ carbonSchema.mkString(", ") })
[25/50] [abbrv] incubator-carbondata git commit: change columnSchenma
to columnSchema
Posted by ja...@apache.org.
change columnSchenma to columnSchema
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/4a11fbd2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/4a11fbd2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/4a11fbd2
Branch: refs/heads/12-dev
Commit: 4a11fbd2c0aaa2a7808c18efba5926c5dd001c1b
Parents: 53a0a07
Author: mayun <si...@163.com>
Authored: Tue Apr 25 18:03:30 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Thu Apr 27 13:21:32 2017 -0700
----------------------------------------------------------------------
.../core/util/CarbonMetadataUtil.java | 34 ++++++++++----------
1 file changed, 17 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/4a11fbd2/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index 23d8baa..6398f30 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -324,7 +324,7 @@ public class CarbonMetadataUtil {
}
private static BlockletInfo getBlockletInfo(BlockletInfoColumnar blockletInfoColumnar,
- List<ColumnSchema> columnSchenma, SegmentProperties segmentProperties) throws IOException {
+ List<ColumnSchema> columnSchema, SegmentProperties segmentProperties) throws IOException {
BlockletInfo blockletInfo = new BlockletInfo();
blockletInfo.setNum_rows(blockletInfoColumnar.getNumberOfKeys());
@@ -339,10 +339,10 @@ public class CarbonMetadataUtil {
DataChunk dataChunk = new DataChunk();
dataChunk.setChunk_meta(getChunkCompressionMeta());
List<Encoding> encodings = new ArrayList<Encoding>();
- if (containsEncoding(i, Encoding.DICTIONARY, columnSchenma, segmentProperties)) {
+ if (containsEncoding(i, Encoding.DICTIONARY, columnSchema, segmentProperties)) {
encodings.add(Encoding.DICTIONARY);
}
- if (containsEncoding(i, Encoding.DIRECT_DICTIONARY, columnSchenma, segmentProperties)) {
+ if (containsEncoding(i, Encoding.DIRECT_DICTIONARY, columnSchema, segmentProperties)) {
encodings.add(Encoding.DIRECT_DICTIONARY);
}
dataChunk.setRowMajor(colGrpblock[i]);
@@ -642,13 +642,13 @@ public class CarbonMetadataUtil {
* Below method will be used to get the data chunk object for all the columns
*
* @param blockletInfoColumnar blocklet info
- * @param columnSchenma list of columns
+ * @param columnSchema list of columns
* @param segmentProperties segment properties
* @return list of data chunks
* @throws IOException
*/
public static List<DataChunk2> getDatachunk2(BlockletInfoColumnar blockletInfoColumnar,
- List<ColumnSchema> columnSchenma, SegmentProperties segmentProperties) throws IOException {
+ List<ColumnSchema> columnSchema, SegmentProperties segmentProperties) throws IOException {
List<DataChunk2> colDataChunks = new ArrayList<DataChunk2>();
int rowIdIndex = 0;
int aggregateIndex = 0;
@@ -659,10 +659,10 @@ public class CarbonMetadataUtil {
DataChunk2 dataChunk = new DataChunk2();
dataChunk.setChunk_meta(getChunkCompressionMeta());
List<Encoding> encodings = new ArrayList<Encoding>();
- if (containsEncoding(i, Encoding.DICTIONARY, columnSchenma, segmentProperties)) {
+ if (containsEncoding(i, Encoding.DICTIONARY, columnSchema, segmentProperties)) {
encodings.add(Encoding.DICTIONARY);
}
- if (containsEncoding(i, Encoding.DIRECT_DICTIONARY, columnSchenma, segmentProperties)) {
+ if (containsEncoding(i, Encoding.DIRECT_DICTIONARY, columnSchema, segmentProperties)) {
encodings.add(Encoding.DIRECT_DICTIONARY);
}
dataChunk.setRowMajor(colGrpblock[i]);
@@ -725,13 +725,13 @@ public class CarbonMetadataUtil {
* Below method will be used to get the data chunk object for all the columns
*
* @param nodeHolderList blocklet info
- * @param columnSchenma list of columns
+ * @param columnSchema list of columns
* @param segmentProperties segment properties
* @return list of data chunks
* @throws IOException
*/
private static List<DataChunk2> getDatachunk2(List<NodeHolder> nodeHolderList,
- List<ColumnSchema> columnSchenma, SegmentProperties segmentProperties, int index,
+ List<ColumnSchema> columnSchema, SegmentProperties segmentProperties, int index,
boolean isDimensionColumn) throws IOException {
List<DataChunk2> colDataChunks = new ArrayList<DataChunk2>();
DataChunk2 dataChunk = null;
@@ -745,10 +745,10 @@ public class CarbonMetadataUtil {
List<Encoding> encodings = new ArrayList<Encoding>();
if (isDimensionColumn) {
dataChunk.setData_page_length(nodeHolder.getKeyLengths()[index]);
- if (containsEncoding(index, Encoding.DICTIONARY, columnSchenma, segmentProperties)) {
+ if (containsEncoding(index, Encoding.DICTIONARY, columnSchema, segmentProperties)) {
encodings.add(Encoding.DICTIONARY);
}
- if (containsEncoding(index, Encoding.DIRECT_DICTIONARY, columnSchenma, segmentProperties)) {
+ if (containsEncoding(index, Encoding.DIRECT_DICTIONARY, columnSchema, segmentProperties)) {
encodings.add(Encoding.DIRECT_DICTIONARY);
}
dataChunk.setRowMajor(nodeHolder.getColGrpBlocks()[index]);
@@ -802,10 +802,10 @@ public class CarbonMetadataUtil {
}
public static DataChunk3 getDataChunk3(List<NodeHolder> nodeHolderList,
- List<ColumnSchema> columnSchenma, SegmentProperties segmentProperties, int index,
+ List<ColumnSchema> columnSchema, SegmentProperties segmentProperties, int index,
boolean isDimensionColumn) throws IOException {
List<DataChunk2> dataChunksList =
- getDatachunk2(nodeHolderList, columnSchenma, segmentProperties, index, isDimensionColumn);
+ getDatachunk2(nodeHolderList, columnSchema, segmentProperties, index, isDimensionColumn);
int offset = 0;
DataChunk3 dataChunk = new DataChunk3();
List<Integer> pageOffsets = new ArrayList<>();
@@ -928,13 +928,13 @@ public class CarbonMetadataUtil {
* Below method will be used to get the data chunk2 serialize object list
*
* @param nodeHolder node holder
- * @param columnSchenma table columns
+ * @param columnSchema table columns
* @param segmentProperties segment properties
* @param isDimensionColumn to get the list of dimension column or measure column
* @return list of data chunk2
* @throws IOException
*/
- public static List<byte[]> getDataChunk2(NodeHolder nodeHolder, List<ColumnSchema> columnSchenma,
+ public static List<byte[]> getDataChunk2(NodeHolder nodeHolder, List<ColumnSchema> columnSchema,
SegmentProperties segmentProperties, boolean isDimensionColumn) throws IOException {
List<byte[]> dataChunkBuffer = new ArrayList<>();
if (isDimensionColumn) {
@@ -945,10 +945,10 @@ public class CarbonMetadataUtil {
dataChunk.setNumberOfRowsInpage(nodeHolder.getEntryCount());
List<Encoding> encodings = new ArrayList<Encoding>();
dataChunk.setData_page_length(nodeHolder.getKeyLengths()[i]);
- if (containsEncoding(i, Encoding.DICTIONARY, columnSchenma, segmentProperties)) {
+ if (containsEncoding(i, Encoding.DICTIONARY, columnSchema, segmentProperties)) {
encodings.add(Encoding.DICTIONARY);
}
- if (containsEncoding(i, Encoding.DIRECT_DICTIONARY, columnSchenma, segmentProperties)) {
+ if (containsEncoding(i, Encoding.DIRECT_DICTIONARY, columnSchema, segmentProperties)) {
encodings.add(Encoding.DIRECT_DICTIONARY);
}
dataChunk.setRowMajor(nodeHolder.getColGrpBlocks()[i]);
[45/50] [abbrv] incubator-carbondata git commit: no sort datawriter
Posted by ja...@apache.org.
no sort datawriter
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/49cabeca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/49cabeca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/49cabeca
Branch: refs/heads/12-dev
Commit: 49cabeca67622486f0331757f3746728133d4f3d
Parents: 7dafae7
Author: QiangCai <qi...@qq.com>
Authored: Thu Apr 6 00:09:06 2017 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Tue May 2 09:42:20 2017 +0800
----------------------------------------------------------------------
.../processing/merger/CarbonDataMergerUtil.java | 2 +-
.../processing/model/CarbonLoadModel.java | 6 +-
.../newflow/DataLoadProcessBuilder.java | 38 ++-
.../CarbonRowDataWriterProcessorStepImpl.java | 321 +++++++++++++++++++
.../newflow/steps/NoSortProcessorStepImpl.java | 153 ---------
.../store/CarbonDataFileAttributes.java | 10 +-
.../store/CarbonFactDataHandlerModel.java | 2 +-
.../store/writer/AbstractFactDataWriter.java | 4 +-
8 files changed, 361 insertions(+), 175 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/49cabeca/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
index 65ec4fc..b9ea447 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
@@ -170,7 +170,7 @@ public final class CarbonDataMergerUtil {
boolean updateLockStatus = false;
boolean tableLockStatus = false;
- String timestamp = carbonLoadModel.getFactTimeStamp();
+ String timestamp = "" + carbonLoadModel.getFactTimeStamp();
List<String> updatedDeltaFilesList =
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/49cabeca/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
index d8f84bf..f63cb03 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
@@ -82,7 +82,7 @@ public class CarbonLoadModel implements Serializable {
/**
* new load start time
*/
- private String factTimeStamp;
+ private long factTimeStamp;
/**
* load Id
*/
@@ -588,7 +588,7 @@ public class CarbonLoadModel implements Serializable {
/**
* @return
*/
- public String getFactTimeStamp() {
+ public long getFactTimeStamp() {
return factTimeStamp;
}
@@ -596,7 +596,7 @@ public class CarbonLoadModel implements Serializable {
* @param factTimeStamp
*/
public void setFactTimeStamp(long factTimeStamp) {
- this.factTimeStamp = factTimeStamp + "";
+ this.factTimeStamp = factTimeStamp;
}
public String[] getDelimiters() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/49cabeca/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
index 11e5db1..de469f9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
@@ -35,12 +35,12 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.processing.model.CarbonLoadModel;
import org.apache.carbondata.processing.newflow.constants.DataLoadProcessorConstants;
+import org.apache.carbondata.processing.newflow.steps.CarbonRowDataWriterProcessorStepImpl;
import org.apache.carbondata.processing.newflow.steps.DataConverterProcessorStepImpl;
import org.apache.carbondata.processing.newflow.steps.DataConverterProcessorWithBucketingStepImpl;
import org.apache.carbondata.processing.newflow.steps.DataWriterBatchProcessorStepImpl;
import org.apache.carbondata.processing.newflow.steps.DataWriterProcessorStepImpl;
import org.apache.carbondata.processing.newflow.steps.InputProcessorStepImpl;
-import org.apache.carbondata.processing.newflow.steps.NoSortProcessorStepImpl;
import org.apache.carbondata.processing.newflow.steps.SortProcessorStepImpl;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
@@ -59,7 +59,9 @@ public final class DataLoadProcessBuilder {
CarbonCommonConstants.LOAD_USE_BATCH_SORT_DEFAULT));
CarbonDataLoadConfiguration configuration =
createConfiguration(loadModel, storeLocation);
- if (configuration.getBucketingInfo() != null) {
+ if (!configuration.isSortTable()) {
+ return buildInternalForNoSort(inputIterators, configuration);
+ } else if (configuration.getBucketingInfo() != null) {
return buildInternalForBucketing(inputIterators, configuration);
} else if (batchSort) {
return buildInternalForBatchSort(inputIterators, configuration);
@@ -77,16 +79,30 @@ public final class DataLoadProcessBuilder {
// data types and configurations.
AbstractDataLoadProcessorStep converterProcessorStep =
new DataConverterProcessorStepImpl(configuration, inputProcessorStep);
- // 3. Sorts the data by SortColumn or not
- AbstractDataLoadProcessorStep sortProcessorStep = configuration.isSortTable() ?
- new SortProcessorStepImpl(configuration, converterProcessorStep) :
- new NoSortProcessorStepImpl(configuration, converterProcessorStep);
+ // 3. Sorts the data by SortColumn
+ AbstractDataLoadProcessorStep sortProcessorStep =
+ new SortProcessorStepImpl(configuration, converterProcessorStep);
// 4. Writes the sorted data in carbondata format.
AbstractDataLoadProcessorStep writerProcessorStep =
new DataWriterProcessorStepImpl(configuration, sortProcessorStep);
return writerProcessorStep;
}
+ private AbstractDataLoadProcessorStep buildInternalForNoSort(CarbonIterator[] inputIterators,
+ CarbonDataLoadConfiguration configuration) {
+ // 1. Reads the data input iterators and parses the data.
+ AbstractDataLoadProcessorStep inputProcessorStep =
+ new InputProcessorStepImpl(configuration, inputIterators);
+ // 2. Converts the data like dictionary or non dictionary or complex objects depends on
+ // data types and configurations.
+ AbstractDataLoadProcessorStep converterProcessorStep =
+ new DataConverterProcessorStepImpl(configuration, inputProcessorStep);
+ // 3. Writes the sorted data in carbondata format.
+ AbstractDataLoadProcessorStep writerProcessorStep =
+ new CarbonRowDataWriterProcessorStepImpl(configuration, converterProcessorStep);
+ return writerProcessorStep;
+ }
+
private AbstractDataLoadProcessorStep buildInternalForBatchSort(CarbonIterator[] inputIterators,
CarbonDataLoadConfiguration configuration) {
// 1. Reads the data input iterators and parses the data.
@@ -97,9 +113,8 @@ public final class DataLoadProcessBuilder {
AbstractDataLoadProcessorStep converterProcessorStep =
new DataConverterProcessorStepImpl(configuration, inputProcessorStep);
// 3. Sorts the data by SortColumn or not
- AbstractDataLoadProcessorStep sortProcessorStep = configuration.isSortTable() ?
- new SortProcessorStepImpl(configuration, converterProcessorStep) :
- new NoSortProcessorStepImpl(configuration, converterProcessorStep);
+ AbstractDataLoadProcessorStep sortProcessorStep =
+ new SortProcessorStepImpl(configuration, converterProcessorStep);
// 4. Writes the sorted data in carbondata format.
AbstractDataLoadProcessorStep writerProcessorStep =
new DataWriterBatchProcessorStepImpl(configuration, sortProcessorStep);
@@ -116,9 +131,8 @@ public final class DataLoadProcessBuilder {
AbstractDataLoadProcessorStep converterProcessorStep =
new DataConverterProcessorWithBucketingStepImpl(configuration, inputProcessorStep);
// 3. Sorts the data by SortColumn or not
- AbstractDataLoadProcessorStep sortProcessorStep = configuration.isSortTable() ?
- new SortProcessorStepImpl(configuration, converterProcessorStep) :
- new NoSortProcessorStepImpl(configuration, converterProcessorStep);
+ AbstractDataLoadProcessorStep sortProcessorStep =
+ new SortProcessorStepImpl(configuration, converterProcessorStep);
// 4. Writes the sorted data in carbondata format.
AbstractDataLoadProcessorStep writerProcessorStep =
new DataWriterProcessorStepImpl(configuration, sortProcessorStep);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/49cabeca/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
new file mode 100644
index 0000000..929a09e
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.processing.newflow.steps;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.Iterator;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.keygenerator.KeyGenerator;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.processing.newflow.AbstractDataLoadProcessorStep;
+import org.apache.carbondata.processing.newflow.CarbonDataLoadConfiguration;
+import org.apache.carbondata.processing.newflow.DataField;
+import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException;
+import org.apache.carbondata.processing.newflow.row.CarbonRow;
+import org.apache.carbondata.processing.newflow.row.CarbonRowBatch;
+import org.apache.carbondata.processing.store.CarbonFactDataHandlerModel;
+import org.apache.carbondata.processing.store.CarbonFactHandler;
+import org.apache.carbondata.processing.store.CarbonFactHandlerFactory;
+import org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException;
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
+
+/**
+ * It reads data from sorted files which are generated in previous sort step.
+ * And it writes data to carbondata file. It also generates mdk key while writing to carbondata file
+ */
+public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CarbonRowDataWriterProcessorStepImpl.class.getName());
+
+ private SegmentProperties segmentProperties;
+
+ private KeyGenerator keyGenerator;
+
+ private int dimensionWithComplexCount;
+
+ private int noDictWithComplextCount;
+
+ private boolean[] isNoDictionaryDimensionColumn;
+
+ private char[] aggType;
+
+ private int dimensionCount;
+
+ private int measureCount;
+
+ private long[] readCounter;
+
+ private long[] writeCounter;
+
+ private int outputLength;
+
+ private CarbonTableIdentifier tableIdentifier;
+
+ private String tableName;
+
+ public CarbonRowDataWriterProcessorStepImpl(CarbonDataLoadConfiguration configuration,
+ AbstractDataLoadProcessorStep child) {
+ super(configuration, child);
+ }
+
+ @Override public DataField[] getOutput() {
+ return child.getOutput();
+ }
+
+ @Override public void initialize() throws IOException {
+ child.initialize();
+ }
+
+ private String getStoreLocation(CarbonTableIdentifier tableIdentifier, String partitionId) {
+ String storeLocation = CarbonDataProcessorUtil
+ .getLocalDataFolderLocation(tableIdentifier.getDatabaseName(),
+ tableIdentifier.getTableName(), String.valueOf(configuration.getTaskNo()), partitionId,
+ configuration.getSegmentId() + "", false);
+ new File(storeLocation).mkdirs();
+ return storeLocation;
+ }
+
+ @Override public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
+ final Iterator<CarbonRowBatch>[] iterators = child.execute();
+ tableIdentifier = configuration.getTableIdentifier().getCarbonTableIdentifier();
+ tableName = tableIdentifier.getTableName();
+ try {
+ readCounter = new long[iterators.length];
+ writeCounter = new long[iterators.length];
+ dimensionWithComplexCount = configuration.getDimensionCount();
+ noDictWithComplextCount =
+ configuration.getNoDictionaryCount() + configuration.getComplexDimensionCount();
+ dimensionCount = configuration.getDimensionCount() - noDictWithComplextCount;
+ isNoDictionaryDimensionColumn =
+ CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields());
+ aggType = CarbonDataProcessorUtil
+ .getAggType(configuration.getMeasureCount(), configuration.getMeasureFields());
+
+ CarbonFactDataHandlerModel dataHandlerModel = CarbonFactDataHandlerModel
+ .createCarbonFactDataHandlerModel(configuration,
+ getStoreLocation(tableIdentifier, String.valueOf(0)), 0, 0);
+ measureCount = dataHandlerModel.getMeasureCount();
+ outputLength = measureCount + (this.noDictWithComplextCount > 0 ? 1 : 0) + 1;
+ segmentProperties = dataHandlerModel.getSegmentProperties();
+ keyGenerator = segmentProperties.getDimensionKeyGenerator();
+
+ CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
+ .recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(),
+ System.currentTimeMillis());
+
+ if (iterators.length == 1) {
+ doExecute(iterators[0], 0, 0);
+ } else {
+ ExecutorService executorService = Executors.newFixedThreadPool(iterators.length);
+ Future[] futures = new Future[iterators.length];
+ for (int i = 0; i < iterators.length; i++) {
+ futures[i] = executorService.submit(new DataWriterRunnable(iterators[i], i));
+ }
+ for (Future future : futures) {
+ future.get();
+ }
+ }
+ } catch (CarbonDataWriterException e) {
+ LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterProcessorStepImpl");
+ throw new CarbonDataLoadingException(
+ "Error while initializing data handler : " + e.getMessage());
+ } catch (Exception e) {
+ LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterProcessorStepImpl");
+ throw new CarbonDataLoadingException("There is an unexpected error: " + e.getMessage(), e);
+ }
+ return null;
+ }
+
+ private void doExecute(Iterator<CarbonRowBatch> iterator, int partitionId, int iteratorIndex) {
+ String storeLocation = getStoreLocation(tableIdentifier, String.valueOf(partitionId));
+ CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel
+ .createCarbonFactDataHandlerModel(configuration, storeLocation, partitionId, 0);
+ model.getCarbonDataFileAttributes()
+ .setFactTimeStamp(model.getCarbonDataFileAttributes().getFactTimeStamp() + iteratorIndex);
+ CarbonFactHandler dataHandler = null;
+ boolean rowsNotExist = true;
+ while (iterator.hasNext()) {
+ if (rowsNotExist) {
+ rowsNotExist = false;
+ dataHandler = CarbonFactHandlerFactory
+ .createCarbonFactHandler(model, CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
+ dataHandler.initialise();
+ }
+ processBatch(iterator.next(), dataHandler, iteratorIndex);
+ }
+ if (!rowsNotExist) {
+ finish(dataHandler, iteratorIndex);
+ }
+ }
+
+ @Override protected String getStepName() {
+ return "Data Writer";
+ }
+
+ private void finish(CarbonFactHandler dataHandler, int iteratorIndex) {
+ try {
+ dataHandler.finish();
+ } catch (Exception e) {
+ LOGGER.error(e, "Failed for table: " + tableName + " in finishing data handler");
+ }
+ LOGGER.info("Record Processed For table: " + tableName);
+ String logMessage =
+ "Finished Carbon DataWriterProcessorStepImpl: Read: " + readCounter[iteratorIndex]
+ + ": Write: " + readCounter[iteratorIndex];
+ LOGGER.info(logMessage);
+ CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordTotalRecords(rowCounter.get());
+ processingComplete(dataHandler);
+ CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
+ .recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(),
+ System.currentTimeMillis());
+ CarbonTimeStatisticsFactory.getLoadStatisticsInstance()
+ .recordMdkGenerateTotalTime(configuration.getPartitionId(), System.currentTimeMillis());
+ }
+
+ private void processingComplete(CarbonFactHandler dataHandler) throws CarbonDataLoadingException {
+ if (null != dataHandler) {
+ try {
+ dataHandler.closeHandler();
+ } catch (CarbonDataWriterException e) {
+ LOGGER.error(e, e.getMessage());
+ throw new CarbonDataLoadingException(e.getMessage());
+ } catch (Exception e) {
+ LOGGER.error(e, e.getMessage());
+ throw new CarbonDataLoadingException("There is an unexpected error: " + e.getMessage());
+ }
+ }
+ }
+
+ /**
+ * convert input CarbonRow to output CarbonRow
+ * e.g. There is a table as following,
+ * the number of dictionary dimensions is a,
+ * the number of no-dictionary dimensions is b,
+ * the number of complex dimensions is c,
+ * the number of measures is d.
+ * input CarbonRow format: the length of Object[] data is a+b+c+d, the number of all columns.
+ * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ * | Part | Object item | describe |
+ * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ * | Object[0 ~ a+b-1] | Integer, byte[], Integer, ... | dict + no dict dimensions|
+ * ----------------------------------------------------------------------------------------
+ * | Object[a+b ~ a+b+c-1] | byte[], byte[], ... | complex dimensions |
+ * ----------------------------------------------------------------------------------------
+ * | Object[a+b+c ~ a+b+c+d-1]| int, byte[], ... | measures |
+ * ----------------------------------------------------------------------------------------
+ * output CarbonRow format: the length of object[] data is d + (b+c>0?1:0) + 1.
+ * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ * | Part | Object item | describe |
+ * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ * | Object[0 ~ d-1] | int, byte[], ... | measures |
+ * ----------------------------------------------------------------------------------------
+ * | Object[d] | byte[b+c][] | no dict + complex dim |
+ * ----------------------------------------------------------------------------------------
+ * | Object[d+1] | byte[] | mdkey |
+ * ----------------------------------------------------------------------------------------
+ *
+ * @param row
+ * @return
+ */
+ private Object[] convertRow(CarbonRow row) throws KeyGenException {
+ int dictIndex = 0;
+ int nonDicIndex = 0;
+ int[] dim = new int[this.dimensionCount];
+ byte[][] nonDicArray = new byte[this.noDictWithComplextCount][];
+ // read dimension values
+ int dimCount = 0;
+ for (; dimCount < isNoDictionaryDimensionColumn.length; dimCount++) {
+ if (isNoDictionaryDimensionColumn[dimCount]) {
+ nonDicArray[nonDicIndex++] = (byte[]) row.getObject(dimCount);
+ } else {
+ dim[dictIndex++] = (int) row.getObject(dimCount);
+ }
+ }
+
+ for (; dimCount < this.dimensionWithComplexCount; dimCount++) {
+ nonDicArray[nonDicIndex++] = (byte[]) row.getObject(dimCount);
+ }
+
+ int l = 0;
+ Object[] outputRow = new Object[outputLength];
+ for (; l < this.measureCount; l++) {
+ Object value = row.getObject(l + this.dimensionWithComplexCount);
+ if (null != value) {
+ if (aggType[l] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
+ BigDecimal val = (BigDecimal) value;
+ outputRow[l] = DataTypeUtil.bigDecimalToByte(val);
+ } else {
+ outputRow[l] = value;
+ }
+ } else {
+ outputRow[l] = null;
+ }
+ }
+
+ if (this.noDictWithComplextCount > 0) {
+ outputRow[l++] = nonDicArray;
+ }
+ outputRow[l] = keyGenerator.generateKey(dim);
+ return outputRow;
+ }
+
+ private void processBatch(CarbonRowBatch batch, CarbonFactHandler dataHandler, int iteratorIndex)
+ throws CarbonDataLoadingException {
+ try {
+ while (batch.hasNext()) {
+ dataHandler.addDataToStore(convertRow(batch.next()));
+ readCounter[iteratorIndex]++;
+ }
+ writeCounter[iteratorIndex] += batch.getSize();
+ } catch (Exception e) {
+ throw new CarbonDataLoadingException("unable to generate the mdkey", e);
+ }
+ rowCounter.getAndAdd(batch.getSize());
+ }
+
+ @Override protected CarbonRow processRow(CarbonRow row) {
+ return null;
+ }
+
+ class DataWriterRunnable implements Runnable {
+
+ private Iterator<CarbonRowBatch> iterator;
+ private int iteratorIndex = 0;
+
+ DataWriterRunnable(Iterator<CarbonRowBatch> iterator, int iteratorIndex) {
+ this.iterator = iterator;
+ this.iteratorIndex = iteratorIndex;
+ }
+
+ @Override public void run() {
+ doExecute(this.iterator, 0, iteratorIndex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/49cabeca/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java
deleted file mode 100644
index bde89ed..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.processing.newflow.steps;
-
-import java.io.IOException;
-import java.math.BigDecimal;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.processing.newflow.AbstractDataLoadProcessorStep;
-import org.apache.carbondata.processing.newflow.CarbonDataLoadConfiguration;
-import org.apache.carbondata.processing.newflow.DataField;
-import org.apache.carbondata.processing.newflow.row.CarbonRow;
-import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
-import org.apache.carbondata.processing.util.NonDictionaryUtil;
-
-/**
- * if the table doesn't have sort_columns, just convert row format.
- */
-public class NoSortProcessorStepImpl extends AbstractDataLoadProcessorStep {
-
- private int dimensionCount;
-
- private int dimensionWithComplexCount;
-
- private int noDictCount;
-
- private int measureCount;
-
- private boolean[] isNoDictionaryDimensionColumn;
-
- private char[] aggType;
-
- public NoSortProcessorStepImpl(CarbonDataLoadConfiguration configuration,
- AbstractDataLoadProcessorStep child) {
- super(configuration, child);
- this.dimensionWithComplexCount = configuration.getDimensionCount();
- this.noDictCount =
- configuration.getNoDictionaryCount() + configuration.getComplexDimensionCount();
- this.dimensionCount = configuration.getDimensionCount() - this.noDictCount;
- this.measureCount = configuration.getMeasureCount();
- this.isNoDictionaryDimensionColumn =
- CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields());
- this.aggType = CarbonDataProcessorUtil
- .getAggType(configuration.getMeasureCount(), configuration.getMeasureFields());
- }
-
- @Override public DataField[] getOutput() {
- return child.getOutput();
- }
-
- @Override public void initialize() throws IOException {
- child.initialize();
- }
-
- /**
- * convert input CarbonRow to output CarbonRow
- * e.g. There is a table as following,
- * the number of dictionary dimensions is a,
- * the number of no-dictionary dimensions is b,
- * the number of complex dimensions is c,
- * the number of measures is d.
- * input CarbonRow format: the length of Object[] data is a+b+c+d, the number of all columns.
- * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
- * | Part | Object item | describe |
- * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
- * | Object[0 ~ a+b-1] | Integer, byte[], Integer, ... | dict + no dict dimensions|
- * ----------------------------------------------------------------------------------------
- * | Object[a+b ~ a+b+c-1] | byte[], byte[], ... | complex dimensions |
- * ----------------------------------------------------------------------------------------
- * | Object[a+b+c ~ a+b+c+d-1]| int, byte[], ... | measures |
- * ----------------------------------------------------------------------------------------
- * output CarbonRow format: the length of object[] data is 3.
- * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
- * | Part | Object item | describe |
- * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
- * | Object[0] | int[a] | dict dimension array |
- * ----------------------------------------------------------------------------------------
- * | Object[1] | byte[b+c][] | no dict + complex dim |
- * ----------------------------------------------------------------------------------------
- * | Object[2] | Object[d] | measures |
- * ----------------------------------------------------------------------------------------
- * @param row
- * @return
- */
- @Override protected CarbonRow processRow(CarbonRow row) {
- int dictIndex = 0;
- int nonDicIndex = 0;
- int[] dim = new int[this.dimensionCount];
- byte[][] nonDicArray = new byte[this.noDictCount][];
- Object[] measures = new Object[this.measureCount];
- // read dimension values
- int dimCount = 0;
- for (; dimCount < isNoDictionaryDimensionColumn.length; dimCount++) {
- if (isNoDictionaryDimensionColumn[dimCount]) {
- nonDicArray[nonDicIndex++] = (byte[]) row.getObject(dimCount);
- } else {
- dim[dictIndex++] = (int) row.getObject(dimCount);
- }
- }
-
- for (; dimCount < this.dimensionWithComplexCount; dimCount++) {
- nonDicArray[nonDicIndex++] = (byte[]) row.getObject(dimCount);
- }
-
- // measure values
- for (int mesCount = 0; mesCount < this.measureCount; mesCount++) {
- Object value = row.getObject(mesCount + this.dimensionWithComplexCount);
- if (null != value) {
- if (aggType[mesCount] == CarbonCommonConstants.DOUBLE_MEASURE) {
- measures[mesCount] = value;
- } else if (aggType[mesCount] == CarbonCommonConstants.BIG_INT_MEASURE) {
- measures[mesCount] = value;
- } else if (aggType[mesCount] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
- BigDecimal val = (BigDecimal) value;
- measures[mesCount] = DataTypeUtil.bigDecimalToByte(val);
- }
- } else {
- measures[mesCount] = null;
- }
- }
- // create new row of size 3 (1 for dims , 1 for high card , 1 for measures)
- Object[] holder = new Object[3];
- NonDictionaryUtil.prepareOutObj(holder, dim, nonDicArray, measures);
- //return out row
- return new CarbonRow(holder);
- }
-
- @Override
- public void close() {
- if (!closed) {
- super.close();
- }
- }
-
- @Override protected String getStepName() {
- return "No Sort Processor";
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/49cabeca/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataFileAttributes.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataFileAttributes.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataFileAttributes.java
index 188c468..0b606b0 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataFileAttributes.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataFileAttributes.java
@@ -39,13 +39,13 @@ public class CarbonDataFileAttributes {
/**
* load start time
*/
- private String factTimeStamp;
+ private long factTimeStamp;
/**
* @param taskId
* @param factTimeStamp
*/
- public CarbonDataFileAttributes(int taskId, String factTimeStamp) {
+ public CarbonDataFileAttributes(int taskId, long factTimeStamp) {
this.taskId = taskId;
this.factTimeStamp = factTimeStamp;
}
@@ -57,10 +57,14 @@ public class CarbonDataFileAttributes {
return taskId;
}
+ public void setFactTimeStamp(long factTimeStamp) {
+ this.factTimeStamp = factTimeStamp;
+ }
+
/**
* @return fact time stamp which is load start time
*/
- public String getFactTimeStamp() {
+ public long getFactTimeStamp() {
return factTimeStamp;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/49cabeca/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index d3b5605..61322bc 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -265,7 +265,7 @@ public class CarbonFactDataHandlerModel {
CarbonDataFileAttributes carbonDataFileAttributes =
new CarbonDataFileAttributes(Integer.parseInt(configuration.getTaskNo()),
- (String) configuration.getDataLoadProperty(DataLoadProcessorConstants.FACT_TIME_STAMP));
+ (Long) configuration.getDataLoadProperty(DataLoadProcessorConstants.FACT_TIME_STAMP));
String carbonDataDirectoryPath = getCarbonDataFolderLocation(configuration);
CarbonFactDataHandlerModel carbonFactDataHandlerModel = new CarbonFactDataHandlerModel();
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/49cabeca/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index cda907c..770d24c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -296,7 +296,7 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
this.carbonDataFileName = carbonTablePath
.getCarbonDataFileName(fileCount, dataWriterVo.getCarbonDataFileAttributes().getTaskId(),
dataWriterVo.getBucketNumber(), dataWriterVo.getTaskExtension(),
- dataWriterVo.getCarbonDataFileAttributes().getFactTimeStamp());
+ "" + dataWriterVo.getCarbonDataFileAttributes().getFactTimeStamp());
String actualFileNameVal = carbonDataFileName + CarbonCommonConstants.FILE_INPROGRESS_STATUS;
FileData fileData = new FileData(actualFileNameVal, dataWriterVo.getStoreLocation());
dataWriterVo.getFileManager().add(fileData);
@@ -447,7 +447,7 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
String fileName = dataWriterVo.getStoreLocation() + File.separator + carbonTablePath
.getCarbonIndexFileName(dataWriterVo.getCarbonDataFileAttributes().getTaskId(),
dataWriterVo.getBucketNumber(), dataWriterVo.getTaskExtension(),
- dataWriterVo.getCarbonDataFileAttributes().getFactTimeStamp());
+ "" + dataWriterVo.getCarbonDataFileAttributes().getFactTimeStamp());
CarbonIndexFileWriter writer = new CarbonIndexFileWriter();
// open file
writer.openThriftWriter(fileName);
[12/50] [abbrv] incubator-carbondata git commit: [CARBONDATA-981] Get
HIVE_CONNECTION_URL from sparkContext.hadoopConfiguration. This closes #844
Posted by ja...@apache.org.
[CARBONDATA-981] Get HIVE_CONNECTION_URL from sparkContext.hadoopConfiguration. This closes #844
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/373f4e79
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/373f4e79
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/373f4e79
Branch: refs/heads/12-dev
Commit: 373f4e79559971cbe2beb7ecb570b2d3306847c4
Parents: d695a8a 075e74e
Author: ravipesala <ra...@gmail.com>
Authored: Wed Apr 26 11:24:04 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Apr 26 11:24:04 2017 +0530
----------------------------------------------------------------------
.../scala/org/apache/spark/sql/CarbonSession.scala | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
[30/50] [abbrv] incubator-carbondata git commit: [CARBONDATA-781]
Reuse SegmentProperties object when schema is the same in table level to
reduce the memory This closes #659
Posted by ja...@apache.org.
[CARBONDATA-781] Reuse SegmentProperties object when schema is the same in table level to reduce the memory This closes #659
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/b728b8f9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/b728b8f9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/b728b8f9
Branch: refs/heads/12-dev
Commit: b728b8f9408e868d782c0b26bf499fea76b7f7e8
Parents: 58be73f 33a3858
Author: jackylk <ja...@huawei.com>
Authored: Thu Apr 27 22:28:17 2017 -0700
Committer: jackylk <ja...@huawei.com>
Committed: Thu Apr 27 22:28:17 2017 -0700
----------------------------------------------------------------------
.../core/datastore/SegmentTaskIndexStore.java | 61 +++++++++++++++++++-
.../core/datastore/block/SegmentTaskIndex.java | 10 ++--
.../schema/table/column/ColumnSchema.java | 10 +++-
.../datastore/block/SegmentTaskIndexTest.java | 4 +-
4 files changed, 76 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
[03/50] [abbrv] incubator-carbondata git commit: [CARBONDATA-974]
Improve Carbon index file loading file in case of big cluster. This closes
#838
Posted by ja...@apache.org.
[CARBONDATA-974] Improve Carbon index file loading file in case of big cluster. This closes #838
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/0a11c2ad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/0a11c2ad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/0a11c2ad
Branch: refs/heads/12-dev
Commit: 0a11c2ade62ba983eedcae9efb3a38678238e108
Parents: 174bfad 8fc9c7b
Author: ravipesala <ra...@gmail.com>
Authored: Sat Apr 22 00:31:07 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Sat Apr 22 00:31:07 2017 +0530
----------------------------------------------------------------------
.../apache/carbondata/core/util/CarbonUtil.java | 8 +++--
.../core/util/path/CarbonTablePath.java | 33 ++++++++++++++++++++
2 files changed, 39 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[33/50] [abbrv] incubator-carbondata git commit: Cast Push Down
Optimization
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96e5b5cc/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
new file mode 100644
index 0000000..eb8a7a0
--- /dev/null
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, ExprId, LeafExpression, NamedExpression}
+import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
+import org.apache.spark.sql.sources.Filter
+import org.apache.spark.sql.types.DataType
+
+import org.apache.carbondata.core.scan.expression.ColumnExpression
+
+case class CastExpr(expr: Expression) extends Filter {
+ override def references: Array[String] = null
+}
+
+case class CarbonBoundReference(colExp: ColumnExpression, dataType: DataType, nullable: Boolean)
+ extends LeafExpression with NamedExpression with CodegenFallback {
+
+ type EvaluatedType = Any
+
+ override def toString: String = s"input[" + colExp.getColIndex + "]"
+
+ override def eval(input: InternalRow): Any = input.get(colExp.getColIndex, dataType)
+
+ override def name: String = colExp.getColumnName
+
+ override def toAttribute: Attribute = throw new UnsupportedOperationException
+
+ override def exprId: ExprId = throw new UnsupportedOperationException
+
+ override def qualifier: Option[String] = null
+
+ override def newInstance(): NamedExpression = throw new UnsupportedOperationException
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96e5b5cc/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala
index 85046cf..0bcbcfc 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala
@@ -46,17 +46,9 @@ case class CarbonScan(
def processFilterExpressions(plan: CarbonQueryPlan) {
if (dimensionPredicatesRaw.nonEmpty) {
- val expressionVal = CarbonFilters.processExpression(
- dimensionPredicatesRaw,
- attributesNeedToDecode,
- unprocessedExprs,
- carbonTable)
- expressionVal match {
- case Some(ce) =>
- // adding dimension used in expression in querystats
- plan.setFilterExpression(ce)
- case _ =>
- }
+ val exps = CarbonFilters.preProcessExpressions(dimensionPredicatesRaw)
+ val expressionVal = CarbonFilters.transformExpression(exps.head)
+ plan.setFilterExpression(expressionVal)
}
processExtraAttributes(plan)
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96e5b5cc/integration/spark2/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
index 398a0b7..943c2da 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
@@ -15,12 +15,13 @@
* limitations under the License.
*/
-package org.apache.spark.sql
+package org.apache.carbondata.spark
import java.util.{ArrayList, List}
import scala.collection.JavaConverters._
+import org.apache.spark.sql.CarbonBoundReference
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.{Expression => SparkExpression, GenericInternalRow}
@@ -108,14 +109,30 @@ class SparkUnknownExpression(var sparkExp: SparkExpression)
}
def getColumnListFromExpressionTree(sparkCurrentExp: SparkExpression,
- list: java.util.List[ColumnExpression]): Unit = {
- sparkCurrentExp.children.foreach(getColumnListFromExpressionTree(_, list))
+ lst: java.util.List[ColumnExpression]): Unit = {
+ sparkCurrentExp match {
+ case carbonBoundRef: CarbonBoundReference =>
+ val foundExp = lst.asScala
+ .find(p => p.getColumnName() == carbonBoundRef.colExp.getColumnName())
+ if (foundExp.isEmpty) {
+ carbonBoundRef.colExp.setColIndex(lst.size)
+ lst.add(carbonBoundRef.colExp)
+ } else {
+ carbonBoundRef.colExp.setColIndex(foundExp.get.getColIndex())
+ }
+ case _ => sparkCurrentExp.children.foreach(getColumnListFromExpressionTree(_, lst))
+ }
}
+
+
def getAllColumnListFromExpressionTree(sparkCurrentExp: SparkExpression,
list: List[ColumnExpression]): List[ColumnExpression] = {
- sparkCurrentExp.children.foreach(getColumnListFromExpressionTree(_, list))
+ sparkCurrentExp match {
+ case carbonBoundRef: CarbonBoundReference => list.add(carbonBoundRef.colExp)
+ case _ => sparkCurrentExp.children.foreach(getColumnListFromExpressionTree(_, list))
+ }
list
}
@@ -128,4 +145,6 @@ class SparkUnknownExpression(var sparkExp: SparkExpression)
false
}
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96e5b5cc/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala
index 39fbb09..595c173 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala
@@ -17,6 +17,9 @@
package org.apache.spark.sql.execution
+import java.text.SimpleDateFormat
+import java.util.Date
+
import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
@@ -31,13 +34,14 @@ import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partition
import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.sql.optimizer.CarbonDecoderRelation
import org.apache.spark.sql.sources.{BaseRelation, Filter}
-import org.apache.spark.sql.types.{AtomicType, IntegerType}
+import org.apache.spark.sql.types.{AtomicType, DoubleType, IntegerType, StringType, TimestampType}
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampDirectDictionaryGenerator
import org.apache.carbondata.core.metadata.schema.BucketingInfo
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.spark.CarbonAliasDecoderRelation
+import org.apache.carbondata.spark.{CarbonAliasDecoderRelation}
import org.apache.carbondata.spark.rdd.CarbonScanRDD
import org.apache.carbondata.spark.util.CarbonScalaUtil
@@ -351,7 +355,6 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
}
}
-
protected[sql] def selectFilters(
relation: BaseRelation,
predicates: Seq[Expression]): (Seq[Expression], Seq[Filter]) = {
@@ -397,8 +400,10 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
(unrecognizedPredicates ++ unhandledPredicates, translatedFilters)
}
+
/**
* Tries to translate a Catalyst [[Expression]] into data source [[Filter]].
+ *
* @return a `Some[Filter]` if the input [[Expression]] is convertible, otherwise a `None`.
*/
protected[sql] def translateFilter(predicate: Expression, or: Boolean = false): Option[Filter] = {
@@ -425,15 +430,22 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
} else {
(translateFilter(left) ++ translateFilter(right)).reduceOption(sources.And)
}
-
case EqualTo(a: Attribute, Literal(v, t)) =>
Some(sources.EqualTo(a.name, v))
case EqualTo(l@Literal(v, t), a: Attribute) =>
Some(sources.EqualTo(a.name, v))
+ case c@EqualTo(Cast(a: Attribute, _), Literal(v, t)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case c@EqualTo(Literal(v, t), Cast(a: Attribute, _)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
case Not(EqualTo(a: Attribute, Literal(v, t))) =>
Some(sources.Not(sources.EqualTo(a.name, v)))
case Not(EqualTo(Literal(v, t), a: Attribute)) =>
Some(sources.Not(sources.EqualTo(a.name, v)))
+ case c@Not(EqualTo(Cast(a: Attribute, _), Literal(v, t))) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case c@Not(EqualTo(Literal(v, t), Cast(a: Attribute, _))) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
case IsNotNull(a: Attribute) => Some(sources.IsNotNull(a.name))
case IsNull(a: Attribute) => Some(sources.IsNull(a.name))
case Not(In(a: Attribute, list)) if !list.exists(!_.isInstanceOf[Literal]) =>
@@ -442,22 +454,43 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
case In(a: Attribute, list) if !list.exists(!_.isInstanceOf[Literal]) =>
val hSet = list.map(e => e.eval(EmptyRow))
Some(sources.In(a.name, hSet.toArray))
+ case c@Not(In(Cast(a: Attribute, _), list))
+ if !list.exists(!_.isInstanceOf[Literal]) =>
+ Some(CastExpr(c))
+ case c@In(Cast(a: Attribute, _), list) if !list.exists(!_.isInstanceOf[Literal]) =>
+ Some(CastExpr(c))
case GreaterThan(a: Attribute, Literal(v, t)) =>
Some(sources.GreaterThan(a.name, v))
case GreaterThan(Literal(v, t), a: Attribute) =>
Some(sources.LessThan(a.name, v))
+ case c@GreaterThan(Cast(a: Attribute, _), Literal(v, t)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case c@GreaterThan(Literal(v, t), Cast(a: Attribute, _)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
case LessThan(a: Attribute, Literal(v, t)) =>
Some(sources.LessThan(a.name, v))
case LessThan(Literal(v, t), a: Attribute) =>
Some(sources.GreaterThan(a.name, v))
+ case c@LessThan(Cast(a: Attribute, _), Literal(v, t)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case c@LessThan(Literal(v, t), Cast(a: Attribute, _)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
case GreaterThanOrEqual(a: Attribute, Literal(v, t)) =>
Some(sources.GreaterThanOrEqual(a.name, v))
case GreaterThanOrEqual(Literal(v, t), a: Attribute) =>
Some(sources.LessThanOrEqual(a.name, v))
+ case c@GreaterThanOrEqual(Cast(a: Attribute, _), Literal(v, t)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case c@GreaterThanOrEqual(Literal(v, t), Cast(a: Attribute, _)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
case LessThanOrEqual(a: Attribute, Literal(v, t)) =>
Some(sources.LessThanOrEqual(a.name, v))
case LessThanOrEqual(Literal(v, t), a: Attribute) =>
Some(sources.GreaterThanOrEqual(a.name, v))
+ case c@LessThanOrEqual(Cast(a: Attribute, _), Literal(v, t)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case c@LessThanOrEqual(Literal(v, t), Cast(a: Attribute, _)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
case others => None
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96e5b5cc/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CastExpressionOptimization.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CastExpressionOptimization.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CastExpressionOptimization.scala
new file mode 100644
index 0000000..71a1032
--- /dev/null
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CastExpressionOptimization.scala
@@ -0,0 +1,373 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution
+
+import java.text.{ParseException, SimpleDateFormat}
+import java.util
+import java.util.{Locale, TimeZone}
+
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, EmptyRow, EqualTo, Expression, GreaterThan, GreaterThanOrEqual, In, LessThan, LessThanOrEqual, Literal, Not}
+import org.apache.spark.sql.CastExpr
+import org.apache.spark.sql.sources
+import org.apache.spark.sql.types.{DoubleType, IntegerType, StringType, TimestampType}
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+object CastExpressionOptimization {
+
+
+ def typeCastStringToLong(v: Any): Any = {
+ val parser: SimpleDateFormat = new SimpleDateFormat(CarbonProperties.getInstance
+ .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT))
+ try {
+ val value = parser.parse(v.toString).getTime() * 1000L
+ value
+ } catch {
+ case e: ParseException =>
+ try {
+ val parsenew: SimpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSz")
+ parsenew.parse(v.toString).getTime() * 1000L
+ } catch {
+ case e: ParseException =>
+ val gmtDay = new SimpleDateFormat("yyyy-MM-dd", Locale.US)
+ gmtDay.setTimeZone(TimeZone.getTimeZone("GMT"))
+ try {
+ gmtDay.parse(v.toString).getTime()
+ } catch {
+ case e: ParseException =>
+ v
+ case e: Exception =>
+ v
+ }
+ case e: Exception =>
+ v
+ }
+ case e: Exception =>
+ v
+ }
+ }
+
+ def typeCastStringToLongList(list: Seq[Expression]): Seq[Expression] = {
+ val tempList = new util.ArrayList[Expression]()
+ list.foreach { value =>
+ val output = typeCastStringToLong(value)
+ if (!output.equals(value)) {
+ tempList.add(output.asInstanceOf[Expression])
+ }
+ }
+ if (tempList.size() != list.size) {
+ list
+ } else {
+ tempList.asScala
+ }
+ }
+
+ def typeCastDoubleToIntList(list: Seq[Expression]): Seq[Expression] = {
+ val tempList = new util.ArrayList[Expression]()
+ list.foreach { value =>
+ val output = value.asInstanceOf[Double].toInt
+ if (value.asInstanceOf[Double].toInt.equals(output)) {
+ tempList.add(output.asInstanceOf[Expression])
+ }
+ }
+ if (tempList.size() != list.size) {
+ list
+ } else {
+ tempList.asScala
+ }
+ }
+
+ /**
+ * This routines tries to apply rules on Cast Filter Predicates and if the rules applied and the
+ * values can be toss back to native datatypes the cast is removed. Current two rules are applied
+ * a) Left : timestamp column Right : String Value
+ * Input from Spark : cast (col as string) <> 'String Literal'
+ * Change to : Column <> 'Long value of Timestamp String'
+ *
+ * b) Left : Integer Column Right : String Value
+ * Input from Spark : cast (col as double) <> 'Double Literal'
+ * Change to : Column <> 'Int value'
+ *
+ * @param expr
+ * @return
+ */
+ def checkIfCastCanBeRemove(expr: Expression): Option[sources.Filter] = {
+ expr match {
+ case c@EqualTo(Cast(a: Attribute, _), Literal(v, t)) =>
+ a.dataType match {
+ case t: TimestampType if t.sameType(StringType) =>
+ val value = typeCastStringToLong(v)
+ if (!value.equals(v)) {
+ Some(sources.EqualTo(a.name, value))
+ } else {
+ Some(CastExpr(c))
+ }
+ case i: IntegerType if t.sameType(DoubleType) =>
+ val value = v.asInstanceOf[Double].toInt
+ if (value.toDouble.equals(v)) {
+ Some(sources.EqualTo(a.name, value))
+ } else {
+ Some(CastExpr(c))
+ }
+ case _ => Some(CastExpr(c))
+ }
+ case c@EqualTo(Literal(v, t), Cast(a: Attribute, _)) =>
+ a.dataType match {
+ case t: TimestampType if t.sameType(StringType) =>
+ val value = typeCastStringToLong(v)
+ if (!value.equals(v)) {
+ Some(sources.EqualTo(a.name, value))
+ } else {
+ Some(CastExpr(c))
+ }
+ case i: IntegerType if t.sameType(DoubleType) =>
+ val value = v.asInstanceOf[Double].toInt
+ if (value.toDouble.equals(v)) {
+ Some(sources.EqualTo(a.name, value))
+ } else {
+ Some(CastExpr(c))
+ }
+ case _ => Some(CastExpr(c))
+ }
+ case c@Not(EqualTo(Cast(a: Attribute, _), Literal(v, t))) =>
+ a.dataType match {
+ case t: TimestampType if t.sameType(StringType) =>
+ val value = typeCastStringToLong(v)
+ if (!value.equals(v)) {
+ Some(sources.Not(sources.EqualTo(a.name, value)))
+ } else {
+ Some(CastExpr(c))
+ }
+ case i: IntegerType if t.sameType(DoubleType) =>
+ val value = v.asInstanceOf[Double].toInt
+ if (value.toDouble.equals(v)) {
+ Some(sources.Not(sources.EqualTo(a.name, value)))
+ } else {
+ Some(CastExpr(c))
+ }
+ case _ => Some(CastExpr(c))
+ }
+ case c@Not(EqualTo(Literal(v, t), Cast(a: Attribute, _))) =>
+ a.dataType match {
+ case t: TimestampType if t.sameType(StringType) =>
+ val value = typeCastStringToLong(v)
+ if (!value.equals(v)) {
+ Some(sources.Not(sources.EqualTo(a.name, value)))
+ } else {
+ Some(CastExpr(c))
+ }
+ case i: IntegerType if t.sameType(DoubleType) =>
+ val value = v.asInstanceOf[Double].toInt
+ if (value.toDouble.equals(v)) {
+ Some(sources.Not(sources.EqualTo(a.name, value)))
+ } else {
+ Some(CastExpr(c))
+ }
+ case _ => Some(CastExpr(c))
+ }
+ case c@Not(In(Cast(a: Attribute, _), list)) =>
+ a.dataType match {
+ case t: TimestampType if list.head.dataType.sameType(StringType) =>
+ val value = typeCastStringToLongList(list)
+ if (!value.equals(list)) {
+ val hSet = value.map(e => e.eval(EmptyRow))
+ Some(sources.Not(sources.In(a.name, hSet.toArray)))
+ } else {
+ Some(CastExpr(c))
+ }
+ case i: IntegerType if list.head.dataType.sameType(DoubleType) =>
+ val value = typeCastDoubleToIntList(list)
+ if (!value.equals(list)) {
+ val hSet = value.map(e => e.eval(EmptyRow))
+ Some(sources.Not(sources.In(a.name, hSet.toArray)))
+ } else {
+ Some(CastExpr(c))
+ }
+ case _ => Some(CastExpr(c))
+ }
+ case c@In(Cast(a: Attribute, _), list) =>
+ a.dataType match {
+ case t: TimestampType if list.head.dataType.sameType(StringType) =>
+ val value = typeCastStringToLongList(list)
+ if (!value.equals(list)) {
+ val hSet = value.map(e => e.eval(EmptyRow))
+ Some(sources.In(a.name, hSet.toArray))
+ } else {
+ Some(CastExpr(c))
+ }
+ case i: IntegerType if list.head.dataType.sameType(DoubleType) =>
+ val value = typeCastDoubleToIntList(list)
+ if (!value.equals(list)) {
+ val hSet = value.map(e => e.eval(EmptyRow))
+ Some(sources.In(a.name, hSet.toArray))
+ } else {
+ Some(CastExpr(c))
+ }
+ case _ => Some(CastExpr(c))
+ }
+ case c@GreaterThan(Cast(a: Attribute, _), Literal(v, t)) =>
+ a.dataType match {
+ case t: TimestampType if t.sameType(StringType) =>
+ val value = typeCastStringToLong(v)
+ if (!value.equals(v)) {
+ Some(sources.GreaterThan(a.name, value))
+ } else {
+ Some(CastExpr(c))
+ }
+ case i: IntegerType if t.sameType(DoubleType) =>
+ val value = v.asInstanceOf[Double].toInt
+ if (value.toDouble.equals(v)) {
+ Some(sources.GreaterThan(a.name, value))
+ } else {
+ Some(CastExpr(c))
+ }
+ case _ => Some(CastExpr(c))
+ }
+ case c@GreaterThan(Literal(v, t), Cast(a: Attribute, _)) =>
+ a.dataType match {
+ case t: TimestampType if t.sameType(StringType) =>
+ val value = typeCastStringToLong(v)
+ if (!value.equals(v)) {
+ Some(sources.LessThan(a.name, value))
+ } else {
+ Some(CastExpr(c))
+ }
+ case i: IntegerType if t.sameType(DoubleType) =>
+ val value = v.asInstanceOf[Double].toInt
+ if (value.toDouble.equals(v)) {
+ Some(sources.LessThan(a.name, value))
+ } else {
+ Some(CastExpr(c))
+ }
+ case _ => Some(CastExpr(c))
+ }
+ case c@LessThan(Cast(a: Attribute, _), Literal(v, t)) =>
+ a.dataType match {
+ case t: TimestampType if t.sameType(StringType) =>
+ val value = typeCastStringToLong(v)
+ if (!value.equals(v)) {
+ Some(sources.LessThan(a.name, value))
+ } else {
+ Some(CastExpr(c))
+ }
+ case i: IntegerType if t.sameType(DoubleType) =>
+ val value = v.asInstanceOf[Double].toInt
+ if (value.toDouble.equals(v)) {
+ Some(sources.LessThan(a.name, value))
+ } else {
+ Some(CastExpr(c))
+ }
+ case _ => Some(CastExpr(c))
+ }
+ case c@LessThan(Literal(v, t), Cast(a: Attribute, _)) =>
+ a.dataType match {
+ case t: TimestampType if t.sameType(StringType) =>
+ val value = typeCastStringToLong(v)
+ if (!value.equals(v)) {
+ Some(sources.GreaterThan(a.name, value))
+ } else {
+ Some(CastExpr(c))
+ }
+ case i: IntegerType if t.sameType(DoubleType) =>
+ val value = v.asInstanceOf[Double].toInt
+ if (value.toDouble.equals(v)) {
+ Some(sources.GreaterThan(a.name, value))
+ } else {
+ Some(CastExpr(c))
+ }
+ case _ => Some(CastExpr(c))
+ }
+ case c@GreaterThanOrEqual(Cast(a: Attribute, _), Literal(v, t)) =>
+ a.dataType match {
+ case t: TimestampType if t.sameType(StringType) =>
+ val value = typeCastStringToLong(v)
+ if (!value.equals(v)) {
+ Some(sources.GreaterThanOrEqual(a.name, value))
+ } else {
+ Some(CastExpr(c))
+ }
+ case i: IntegerType if t.sameType(DoubleType) =>
+ val value = v.asInstanceOf[Double].toInt
+ if (value.toDouble.equals(v)) {
+ Some(sources.GreaterThanOrEqual(a.name, value))
+ } else {
+ Some(CastExpr(c))
+ }
+ case _ => Some(CastExpr(c))
+ }
+ case c@GreaterThanOrEqual(Literal(v, t), Cast(a: Attribute, _)) =>
+ a.dataType match {
+ case t: TimestampType if t.sameType(StringType) =>
+ val value = typeCastStringToLong(v)
+ if (!value.equals(v)) {
+ Some(sources.LessThanOrEqual(a.name, value))
+ } else {
+ Some(CastExpr(c))
+ }
+ case i: IntegerType if t.sameType(DoubleType) =>
+ val value = v.asInstanceOf[Double].toInt
+ if (value.toDouble.equals(v)) {
+ Some(sources.LessThanOrEqual(a.name, value))
+ } else {
+ Some(CastExpr(c))
+ }
+ case _ => Some(CastExpr(c))
+ }
+ case c@LessThanOrEqual(Cast(a: Attribute, _), Literal(v, t)) =>
+ a.dataType match {
+ case t: TimestampType if t.sameType(StringType) =>
+ val value = typeCastStringToLong(v)
+ if (!value.equals(v)) {
+ Some(sources.LessThanOrEqual(a.name, value))
+ } else {
+ Some(CastExpr(c))
+ }
+ case i: IntegerType if t.sameType(DoubleType) =>
+ val value = v.asInstanceOf[Double].toInt
+ if (value.toDouble.equals(v)) {
+ Some(sources.LessThanOrEqual(a.name, value))
+ } else {
+ Some(CastExpr(c))
+ }
+ case _ => Some(CastExpr(c))
+ }
+ case c@LessThanOrEqual(Literal(v, t), Cast(a: Attribute, _)) =>
+ a.dataType match {
+ case t: TimestampType if t.sameType(StringType) =>
+ val value = typeCastStringToLong(v)
+ if (!value.equals(v)) {
+ Some(sources.GreaterThanOrEqual(a.name, value))
+ } else {
+ Some(CastExpr(c))
+ }
+ case i: IntegerType if t.sameType(DoubleType) =>
+ val value = v.asInstanceOf[Double].toInt
+ if (value.toDouble.equals(v)) {
+ Some(sources.GreaterThanOrEqual(a.name, value))
+ } else {
+ Some(CastExpr(c))
+ }
+ case _ => Some(CastExpr(c))
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96e5b5cc/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
new file mode 100644
index 0000000..88343c0
--- /dev/null
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
@@ -0,0 +1,386 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark
+
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.execution.CastExpressionOptimization
+import org.apache.spark.sql.optimizer.AttributeReferenceWrapper
+import org.apache.spark.sql.CarbonBoundReference
+import org.apache.spark.sql.CastExpr
+import org.apache.spark.sql.sources
+import org.apache.spark.sql.types._
+
+import org.apache.carbondata.core.metadata.datatype.DataType
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn
+import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonColumnExpression, Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression}
+import org.apache.carbondata.core.scan.expression.conditional._
+import org.apache.carbondata.core.scan.expression.logical.{AndExpression, FalseExpression, OrExpression}
+import org.apache.carbondata.spark.util.CarbonScalaUtil
+
+
+/**
+ * All filter conversions are done here.
+ */
+object CarbonFilters {
+
+
+ /**
+ * Converts data sources filters to carbon filter predicates.
+ */
+ def createCarbonFilter(schema: StructType,
+ predicate: sources.Filter): Option[CarbonExpression] = {
+ val dataTypeOf = schema.map(f => f.name -> f.dataType).toMap
+
+ def createFilter(predicate: sources.Filter): Option[CarbonExpression] = {
+ predicate match {
+
+ case sources.EqualTo(name, value) =>
+ Some(new EqualToExpression(getCarbonExpression(name),
+ getCarbonLiteralExpression(name, value)))
+ case sources.Not(sources.EqualTo(name, value)) =>
+ Some(new NotEqualsExpression(getCarbonExpression(name),
+ getCarbonLiteralExpression(name, value)))
+ case sources.EqualNullSafe(name, value) =>
+ Some(new EqualToExpression(getCarbonExpression(name),
+ getCarbonLiteralExpression(name, value)))
+ case sources.Not(sources.EqualNullSafe(name, value)) =>
+ Some(new NotEqualsExpression(getCarbonExpression(name),
+ getCarbonLiteralExpression(name, value)))
+ case sources.GreaterThan(name, value) =>
+ Some(new GreaterThanExpression(getCarbonExpression(name),
+ getCarbonLiteralExpression(name, value)))
+ case sources.LessThan(name, value) =>
+ Some(new LessThanExpression(getCarbonExpression(name),
+ getCarbonLiteralExpression(name, value)))
+ case sources.GreaterThanOrEqual(name, value) =>
+ Some(new GreaterThanEqualToExpression(getCarbonExpression(name),
+ getCarbonLiteralExpression(name, value)))
+ case sources.LessThanOrEqual(name, value) =>
+ Some(new LessThanEqualToExpression(getCarbonExpression(name),
+ getCarbonLiteralExpression(name, value)))
+ case sources.In(name, values) =>
+ Some(new InExpression(getCarbonExpression(name),
+ new ListExpression(
+ convertToJavaList(values.map(f => getCarbonLiteralExpression(name, f)).toList))))
+ case sources.Not(sources.In(name, values)) =>
+ Some(new NotInExpression(getCarbonExpression(name),
+ new ListExpression(
+ convertToJavaList(values.map(f => getCarbonLiteralExpression(name, f)).toList))))
+ case sources.IsNull(name) =>
+ Some(new EqualToExpression(getCarbonExpression(name),
+ getCarbonLiteralExpression(name, null), true))
+ case sources.IsNotNull(name) =>
+ Some(new NotEqualsExpression(getCarbonExpression(name),
+ getCarbonLiteralExpression(name, null), true))
+ case sources.And(lhs, rhs) =>
+ (createFilter(lhs) ++ createFilter(rhs)).reduceOption(new AndExpression(_, _))
+ case sources.Or(lhs, rhs) =>
+ for {
+ lhsFilter <- createFilter(lhs)
+ rhsFilter <- createFilter(rhs)
+ } yield {
+ new OrExpression(lhsFilter, rhsFilter)
+ }
+ case CastExpr(expr: Expression) =>
+ Some(transformExpression(expr))
+ case _ => None
+ }
+ }
+
+ def getCarbonExpression(name: String) = {
+ new CarbonColumnExpression(name,
+ CarbonScalaUtil.convertSparkToCarbonDataType(dataTypeOf(name)))
+ }
+
+ def getCarbonLiteralExpression(name: String, value: Any): CarbonExpression = {
+ val dataTypeOfAttribute = CarbonScalaUtil.convertSparkToCarbonDataType(dataTypeOf(name))
+ val dataType = if (Option(value).isDefined
+ && dataTypeOfAttribute == DataType.STRING
+ && value.isInstanceOf[Double]) {
+ DataType.DOUBLE
+ } else {
+ dataTypeOfAttribute
+ }
+ new CarbonLiteralExpression(value, dataType)
+ }
+
+ createFilter(predicate)
+ }
+
+
+ // Check out which filters can be pushed down to carbon, remaining can be handled in spark layer.
+ // Mostly dimension filters are only pushed down since it is faster in carbon.
+ // TODO - The Filters are first converted Intermediate sources filters expression and then these
+ // expressions are again converted back to CarbonExpression. Instead of two step process of
+ // evaluating the filters it can be merged into a single one.
+ def selectFilters(filters: Seq[Expression],
+ attrList: java.util.HashSet[AttributeReferenceWrapper],
+ aliasMap: CarbonAliasDecoderRelation): Unit = {
+ def translate(expr: Expression, or: Boolean = false): Option[sources.Filter] = {
+ expr match {
+ case or@Or(left, right) =>
+
+ val leftFilter = translate(left, or = true)
+ val rightFilter = translate(right, or = true)
+ if (leftFilter.isDefined && rightFilter.isDefined) {
+ Some(sources.Or(leftFilter.get, rightFilter.get))
+ } else {
+ or.collect {
+ case attr: AttributeReference =>
+ attrList.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
+ }
+ None
+ }
+ case And(left, right) =>
+ val leftFilter = translate(left, or)
+ val rightFilter = translate(right, or)
+ if (or) {
+ if (leftFilter.isDefined && rightFilter.isDefined) {
+ (leftFilter ++ rightFilter).reduceOption(sources.And)
+ } else {
+ None
+ }
+ } else {
+ (leftFilter ++ rightFilter).reduceOption(sources.And)
+ }
+ case EqualTo(a: Attribute, Literal(v, t)) =>
+ Some(sources.EqualTo(a.name, v))
+ case EqualTo(l@Literal(v, t), a: Attribute) =>
+ Some(sources.EqualTo(a.name, v))
+ case c@EqualTo(Cast(a: Attribute, _), Literal(v, t)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case c@EqualTo(Literal(v, t), Cast(a: Attribute, _)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case Not(EqualTo(a: Attribute, Literal(v, t))) =>
+ Some(sources.Not(sources.EqualTo(a.name, v)))
+ case Not(EqualTo(Literal(v, t), a: Attribute)) =>
+ Some(sources.Not(sources.EqualTo(a.name, v)))
+ case c@Not(EqualTo(Cast(a: Attribute, _), Literal(v, t))) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case c@Not(EqualTo(Literal(v, t), Cast(a: Attribute, _))) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case IsNotNull(a: Attribute) =>
+ Some(sources.IsNotNull(a.name))
+ case IsNull(a: Attribute) =>
+ Some(sources.IsNull(a.name))
+ case Not(In(a: Attribute, list)) if !list.exists(!_.isInstanceOf[Literal]) =>
+ val hSet = list.map(e => e.eval(EmptyRow))
+ Some(sources.Not(sources.In(a.name, hSet.toArray)))
+ case In(a: Attribute, list) if !list.exists(!_.isInstanceOf[Literal]) =>
+ val hSet = list.map(e => e.eval(EmptyRow))
+ Some(sources.In(a.name, hSet.toArray))
+ case c@Not(In(Cast(a: Attribute, _), list)) if !list.exists(!_.isInstanceOf[Literal]) =>
+ Some(CastExpr(c))
+ case c@In(Cast(a: Attribute, _), list) if !list.exists(!_.isInstanceOf[Literal]) =>
+ Some(CastExpr(c))
+ case GreaterThan(a: Attribute, Literal(v, t)) =>
+ Some(sources.GreaterThan(a.name, v))
+ case GreaterThan(Literal(v, t), a: Attribute) =>
+ Some(sources.LessThan(a.name, v))
+ case c@GreaterThan(Cast(a: Attribute, _), Literal(v, t)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case c@GreaterThan(Literal(v, t), Cast(a: Attribute, _)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case LessThan(a: Attribute, Literal(v, t)) =>
+ Some(sources.LessThan(a.name, v))
+ case LessThan(Literal(v, t), a: Attribute) =>
+ Some(sources.GreaterThan(a.name, v))
+ case c@LessThan(Cast(a: Attribute, _), Literal(v, t)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case c@LessThan(Literal(v, t), Cast(a: Attribute, _)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case GreaterThanOrEqual(a: Attribute, Literal(v, t)) =>
+ Some(sources.GreaterThanOrEqual(a.name, v))
+ case GreaterThanOrEqual(Literal(v, t), a: Attribute) =>
+ Some(sources.LessThanOrEqual(a.name, v))
+ case c@GreaterThanOrEqual(Cast(a: Attribute, _), Literal(v, t)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case c@GreaterThanOrEqual(Literal(v, t), Cast(a: Attribute, _)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case LessThanOrEqual(a: Attribute, Literal(v, t)) =>
+ Some(sources.LessThanOrEqual(a.name, v))
+ case LessThanOrEqual(Literal(v, t), a: Attribute) =>
+ Some(sources.GreaterThanOrEqual(a.name, v))
+ case c@LessThanOrEqual(Cast(a: Attribute, _), Literal(v, t)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case c@LessThanOrEqual(Literal(v, t), Cast(a: Attribute, _)) =>
+ CastExpressionOptimization.checkIfCastCanBeRemove(c)
+ case c@Cast(a: Attribute, _) =>
+ Some(CastExpr(c))
+ case others =>
+ if (!or) {
+ others.collect {
+ case attr: AttributeReference =>
+ attrList.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
+ }
+ }
+ None
+ }
+ }
+
+ filters.flatMap(translate(_, false)).toArray
+ }
+
+ def transformExpression(expr: Expression): CarbonExpression = {
+ expr match {
+ case Or(left, right)
+ if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right)) => new
+ OrExpression(transformExpression(left), transformExpression(right))
+ case And(left, right)
+ if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right)) => new
+ AndExpression(transformExpression(left), transformExpression(right))
+ case EqualTo(left, right)
+ if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right)) => new
+ EqualToExpression(transformExpression(left), transformExpression(right))
+ case Not(EqualTo(left, right))
+ if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right)) => new
+ NotEqualsExpression(transformExpression(left), transformExpression(right))
+ case IsNotNull(child)
+ if (isCarbonSupportedDataTypes(child)) => new
+ NotEqualsExpression(transformExpression(child), transformExpression(Literal(null)), true)
+ case IsNull(child)
+ if (isCarbonSupportedDataTypes(child)) => new
+ EqualToExpression(transformExpression(child), transformExpression(Literal(null)), true)
+ case Not(In(left, right)) if (isCarbonSupportedDataTypes(left)) =>
+ if (right.contains(null)) {
+ new FalseExpression(transformExpression(left))
+ }
+ else {
+ new NotInExpression(transformExpression(left),
+ new ListExpression(convertToJavaList(right.map(transformExpression)))
+ )
+ }
+ case In(left, right) if (isCarbonSupportedDataTypes(left)) =>
+ new InExpression(transformExpression(left),
+ new ListExpression(convertToJavaList(right.filter(_ != null).filter(!isNullLiteral(_))
+ .map(transformExpression))))
+ case InSet(left, right) if (isCarbonSupportedDataTypes(left)) =>
+ val validData = right.filter(_ != null).map { x =>
+ val e = Literal(x.toString)
+ transformExpression(e)
+ }.toList
+ new InExpression(transformExpression(left),
+ new ListExpression(convertToJavaList(validData)))
+ case Not(InSet(left, right)) if (isCarbonSupportedDataTypes(left)) =>
+ if (right.contains(null)) {
+ new FalseExpression(transformExpression(left))
+ }
+ else {
+ val r = right.map { x =>
+ val strVal = if (null == x) {
+ x
+ } else {
+ x.toString
+ }
+ val e = Literal(strVal)
+ transformExpression(e)
+ }.toList
+ new NotInExpression(transformExpression(left), new ListExpression(convertToJavaList(r)))
+ }
+ case GreaterThan(left, right)
+ if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right)) =>
+ new GreaterThanExpression(transformExpression(left), transformExpression(right))
+ case GreaterThanOrEqual(left, right)
+ if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right)) =>
+ new GreaterThanEqualToExpression(transformExpression(left), transformExpression(right))
+ case LessThan(left, right)
+ if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right)) =>
+ new LessThanExpression(transformExpression(left), transformExpression(right))
+ case LessThanOrEqual(left, right)
+ if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right)) =>
+ new LessThanEqualToExpression(transformExpression(left), transformExpression(right))
+ case AttributeReference(name, dataType, _, _) =>
+ new CarbonColumnExpression(name.toString,
+ CarbonScalaUtil.convertSparkToCarbonDataType(dataType))
+ case Literal(name, dataType) => new
+ CarbonLiteralExpression(name, CarbonScalaUtil.convertSparkToCarbonDataType(dataType))
+ case StringTrim(child) => transformExpression(child)
+ case _ =>
+ new SparkUnknownExpression(expr.transform {
+ case AttributeReference(name, dataType, _, _) =>
+ CarbonBoundReference(new CarbonColumnExpression(name.toString,
+ CarbonScalaUtil.convertSparkToCarbonDataType(dataType)), dataType, expr.nullable)
+ }
+ )
+ }
+ }
+
+
+ private def isNullLiteral(exp: Expression): Boolean = {
+ if (null != exp
+ && exp.isInstanceOf[Literal]
+ && (exp.asInstanceOf[Literal].dataType == org.apache.spark.sql.types.DataTypes.NullType)
+ || (exp.asInstanceOf[Literal].value == null)) {
+ true
+ } else {
+ false
+ }
+ }
+
+ def isCarbonSupportedDataTypes(expr: Expression): Boolean = {
+ expr.dataType match {
+ case StringType => true
+ case IntegerType => true
+ case LongType => true
+ case DoubleType => true
+ case FloatType => true
+ case BooleanType => true
+ case TimestampType => true
+ case ArrayType(_, _) => true
+ case StructType(_) => true
+ case DecimalType() => true
+ case _ => false
+ }
+ }
+
+ private def getActualCarbonDataType(column: String, carbonTable: CarbonTable) = {
+ var carbonColumn: CarbonColumn =
+ carbonTable.getDimensionByName(carbonTable.getFactTableName, column)
+ val dataType = if (carbonColumn != null) {
+ carbonColumn.getDataType
+ } else {
+ carbonColumn = carbonTable.getMeasureByName(carbonTable.getFactTableName, column)
+ carbonColumn.getDataType match {
+ case DataType.INT => DataType.LONG
+ case DataType.LONG => DataType.LONG
+ case DataType.DECIMAL => DataType.DECIMAL
+ case _ => DataType.DOUBLE
+ }
+ }
+ CarbonScalaUtil.convertCarbonToSparkDataType(dataType)
+ }
+
+ // Convert scala list to java list, Cannot use scalaList.asJava as while deserializing it is
+ // not able find the classes inside scala list and gives ClassNotFoundException.
+ private def convertToJavaList(
+ scalaList: Seq[CarbonExpression]): java.util.List[CarbonExpression] = {
+ val javaList = new java.util.ArrayList[CarbonExpression]()
+ scalaList.foreach(javaList.add)
+ javaList
+ }
+
+ def preProcessExpressions(expressions: Seq[Expression]): Seq[Expression] = {
+ expressions match {
+ case left :: right :: rest => preProcessExpressions(List(And(left, right)) ::: rest)
+ case List(left, right) => List(And(left, right))
+
+ case _ => expressions
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96e5b5cc/integration/spark2/src/test/scala/org/apache/spark/carbondata/query/TestNotEqualToFilter.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/query/TestNotEqualToFilter.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/query/TestNotEqualToFilter.scala
index fb0188c..cc56509 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/query/TestNotEqualToFilter.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/query/TestNotEqualToFilter.scala
@@ -63,6 +63,9 @@ class TestNotEqualToFilter extends QueryTest with BeforeAndAfterAll {
}
test("select Id from test_not_equal_to_carbon where id != '7'") {
+ // sql("select id from test_not_equal_to_carbon").show(200,false)
+ // sql("select id from test_not_equal_to_hive").show(200,false)
+ sql("select Id from test_not_equal_to_carbon where id > '1.5rre'").show(200, false)
checkAnswer(
sql("select Id from test_not_equal_to_carbon where id != '7'"),
sql("select Id from test_not_equal_to_hive where id != '7'")
[15/50] [abbrv] incubator-carbondata git commit: add alter table
example
Posted by ja...@apache.org.
add alter table example
add alter table example
add alter table example
add alter table example
Just add AlterTableExample
Just add AlterTableExample
Just add AlterTableExample
Add AlterTableExamples
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/1f9721f2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/1f9721f2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/1f9721f2
Branch: refs/heads/12-dev
Commit: 1f9721f2b390304f69987b07594b6c8ea996c3aa
Parents: 4b615d8
Author: erlu <er...@erludeMacBook-Pro.local>
Authored: Wed Apr 26 10:54:19 2017 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Wed Apr 26 23:52:33 2017 +0800
----------------------------------------------------------------------
.../carbondata/examples/AlterTableExample.scala | 97 ++++++++++++++++++++
1 file changed, 97 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/1f9721f2/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
new file mode 100644
index 0000000..1d01de9
--- /dev/null
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.examples
+
+import java.io.File
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.spark.sql.SparkSession
+
+
+object AlterTableExample {
+
+ def main(args: Array[String]): Unit = {
+
+ val rootPath = new File(this.getClass.getResource("/").getPath
+ + "../../../..").getCanonicalPath
+
+ val storeLocation = s"$rootPath/examples/spark2/target/store"
+ val warehouse = s"$rootPath/examples/spark2/target/warehouse"
+ val metastoredb = s"$rootPath/examples/spark2/target/metastore_db"
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
+
+ import org.apache.spark.sql.CarbonSession._
+
+ val spark = SparkSession
+ .builder()
+ .master("local")
+ .appName("AlterTableExample")
+ .config("spark.sql.warehouse.dir", warehouse)
+ .getOrCreateCarbonSession(storeLocation, metastoredb)
+
+ spark.sparkContext.setLogLevel("WARN")
+
+ spark.sql("DROP TABLE IF EXISTS carbon_table")
+ spark.sql("DROP TABLE IF EXISTS new_carbon_table")
+
+ spark.sql(
+ s"""
+ | CREATE TABLE carbon_table(
+ | shortField short,
+ | intField int,
+ | bigintField long,
+ | doubleField double,
+ | stringField string,
+ | timestampField timestamp,
+ | decimalField decimal(18,2),
+ | dateField date,
+ | charField char(5),
+ | floatField float,
+ | complexData array<string>
+ | )
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('DICTIONARY_INCLUDE'='dateField, charField')
+ """.stripMargin)
+
+ // Alter table change data type
+ spark.sql("DESCRIBE FORMATTED carbon_table").show()
+ spark.sql("ALTER TABLE carbon_table CHANGE intField intField bigint").show()
+
+ // Alter table add columns
+ spark.sql("DESCRIBE FORMATTED carbon_table").show()
+ spark.sql("ALTER TABLE carbon_table ADD COLUMNS (newField String) " +
+ "TBLPROPERTIES ('DEFAULT.VALUE.newField'='def')").show()
+
+ // Alter table drop columns
+ spark.sql("DESCRIBE FORMATTED carbon_table").show()
+ spark.sql("ALTER TABLE carbon_table DROP COLUMNS (newField)").show()
+ spark.sql("DESCRIBE FORMATTED carbon_table").show()
+
+ // Alter table rename table name
+ spark.sql("SHOW TABLES").show()
+ spark.sql("ALTER TABLE carbon_table RENAME TO new_carbon_table").show()
+ spark.sql("SHOW TABLES").show()
+
+ spark.sql("DROP TABLE IF EXISTS carbon_table")
+ spark.sql("DROP TABLE IF EXISTS new_carbon_table")
+
+ }
+}
[42/50] [abbrv] incubator-carbondata git commit: add SORT_COLUMNS
options
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java
new file mode 100644
index 0000000..bde89ed
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.processing.newflow.steps;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.processing.newflow.AbstractDataLoadProcessorStep;
+import org.apache.carbondata.processing.newflow.CarbonDataLoadConfiguration;
+import org.apache.carbondata.processing.newflow.DataField;
+import org.apache.carbondata.processing.newflow.row.CarbonRow;
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
+import org.apache.carbondata.processing.util.NonDictionaryUtil;
+
+/**
+ * if the table doesn't have sort_columns, just convert row format.
+ */
+public class NoSortProcessorStepImpl extends AbstractDataLoadProcessorStep {
+
+ private int dimensionCount;
+
+ private int dimensionWithComplexCount;
+
+ private int noDictCount;
+
+ private int measureCount;
+
+ private boolean[] isNoDictionaryDimensionColumn;
+
+ private char[] aggType;
+
+ public NoSortProcessorStepImpl(CarbonDataLoadConfiguration configuration,
+ AbstractDataLoadProcessorStep child) {
+ super(configuration, child);
+ this.dimensionWithComplexCount = configuration.getDimensionCount();
+ this.noDictCount =
+ configuration.getNoDictionaryCount() + configuration.getComplexDimensionCount();
+ this.dimensionCount = configuration.getDimensionCount() - this.noDictCount;
+ this.measureCount = configuration.getMeasureCount();
+ this.isNoDictionaryDimensionColumn =
+ CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields());
+ this.aggType = CarbonDataProcessorUtil
+ .getAggType(configuration.getMeasureCount(), configuration.getMeasureFields());
+ }
+
+ @Override public DataField[] getOutput() {
+ return child.getOutput();
+ }
+
+ @Override public void initialize() throws IOException {
+ child.initialize();
+ }
+
+ /**
+ * convert input CarbonRow to output CarbonRow
+ * e.g. There is a table as following,
+ * the number of dictionary dimensions is a,
+ * the number of no-dictionary dimensions is b,
+ * the number of complex dimensions is c,
+ * the number of measures is d.
+ * input CarbonRow format: the length of Object[] data is a+b+c+d, the number of all columns.
+ * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ * | Part | Object item | describe |
+ * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ * | Object[0 ~ a+b-1] | Integer, byte[], Integer, ... | dict + no dict dimensions|
+ * ----------------------------------------------------------------------------------------
+ * | Object[a+b ~ a+b+c-1] | byte[], byte[], ... | complex dimensions |
+ * ----------------------------------------------------------------------------------------
+ * | Object[a+b+c ~ a+b+c+d-1]| int, byte[], ... | measures |
+ * ----------------------------------------------------------------------------------------
+ * output CarbonRow format: the length of object[] data is 3.
+ * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ * | Part | Object item | describe |
+ * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ * | Object[0] | int[a] | dict dimension array |
+ * ----------------------------------------------------------------------------------------
+ * | Object[1] | byte[b+c][] | no dict + complex dim |
+ * ----------------------------------------------------------------------------------------
+ * | Object[2] | Object[d] | measures |
+ * ----------------------------------------------------------------------------------------
+ * @param row
+ * @return
+ */
+ @Override protected CarbonRow processRow(CarbonRow row) {
+ int dictIndex = 0;
+ int nonDicIndex = 0;
+ int[] dim = new int[this.dimensionCount];
+ byte[][] nonDicArray = new byte[this.noDictCount][];
+ Object[] measures = new Object[this.measureCount];
+ // read dimension values
+ int dimCount = 0;
+ for (; dimCount < isNoDictionaryDimensionColumn.length; dimCount++) {
+ if (isNoDictionaryDimensionColumn[dimCount]) {
+ nonDicArray[nonDicIndex++] = (byte[]) row.getObject(dimCount);
+ } else {
+ dim[dictIndex++] = (int) row.getObject(dimCount);
+ }
+ }
+
+ for (; dimCount < this.dimensionWithComplexCount; dimCount++) {
+ nonDicArray[nonDicIndex++] = (byte[]) row.getObject(dimCount);
+ }
+
+ // measure values
+ for (int mesCount = 0; mesCount < this.measureCount; mesCount++) {
+ Object value = row.getObject(mesCount + this.dimensionWithComplexCount);
+ if (null != value) {
+ if (aggType[mesCount] == CarbonCommonConstants.DOUBLE_MEASURE) {
+ measures[mesCount] = value;
+ } else if (aggType[mesCount] == CarbonCommonConstants.BIG_INT_MEASURE) {
+ measures[mesCount] = value;
+ } else if (aggType[mesCount] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
+ BigDecimal val = (BigDecimal) value;
+ measures[mesCount] = DataTypeUtil.bigDecimalToByte(val);
+ }
+ } else {
+ measures[mesCount] = null;
+ }
+ }
+ // create new row of size 3 (1 for dims , 1 for high card , 1 for measures)
+ Object[] holder = new Object[3];
+ NonDictionaryUtil.prepareOutObj(holder, dim, nonDicArray, measures);
+ //return out row
+ return new CarbonRow(holder);
+ }
+
+ @Override
+ public void close() {
+ if (!closed) {
+ super.close();
+ }
+ }
+
+ @Override protected String getStepName() {
+ return "No Sort Processor";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java
index 5487593..a9e762d 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java
@@ -251,7 +251,8 @@ public class IntermediateFileMerger implements Callable<Void> {
new SortTempFileChunkHolder(tempFile, mergerParameters.getDimColCount(),
mergerParameters.getComplexDimColCount(), mergerParameters.getMeasureColCount(),
mergerParameters.getFileBufferSize(), mergerParameters.getNoDictionaryCount(),
- mergerParameters.getAggType(), mergerParameters.getNoDictionaryDimnesionColumn());
+ mergerParameters.getAggType(), mergerParameters.getNoDictionaryDimnesionColumn(),
+ mergerParameters.getNoDictionarySortColumn());
// initialize
sortTempFileChunkHolder.initialize();
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/NewRowComparator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/NewRowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/NewRowComparator.java
index dd9358c..247251e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/NewRowComparator.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/NewRowComparator.java
@@ -24,15 +24,15 @@ import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
public class NewRowComparator implements Comparator<Object[]> {
/**
- * noDictionaryColMaping mapping of dictionary dimensions and no dictionary dimensions.
+ * mapping of dictionary dimensions and no dictionary of sort_column.
*/
- private boolean[] noDictionaryColMaping;
+ private boolean[] noDictionarySortColumnMaping;
/**
- * @param noDictionaryColMaping
+ * @param noDictionarySortColumnMaping
*/
- public NewRowComparator(boolean[] noDictionaryColMaping) {
- this.noDictionaryColMaping = noDictionaryColMaping;
+ public NewRowComparator(boolean[] noDictionarySortColumnMaping) {
+ this.noDictionarySortColumnMaping = noDictionarySortColumnMaping;
}
/**
@@ -43,7 +43,7 @@ public class NewRowComparator implements Comparator<Object[]> {
int index = 0;
- for (boolean isNoDictionary : noDictionaryColMaping) {
+ for (boolean isNoDictionary : noDictionarySortColumnMaping) {
if (isNoDictionary) {
byte[] byteArr1 = (byte[]) rowA[index];
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/NewRowComparatorForNormalDims.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/NewRowComparatorForNormalDims.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/NewRowComparatorForNormalDims.java
index d913b32..241882e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/NewRowComparatorForNormalDims.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/NewRowComparatorForNormalDims.java
@@ -26,15 +26,15 @@ public class NewRowComparatorForNormalDims implements Comparator<Object[]> {
/**
* dimension count
*/
- private int dimensionCount;
+ private int numberOfSortColumns;
/**
* RowComparatorForNormalDims Constructor
*
- * @param dimensionCount
+ * @param numberOfSortColumns
*/
- public NewRowComparatorForNormalDims(int dimensionCount) {
- this.dimensionCount = dimensionCount;
+ public NewRowComparatorForNormalDims(int numberOfSortColumns) {
+ this.numberOfSortColumns = numberOfSortColumns;
}
/**
@@ -45,7 +45,7 @@ public class NewRowComparatorForNormalDims implements Comparator<Object[]> {
public int compare(Object[] rowA, Object[] rowB) {
int diff = 0;
- for (int i = 0; i < dimensionCount; i++) {
+ for (int i = 0; i < numberOfSortColumns; i++) {
int dimFieldA = (int)rowA[i];
int dimFieldB = (int)rowB[i];
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/RowComparator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/RowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/RowComparator.java
index c282f52..2584048 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/RowComparator.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/RowComparator.java
@@ -33,15 +33,15 @@ public class RowComparator implements Comparator<Object[]> {
/**
* noDictionaryColMaping mapping of dictionary dimensions and no dictionary dimensions.
*/
- private boolean[] noDictionaryColMaping;
+ private boolean[] noDictionarySortColumnMaping;
/**
- * @param noDictionaryColMaping
+ * @param noDictionarySortColumnMaping
* @param noDictionaryCount
*/
- public RowComparator(boolean[] noDictionaryColMaping, int noDictionaryCount) {
+ public RowComparator(boolean[] noDictionarySortColumnMaping, int noDictionaryCount) {
this.noDictionaryCount = noDictionaryCount;
- this.noDictionaryColMaping = noDictionaryColMaping;
+ this.noDictionarySortColumnMaping = noDictionarySortColumnMaping;
}
/**
@@ -53,7 +53,7 @@ public class RowComparator implements Comparator<Object[]> {
int normalIndex = 0;
int noDictionaryindex = 0;
- for (boolean isNoDictionary : noDictionaryColMaping) {
+ for (boolean isNoDictionary : noDictionarySortColumnMaping) {
if (isNoDictionary) {
byte[] byteArr1 = (byte[]) rowA[IgnoreDictionary.BYTE_ARRAY_INDEX_IN_ROW.getIndex()];
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/RowComparatorForNormalDims.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/RowComparatorForNormalDims.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/RowComparatorForNormalDims.java
index ceaf5c6..8d914ea 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/RowComparatorForNormalDims.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/RowComparatorForNormalDims.java
@@ -28,15 +28,15 @@ public class RowComparatorForNormalDims implements Comparator<Object[]> {
/**
* dimension count
*/
- private int dimensionCount;
+ private int numberOfSortColumns;
/**
* RowComparatorForNormalDims Constructor
*
- * @param dimensionCount
+ * @param numberOfSortColumns
*/
- public RowComparatorForNormalDims(int dimensionCount) {
- this.dimensionCount = dimensionCount;
+ public RowComparatorForNormalDims(int numberOfSortColumns) {
+ this.numberOfSortColumns = numberOfSortColumns;
}
/**
@@ -47,7 +47,7 @@ public class RowComparatorForNormalDims implements Comparator<Object[]> {
public int compare(Object[] rowA, Object[] rowB) {
int diff = 0;
- for (int i = 0; i < dimensionCount; i++) {
+ for (int i = 0; i < numberOfSortColumns; i++) {
int dimFieldA = NonDictionaryUtil.getDimension(i, rowA);
int dimFieldB = NonDictionaryUtil.getDimension(i, rowB);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
index ffaa566..eba5433 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortDataRows.java
@@ -192,11 +192,10 @@ public class SortDataRows {
Object[][] toSort;
toSort = new Object[entryCount][];
System.arraycopy(recordHolderList, 0, toSort, 0, entryCount);
-
- if (parameters.getNoDictionaryCount() > 0) {
- Arrays.sort(toSort, new NewRowComparator(parameters.getNoDictionaryDimnesionColumn()));
+ if (parameters.getNumberOfNoDictSortColumns() > 0) {
+ Arrays.sort(toSort, new NewRowComparator(parameters.getNoDictionarySortColumn()));
} else {
- Arrays.sort(toSort, new NewRowComparatorForNormalDims(parameters.getDimColCount()));
+ Arrays.sort(toSort, new NewRowComparatorForNormalDims(parameters.getNumberOfSortColumns()));
}
recordHolderList = toSort;
@@ -386,12 +385,12 @@ public class SortDataRows {
@Override public Void call() throws Exception {
try {
long startTime = System.currentTimeMillis();
- if (parameters.getNoDictionaryCount() > 0) {
+ if (parameters.getNumberOfNoDictSortColumns() > 0) {
Arrays.sort(recordHolderArray,
- new NewRowComparator(parameters.getNoDictionaryDimnesionColumn()));
+ new NewRowComparator(parameters.getNoDictionarySortColumn()));
} else {
Arrays.sort(recordHolderArray,
- new NewRowComparatorForNormalDims(parameters.getDimColCount()));
+ new NewRowComparatorForNormalDims(parameters.getNumberOfSortColumns()));
}
// create a new file every time
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
index 3c3a9d8..7ef8f8e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
@@ -112,6 +112,12 @@ public class SortParameters {
*/
private boolean[] noDictionaryDimnesionColumn;
+ private boolean[] noDictionarySortColumn;
+
+ private int numberOfSortColumns;
+
+ private int numberOfNoDictSortColumns;
+
private int numberOfCores;
public SortParameters getCopy() {
@@ -137,6 +143,9 @@ public class SortParameters {
parameters.segmentId = segmentId;
parameters.taskNo = taskNo;
parameters.noDictionaryDimnesionColumn = noDictionaryDimnesionColumn;
+ parameters.noDictionarySortColumn = noDictionarySortColumn;
+ parameters.numberOfSortColumns = numberOfSortColumns;
+ parameters.numberOfNoDictSortColumns = numberOfNoDictSortColumns;
parameters.numberOfCores = numberOfCores;
return parameters;
}
@@ -317,6 +326,30 @@ public class SortParameters {
this.numberOfCores = numberOfCores;
}
+ public int getNumberOfSortColumns() {
+ return numberOfSortColumns;
+ }
+
+ public void setNumberOfSortColumns(int numberOfSortColumns) {
+ this.numberOfSortColumns = Math.min(numberOfSortColumns, this.dimColCount);
+ }
+
+ public boolean[] getNoDictionarySortColumn() {
+ return noDictionarySortColumn;
+ }
+
+ public void setNoDictionarySortColumn(boolean[] noDictionarySortColumn) {
+ this.noDictionarySortColumn = noDictionarySortColumn;
+ }
+
+ public int getNumberOfNoDictSortColumns() {
+ return numberOfNoDictSortColumns;
+ }
+
+ public void setNumberOfNoDictSortColumns(int numberOfNoDictSortColumns) {
+ this.numberOfNoDictSortColumns = Math.min(numberOfNoDictSortColumns, noDictionaryCount);
+ }
+
public static SortParameters createSortParameters(CarbonDataLoadConfiguration configuration) {
SortParameters parameters = new SortParameters();
CarbonTableIdentifier tableIdentifier =
@@ -334,6 +367,16 @@ public class SortParameters {
parameters.setComplexDimColCount(configuration.getComplexDimensionCount());
parameters.setNoDictionaryDimnesionColumn(
CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields()));
+ parameters.setNumberOfSortColumns(configuration.getNumberOfSortColumns());
+ parameters.setNumberOfNoDictSortColumns(configuration.getNumberOfNoDictSortColumns());
+ if (parameters.getNumberOfSortColumns() == parameters.getNoDictionaryDimnesionColumn().length) {
+ parameters.setNoDictionarySortColumn(parameters.getNoDictionaryDimnesionColumn());
+ } else {
+ boolean[] noDictionarySortColumnTemp = new boolean[parameters.getNumberOfSortColumns()];
+ System.arraycopy(parameters.getNoDictionaryDimnesionColumn(), 0,
+ noDictionarySortColumnTemp, 0, parameters.getNumberOfSortColumns());
+ parameters.setNoDictionarySortColumn(noDictionarySortColumnTemp);
+ }
parameters.setObserver(new SortObserver());
// get sort buffer size
parameters.setSortBufferSize(Integer.parseInt(carbonProperties
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
index b4ccc6f..2b0b8ae 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
@@ -133,6 +133,11 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
private boolean[] isNoDictionaryDimensionColumn;
/**
+ * to store whether sort column is of dictionary type or not
+ */
+ private boolean[] isNoDictionarySortColumn;
+
+ /**
* Constructor to initialize
*
* @param tempFile
@@ -146,7 +151,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
*/
public SortTempFileChunkHolder(File tempFile, int dimensionCount, int complexDimensionCount,
int measureCount, int fileBufferSize, int noDictionaryCount, char[] aggType,
- boolean[] isNoDictionaryDimensionColumn) {
+ boolean[] isNoDictionaryDimensionColumn, boolean[] isNoDictionarySortColumn) {
// set temp file
this.tempFile = tempFile;
@@ -160,7 +165,9 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
this.fileBufferSize = fileBufferSize;
this.executorService = Executors.newFixedThreadPool(1);
this.aggType = aggType;
+
this.isNoDictionaryDimensionColumn = isNoDictionaryDimensionColumn;
+ this.isNoDictionarySortColumn = isNoDictionarySortColumn;
}
/**
@@ -409,7 +416,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
int[] rightMdkArray = (int[]) other.returnRow[0];
byte[][] leftNonDictArray = (byte[][]) returnRow[1];
byte[][] rightNonDictArray = (byte[][]) other.returnRow[1];
- for (boolean isNoDictionary : isNoDictionaryDimensionColumn) {
+ for (boolean isNoDictionary : isNoDictionarySortColumn) {
if (isNoDictionary) {
diff = UnsafeComparer.INSTANCE
.compareTo(leftNonDictArray[noDictionaryIndex], rightNonDictArray[noDictionaryIndex]);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index 7a80f72..8bf4759 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -778,13 +778,15 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
int i = 0;
int dictionaryColumnCount = -1;
int noDictionaryColumnCount = -1;
+ boolean isSortColumn = false;
for (i = 0; i < dimensionType.length; i++) {
+ isSortColumn = i < segmentProperties.getNumberOfSortColumns();
if (dimensionType[i]) {
dictionaryColumnCount++;
if (colGrpModel.isColumnar(dictionaryColumnCount)) {
submit.add(executorService.submit(
- new BlockSortThread(i, dataHolders[dictionaryColumnCount].getData(), true,
- isUseInvertedIndex[i])));
+ new BlockSortThread(i, dataHolders[dictionaryColumnCount].getData(), isSortColumn,
+ isUseInvertedIndex[i] & isSortColumn)));
} else {
submit.add(
executorService.submit(new ColGroupBlockStorage(dataHolders[dictionaryColumnCount])));
@@ -792,7 +794,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
} else {
submit.add(executorService.submit(
new BlockSortThread(i, noDictionaryColumnsData[++noDictionaryColumnCount], false, true,
- true, isUseInvertedIndex[i])));
+ isSortColumn, isUseInvertedIndex[i] & isSortColumn)));
}
}
for (int k = 0; k < complexColCount; k++) {
@@ -816,7 +818,42 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
}
byte[] composedNonDictStartKey = null;
byte[] composedNonDictEndKey = null;
- if (noDictionaryStartKey != null) {
+
+ int numberOfDictSortColumns = segmentProperties.getNumberOfDictSortColumns();
+ // generate start/end key by sort_columns
+ if (numberOfDictSortColumns > 0) {
+ // if sort_columns contain dictionary columns
+ int[] keySize = columnarSplitter.getBlockKeySize();
+ if (keySize.length > numberOfDictSortColumns) {
+ int newMdkLength = 0;
+ for (int index = 0; index < numberOfDictSortColumns; index++) {
+ newMdkLength += keySize[index];
+ }
+ byte[] newStartKeyOfSortKey = new byte[newMdkLength];
+ byte[] newEndKeyOfSortKey = new byte[newMdkLength];
+ System.arraycopy(startkeyLocal, 0, newStartKeyOfSortKey, 0, newMdkLength);
+ System.arraycopy(endKeyLocal, 0, newEndKeyOfSortKey, 0, newMdkLength);
+ startkeyLocal = newStartKeyOfSortKey;
+ endKeyLocal = newEndKeyOfSortKey;
+ }
+ } else {
+ startkeyLocal = new byte[0];
+ endKeyLocal = new byte[0];
+ }
+
+ int numberOfNoDictSortColumns = segmentProperties.getNumberOfNoDictSortColumns();
+ if (numberOfNoDictSortColumns > 0) {
+ // if sort_columns contain no-dictionary columns
+ if (noDictionaryStartKey.length > numberOfNoDictSortColumns) {
+ byte[][] newNoDictionaryStartKey = new byte[numberOfNoDictSortColumns][];
+ byte[][] newNoDictionaryEndKey = new byte[numberOfNoDictSortColumns][];
+ System.arraycopy(noDictionaryStartKey, 0, newNoDictionaryStartKey, 0,
+ numberOfNoDictSortColumns);
+ System
+ .arraycopy(noDictionaryEndKey, 0, newNoDictionaryEndKey, 0, numberOfNoDictSortColumns);
+ noDictionaryStartKey = newNoDictionaryStartKey;
+ noDictionaryEndKey = newNoDictionaryEndKey;
+ }
composedNonDictStartKey =
NonDictionaryUtil.packByteBufferIntoSingleByteArray(noDictionaryStartKey);
composedNonDictEndKey =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index 15c75c1..d3b5605 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -204,6 +204,13 @@ public class CarbonFactDataHandlerModel {
CarbonDataProcessorUtil.getIsUseInvertedIndex(configuration.getDataFields());
int[] dimLensWithComplex = configuration.getCardinalityFinder().getCardinality();
+ if (!configuration.isSortTable()) {
+ for (int i = 0; i < dimLensWithComplex.length; i++) {
+ if (dimLensWithComplex[i] != 0) {
+ dimLensWithComplex[i] = Integer.MAX_VALUE;
+ }
+ }
+ }
List<Integer> dimsLenList = new ArrayList<Integer>();
for (int eachDimLen : dimLensWithComplex) {
if (eachDimLen != 0) dimsLenList.add(eachDimLen);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java b/processing/src/main/java/org/apache/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java
index 68f9bd5..f8454f1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java
@@ -101,9 +101,11 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
*/
private boolean[] isNoDictionaryColumn;
+ private boolean[] isNoDictionarySortColumn;
+
public SingleThreadFinalSortFilesMerger(String tempFileLocation, String tableName,
int dimensionCount, int complexDimensionCount, int measureCount, int noDictionaryCount,
- char[] aggType, boolean[] isNoDictionaryColumn) {
+ char[] aggType, boolean[] isNoDictionaryColumn, boolean[] isNoDictionarySortColumn) {
this.tempFileLocation = tempFileLocation;
this.tableName = tableName;
this.dimensionCount = dimensionCount;
@@ -112,6 +114,7 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
this.aggType = aggType;
this.noDictionaryCount = noDictionaryCount;
this.isNoDictionaryColumn = isNoDictionaryColumn;
+ this.isNoDictionarySortColumn = isNoDictionarySortColumn;
}
/**
@@ -180,7 +183,8 @@ public class SingleThreadFinalSortFilesMerger extends CarbonIterator<Object[]> {
// create chunk holder
SortTempFileChunkHolder sortTempFileChunkHolder =
new SortTempFileChunkHolder(tempFile, dimensionCount, complexDimensionCount,
- measureCount, fileBufferSize, noDictionaryCount, aggType, isNoDictionaryColumn);
+ measureCount, fileBufferSize, noDictionaryCount, aggType, isNoDictionaryColumn,
+ isNoDictionarySortColumn);
// initialize
sortTempFileChunkHolder.initialize();
[07/50] [abbrv] incubator-carbondata git commit: fix jira
CARBONDATA-975: remove unreasonable code
Posted by ja...@apache.org.
fix jira CARBONDATA-975: remove unreasonable code
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/226c4f29
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/226c4f29
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/226c4f29
Branch: refs/heads/12-dev
Commit: 226c4f2958507de4f232c336af9f8c2414ec6c7c
Parents: d3abcd9
Author: mayun <si...@163.com>
Authored: Mon Apr 24 14:24:51 2017 +0800
Committer: mayun <si...@163.com>
Committed: Mon Apr 24 14:24:51 2017 +0800
----------------------------------------------------------------------
.../apache/spark/sql/execution/command/carbonTableSchema.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/226c4f29/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 5108df8..e03cbe5 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -266,7 +266,7 @@ class AlterTableColumnSchemaGenerator(
columnSchema.setDataType(dataType)
columnSchema.setColumnName(colName)
if (alterTableModel.highCardinalityDims.contains(colName)) {
- encoders.remove(encoders.remove(Encoding.DICTIONARY))
+ encoders.remove(Encoding.DICTIONARY)
}
if (dataType == DataType.TIMESTAMP || dataType == DataType.DATE) {
encoders.add(Encoding.DIRECT_DICTIONARY)
@@ -331,7 +331,7 @@ class TableNewProcessor(cm: TableModel) {
columnSchema.setColumnName(colName)
val highCardinalityDims = cm.highcardinalitydims.getOrElse(Seq())
if (highCardinalityDims.contains(colName)) {
- encoders.remove(encoders.remove(Encoding.DICTIONARY))
+ encoders.remove(Encoding.DICTIONARY)
}
if (dataType == DataType.TIMESTAMP || dataType == DataType.DATE) {
encoders.add(Encoding.DIRECT_DICTIONARY)
[08/50] [abbrv] incubator-carbondata git commit: [CARBONDATA-975]
remove unreasonable code This closes #840
Posted by ja...@apache.org.
[CARBONDATA-975] remove unreasonable code This closes #840
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/060b455a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/060b455a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/060b455a
Branch: refs/heads/12-dev
Commit: 060b455a845ac2e4ead3a809f5b1e6eb87e49c3a
Parents: d3abcd9 226c4f2
Author: chenliang613 <ch...@huawei.com>
Authored: Mon Apr 24 22:41:51 2017 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Mon Apr 24 22:41:51 2017 +0800
----------------------------------------------------------------------
.../apache/spark/sql/execution/command/carbonTableSchema.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[04/50] [abbrv] incubator-carbondata git commit: Update faq.md
Posted by ja...@apache.org.
Update faq.md
Update faq.md
Update faq.md
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/9d70dff8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/9d70dff8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/9d70dff8
Branch: refs/heads/12-dev
Commit: 9d70dff81921395109fcb417b766c4cf5e3ecdbc
Parents: 0a11c2a
Author: chenerlu <ch...@huawei.com>
Authored: Thu Apr 20 21:16:42 2017 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Sat Apr 22 10:52:01 2017 +0800
----------------------------------------------------------------------
docs/faq.md | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 55 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9d70dff8/docs/faq.md
----------------------------------------------------------------------
diff --git a/docs/faq.md b/docs/faq.md
index 57ac171..3b22260 100644
--- a/docs/faq.md
+++ b/docs/faq.md
@@ -26,6 +26,7 @@
* [How to specify store location while creating carbon session?](#how-to-specify-store-location-while-creating-carbon-session)
* [What is Carbon Lock Type?](#what-is-carbon-lock-type)
* [How to resolve Abstract Method Error?](#how-to-resolve-abstract-method-error)
+* [How Carbon will behave when execute insert operation in abnormal scenarios?](#how-carbon-will-behave-when-execute-insert-operation-in-abnormal-scenarios)
## What are Bad Records?
Records that fail to get loaded into the CarbonData due to data type incompatibility or are empty or have incompatible format are classified as Bad Records.
@@ -74,4 +75,58 @@ The property carbon.lock.type configuration specifies the type of lock to be acq
## How to resolve Abstract Method Error?
In order to build CarbonData project it is necessary to specify the spark profile. The spark profile sets the Spark Version. You need to specify the ``spark version`` while using Maven to build project.
+## How Carbon will behave when execute insert operation in abnormal scenarios?
+Carbon support insert operation, you can refer to the syntax mentioned in [DML Operations on CarbonData](http://carbondata.apache.org/dml-operation-on-carbondata).
+First, create a soucre table in spark-sql and load data into this created table.
+```
+CREATE TABLE source_table(
+id String,
+name String,
+city String)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ",";
+```
+```
+SELECT * FROM source_table;
+id name city
+1 jack beijing
+2 erlu hangzhou
+3 davi shenzhen
+```
+**Scenario 1** :
+
+Suppose, the column order in carbon table is different from source table, use script "SELECT * FROM carbon table" to query, will get the column order similar as source table, rather than in carbon table's column order as expected.
+```
+CREATE TABLE IF NOT EXISTS carbon_table(
+id String,
+city String,
+name String)
+STORED BY 'carbondata';
+```
+```
+INSERT INTO TABLE carbon_table SELECT * FROM source_table;
+```
+```
+SELECT * FROM carbon_table;
+id city name
+1 jack beijing
+2 erlu hangzhou
+3 davi shenzhen
+```
+As result shows, the second column is city in carbon table, but what inside is name, such as jack. This phenomenon is same with insert data into hive table.
+
+If you want to insert data into corresponding column in carbon table, you have to specify the column order same in insert statment.
+```
+INSERT INTO TABLE carbon_table SELECT id, city, name FROM source_table;
+```
+
+**Scenario 2** :
+
+Insert operation will be failed when the number of column in carbon table is different from the column specified in select statement. The following insert operation will be failed.
+```
+INSERT INTO TABLE carbon_table SELECT id, city FROM source_table;
+```
+**Scenario 3** :
+
+When the column type in carbon table is different from from the column specified in select statement. The insert operation will still success, but you may get NULL in result, because NULL will be substitute value when conversion type failed.
+
[41/50] [abbrv] incubator-carbondata git commit: [CARBONDATA-987] Can
not delete lock file when drop table. This closes #851
Posted by ja...@apache.org.
[CARBONDATA-987] Can not delete lock file when drop table. This closes #851
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/1edabc87
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/1edabc87
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/1edabc87
Branch: refs/heads/12-dev
Commit: 1edabc87a224cf792f2f933ccf46da5fa18a549d
Parents: 3252e7c b731123
Author: ravipesala <ra...@gmail.com>
Authored: Mon May 1 21:24:32 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Mon May 1 21:24:32 2017 +0530
----------------------------------------------------------------------
.../java/org/apache/carbondata/core/locks/LocalFileLock.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[31/50] [abbrv] incubator-carbondata git commit: update the examples
Posted by ja...@apache.org.
update the examples
just update examples
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/2e8ef57d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/2e8ef57d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/2e8ef57d
Branch: refs/heads/12-dev
Commit: 2e8ef57d5a72e7c35fbf851ca28f114501028d17
Parents: b728b8f
Author: chenerlu <ch...@huawei.com>
Authored: Thu Apr 27 19:20:09 2017 +0800
Committer: chenliang613 <ch...@huawei.com>
Committed: Fri Apr 28 17:12:57 2017 +0800
----------------------------------------------------------------------
.../carbondata/examples/AlterTableExample.scala | 32 ++--
.../examples/CarbonDataFrameExample.scala | 4 +-
.../examples/CarbonSessionExample.scala | 114 +++++++-------
.../examples/SparkSessionExample.scala | 156 +++++++++----------
4 files changed, 158 insertions(+), 148 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2e8ef57d/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
index f37a83d..dd2a28a 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/AlterTableExample.scala
@@ -24,6 +24,10 @@ import org.apache.spark.sql.SparkSession
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
+/**
+ * For alter table relative syntax, you can refer to DDL operation
+ * document (ddl-operation-on-carbondata.md)
+ */
object AlterTableExample {
def main(args: Array[String]): Unit = {
@@ -55,17 +59,17 @@ object AlterTableExample {
spark.sql(
s"""
| CREATE TABLE carbon_table(
- | shortField short,
- | intField int,
- | bigintField long,
- | doubleField double,
- | stringField string,
- | timestampField timestamp,
- | decimalField decimal(18,2),
- | dateField date,
- | charField char(5),
- | floatField float,
- | complexData array<string>
+ | shortField SHORT,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>
| )
| STORED BY 'carbondata'
| TBLPROPERTIES('DICTIONARY_INCLUDE'='dateField, charField')
@@ -73,11 +77,11 @@ object AlterTableExample {
// Alter table change data type
spark.sql("DESCRIBE FORMATTED carbon_table").show()
- spark.sql("ALTER TABLE carbon_table CHANGE intField intField bigint").show()
+ spark.sql("ALTER TABLE carbon_table CHANGE intField intField BIGINT").show()
// Alter table add columns
spark.sql("DESCRIBE FORMATTED carbon_table").show()
- spark.sql("ALTER TABLE carbon_table ADD COLUMNS (newField String) " +
+ spark.sql("ALTER TABLE carbon_table ADD COLUMNS (newField STRING) " +
"TBLPROPERTIES ('DEFAULT.VALUE.newField'='def')").show()
// Alter table drop columns
@@ -93,5 +97,7 @@ object AlterTableExample {
spark.sql("DROP TABLE IF EXISTS carbon_table")
spark.sql("DROP TABLE IF EXISTS new_carbon_table")
+ spark.stop()
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2e8ef57d/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
index e4d1646..74740e7 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
@@ -24,7 +24,6 @@ import org.apache.spark.sql.{SaveMode, SparkSession}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
-// scalastyle:off println
object CarbonDataFrameExample {
def main(args: Array[String]) {
@@ -84,6 +83,7 @@ object CarbonDataFrameExample {
carbondf.filter($"number" > 31).show()
spark.sql("DROP TABLE IF EXISTS carbon_table")
+
+ spark.stop()
}
}
-// scalastyle:on println
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2e8ef57d/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
index 2ae9341..371b9bb 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
@@ -53,17 +53,17 @@ object CarbonSessionExample {
spark.sql(
s"""
| CREATE TABLE carbon_table(
- | shortField short,
- | intField int,
- | bigintField long,
- | doubleField double,
- | stringField string,
- | timestampField timestamp,
- | decimalField decimal(18,2),
- | dateField date,
- | charField char(5),
- | floatField float,
- | complexData array<string>
+ | shortField SHORT,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5),
+ | floatField FLOAT,
+ | complexData ARRAY<STRING>
| )
| STORED BY 'carbondata'
| TBLPROPERTIES('DICTIONARY_INCLUDE'='dateField, charField')
@@ -76,63 +76,69 @@ object CarbonSessionExample {
s"""
| LOAD DATA LOCAL INPATH '$path'
| INTO TABLE carbon_table
- | options('FILEHEADER'='shortField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData','COMPLEX_DELIMITER_LEVEL_1'='#')
+ | OPTIONS('FILEHEADER'='shortField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData',
+ | 'COMPLEX_DELIMITER_LEVEL_1'='#')
""".stripMargin)
// scalastyle:on
- spark.sql("""
- SELECT *
- FROM carbon_table
- where stringfield = 'spark' and decimalField > 40
- """).show
+ spark.sql(
+ s"""
+ | SELECT *
+ | FROM carbon_table
+ | WHERE stringfield = 'spark' AND decimalField > 40
+ """.stripMargin).show()
- spark.sql("""
- SELECT *
- FROM carbon_table where length(stringField) = 5
- """).show
+ spark.sql(
+ s"""
+ | SELECT *
+ | FROM carbon_table WHERE length(stringField) = 5
+ """.stripMargin).show()
+
+ spark.sql(
+ s"""
+ | SELECT *
+ | FROM carbon_table WHERE date_format(dateField, "yyyy-MM-dd") = "2015-07-23"
+ """.stripMargin).show()
- spark.sql("""
- SELECT *
- FROM carbon_table where date_format(dateField, "yyyy-MM-dd") = "2015-07-23"
- """).show
+ spark.sql("SELECT count(stringField) FROM carbon_table").show()
- spark.sql("""
- select count(stringField) from carbon_table
- """.stripMargin).show
+ spark.sql(
+ s"""
+ | SELECT sum(intField), stringField
+ | FROM carbon_table
+ | GROUP BY stringField
+ """.stripMargin).show()
- spark.sql("""
- SELECT sum(intField), stringField
- FROM carbon_table
- GROUP BY stringField
- """).show
+ spark.sql(
+ s"""
+ | SELECT t1.*, t2.*
+ | FROM carbon_table t1, carbon_table t2
+ | WHERE t1.stringField = t2.stringField
+ """.stripMargin).show()
spark.sql(
- """
- |select t1.*, t2.*
- |from carbon_table t1, carbon_table t2
- |where t1.stringField = t2.stringField
- """.stripMargin).show
+ s"""
+ | WITH t1 AS (
+ | SELECT * FROM carbon_table
+ | UNION ALL
+ | SELECT * FROM carbon_table
+ | )
+ | SELECT t1.*, t2.*
+ | FROM t1, carbon_table t2
+ | WHERE t1.stringField = t2.stringField
+ """.stripMargin).show()
spark.sql(
- """
- |with t1 as (
- |select * from carbon_table
- |union all
- |select * from carbon_table
- |)
- |select t1.*, t2.*
- |from t1, carbon_table t2
- |where t1.stringField = t2.stringField
- """.stripMargin).show
-
- spark.sql("""
- SELECT *
- FROM carbon_table
- where stringfield = 'spark' and floatField > 2.8
- """).show
+ s"""
+ | SELECT *
+ | FROM carbon_table
+ | WHERE stringField = 'spark' and floatField > 2.8
+ """.stripMargin).show()
// Drop table
spark.sql("DROP TABLE IF EXISTS carbon_table")
+
+ spark.stop()
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/2e8ef57d/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkSessionExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkSessionExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkSessionExample.scala
index 63e680a..b823be5 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkSessionExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/SparkSessionExample.scala
@@ -30,7 +30,7 @@ object SparkSessionExample {
def main(args: Array[String]): Unit = {
val rootPath = new File(this.getClass.getResource("/").getPath
- + "../../../..").getCanonicalPath
+ + "../../../..").getCanonicalPath
val storeLocation = s"$rootPath/examples/spark2/target/store"
val warehouse = s"$rootPath/examples/spark2/target/warehouse"
val metastoredb = s"$rootPath/examples/spark2/target/metastore_db"
@@ -44,14 +44,14 @@ object SparkSessionExample {
}
val spark = SparkSession
- .builder()
- .master("local")
- .appName("SparkSessionExample")
- .enableHiveSupport()
- .config("spark.sql.warehouse.dir", warehouse)
- .config("javax.jdo.option.ConnectionURL",
- s"jdbc:derby:;databaseName=$metastoredb;create=true")
- .getOrCreate()
+ .builder()
+ .master("local")
+ .appName("SparkSessionExample")
+ .enableHiveSupport()
+ .config("spark.sql.warehouse.dir", warehouse)
+ .config("javax.jdo.option.ConnectionURL",
+ s"jdbc:derby:;databaseName=$metastoredb;create=true")
+ .getOrCreate()
CarbonProperties.getInstance()
.addProperty("carbon.storelocation", storeLocation)
@@ -59,46 +59,43 @@ object SparkSessionExample {
spark.sparkContext.setLogLevel("WARN")
CarbonProperties.getInstance()
- .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd HH:mm:ss")
- .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
+ .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd HH:mm:ss")
+ .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "yyyy/MM/dd")
// Create table
spark.sql(
s"""
| CREATE TABLE carbon_table(
- | shortField short,
- | intField int,
- | bigintField long,
- | doubleField double,
- | stringField string,
- | timestampField timestamp,
- | decimalField decimal(18,2),
- | dateField date,
- | charField char(5)
+ | shortField SHORT,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField TIMESTAMP,
+ | decimalField DECIMAL(18,2),
+ | dateField DATE,
+ | charField CHAR(5)
| )
| USING org.apache.spark.sql.CarbonSource
| OPTIONS('DICTIONARY_INCLUDE'='dateField, charField',
- | 'dbName'='default', 'tableName'='carbon_table')
+ | 'dbName'='default', 'tableName'='carbon_table')
""".stripMargin)
- // val prop = s"$rootPath/conf/dataload.properties.template"
- // val tableName = "carbon_table"
val path = s"$rootPath/examples/spark2/src/main/resources/data.csv"
- // TableLoader.main(Array[String](prop, tableName, path))
spark.sql(
s"""
- | CREATE TABLE csv_table
- | ( shortField short,
- | intField int,
- | bigintField long,
- | doubleField double,
- | stringField string,
- | timestampField string,
- | decimalField decimal(18,2),
- | dateField string,
- | charField char(5))
- | ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ | CREATE TABLE csv_table(
+ | shortField SHORT,
+ | intField INT,
+ | bigintField LONG,
+ | doubleField DOUBLE,
+ | stringField STRING,
+ | timestampField STRING,
+ | decimalField DECIMAL(18,2),
+ | dateField STRING,
+ | charField CHAR(5))
+ | ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
""".stripMargin)
spark.sql(
@@ -107,10 +104,7 @@ object SparkSessionExample {
| INTO TABLE csv_table
""".stripMargin)
- spark.sql("""
- SELECT *
- FROM csv_table
- """).show
+ spark.sql("SELECT * FROM csv_table").show()
spark.sql(
s"""
@@ -121,56 +115,60 @@ object SparkSessionExample {
| FROM csv_table
""".stripMargin)
- spark.sql("""
- SELECT *
- FROM carbon_table
- where stringfield = 'spark' and decimalField > 40
- """).show
+ spark.sql(
+ s"""
+ | SELECT *
+ | FROM carbon_table
+ | WHERE stringfield = 'spark' AND decimalField > 40
+ """.stripMargin).show()
// Shows with raw data's timestamp format
- spark.sql("""
- SELECT
- stringField, date_format(timestampField, "yyyy/MM/dd HH:mm:ss") as timestampField
- FROM carbon_table where length(stringField) = 5
- """).show
-
- spark.sql("""
- SELECT *
- FROM carbon_table where date_format(dateField, "yyyy-MM-dd") = "2015-07-23"
- """).show
-
- spark.sql("""
- select count(stringField) from carbon_table
- """.stripMargin).show
-
- spark.sql("""
- SELECT sum(intField), stringField
- FROM carbon_table
- GROUP BY stringField
- """).show
+ spark.sql(
+ s"""
+ | SELECT
+ | stringField, date_format(timestampField, "yyyy/MM/dd HH:mm:ss") AS
+ | timestampField
+ | FROM carbon_table WHERE length(stringField) = 5
+ """.stripMargin).show()
spark.sql(
- """
- |select t1.*, t2.*
- |from carbon_table t1, carbon_table t2
- |where t1.stringField = t2.stringField
- """.stripMargin).show
+ s"""
+ | SELECT *
+ | FROM carbon_table where date_format(dateField, "yyyy-MM-dd") = "2015-07-23"
+ """.stripMargin).show()
+
+ spark.sql("SELECT count(stringField) FROM carbon_table").show()
spark.sql(
- """
- |with t1 as (
- |select * from carbon_table
- |union all
- |select * from carbon_table
- |)
- |select t1.*, t2.*
- |from t1, carbon_table t2
- |where t1.stringField = t2.stringField
- """.stripMargin).show
+ s"""
+ | SELECT sum(intField), stringField
+ | FROM carbon_table
+ | GROUP BY stringField
+ """.stripMargin).show()
+
+ spark.sql(
+ s"""
+ | SELECT t1.*, t2.*
+ | FROM carbon_table t1, carbon_table t2
+ | WHERE t1.stringField = t2.stringField
+ """.stripMargin).show()
+
+ spark.sql(
+ s"""
+ | WITH t1 AS (
+ | SELECT * FROM carbon_table
+ | UNION ALL
+ | SELECT * FROM carbon_table
+ | )
+ | SELECT t1.*, t2.*
+ | FROM t1, carbon_table t2
+ | WHERE t1.stringField = t2.stringField
+ """.stripMargin).show()
// Drop table
spark.sql("DROP TABLE IF EXISTS carbon_table")
spark.sql("DROP TABLE IF EXISTS csv_table")
+
spark.stop()
}
}
[02/50] [abbrv] incubator-carbondata git commit: Improve Carbon index
file loading file in case of big cluster
Posted by ja...@apache.org.
Improve Carbon index file loading file in case of big cluster
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8fc9c7bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8fc9c7bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8fc9c7bf
Branch: refs/heads/12-dev
Commit: 8fc9c7bf67c1e5678cfa955266dfe005fed4a5d5
Parents: 174bfad
Author: kumarvishal <ku...@gmail.com>
Authored: Fri Apr 21 20:42:10 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Sat Apr 22 00:30:32 2017 +0530
----------------------------------------------------------------------
.../apache/carbondata/core/util/CarbonUtil.java | 8 +++--
.../core/util/path/CarbonTablePath.java | 33 ++++++++++++++++++++
2 files changed, 39 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8fc9c7bf/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 7d57fb8..8956549 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -990,7 +990,9 @@ public final class CarbonUtil {
//TODO need to pass proper partition number when partiton will be supported
String carbonIndexFilePath = carbonTablePath
.getCarbonIndexFilePath(taskId, "0", tableBlockInfoList.get(0).getSegmentId(),
- bucketNumber);
+ bucketNumber, CarbonTablePath.DataFileUtil
+ .getTimeStampFromFileName(tableBlockInfoList.get(0).getFilePath()),
+ tableBlockInfoList.get(0).getVersion());
CarbonFile carbonFile = FileFactory
.getCarbonFile(carbonIndexFilePath, FileFactory.getFileType(carbonIndexFilePath));
// in case of carbonIndex file whole file is meta only so reading complete file.
@@ -1232,7 +1234,9 @@ public final class CarbonUtil {
//TODO need to pass proper partition number when partiton will be supported
String carbonIndexFilePath = carbonTablePath
.getCarbonIndexFilePath(taskId, "0", tableBlockInfoList.get(0).getSegmentId(),
- bucketNumber);
+ bucketNumber, CarbonTablePath.DataFileUtil
+ .getTimeStampFromFileName(tableBlockInfoList.get(0).getFilePath()),
+ tableBlockInfoList.get(0).getVersion());
DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
// read the index info and return
return fileFooterConverter.getIndexInfo(carbonIndexFilePath, tableBlockInfoList);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8fc9c7bf/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index 3e81fbb..918772a 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -23,6 +23,7 @@ import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.hadoop.fs.Path;
@@ -274,6 +275,38 @@ public class CarbonTablePath extends Path {
+ "]");
}
}
+
+ /**
+ * Below method will be used to get the carbon index file path
+ * @param taskId
+ * task id
+ * @param partitionId
+ * partition id
+ * @param segmentId
+ * segment id
+ * @param bucketNumber
+ * bucket number
+ * @param timeStamp
+ * timestamp
+ * @return carbon index file path
+ */
+ public String getCarbonIndexFilePath(String taskId, String partitionId, String segmentId,
+ String bucketNumber, String timeStamp, ColumnarFormatVersion columnarFormatVersion) {
+ switch (columnarFormatVersion) {
+ case V1:
+ case V2:
+ return getCarbonIndexFilePath(taskId, partitionId, segmentId, bucketNumber);
+ default:
+ String segmentDir = getSegmentDir(partitionId, segmentId);
+ return segmentDir + File.separator + getCarbonIndexFileName(taskId,
+ Integer.parseInt(bucketNumber), timeStamp);
+ }
+ }
+
+ private static String getCarbonIndexFileName(String taskNo, int bucketNumber,
+ String factUpdatedtimeStamp) {
+ return taskNo + "-" + bucketNumber + "-" + factUpdatedtimeStamp + INDEX_FILE_EXT;
+ }
/**
* Below method will be used to get the index file present in the segment folder
* based on task id
[43/50] [abbrv] incubator-carbondata git commit: add SORT_COLUMNS
options
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
new file mode 100644
index 0000000..88b0421
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.sortcolumns
+
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+class TestSortColumns extends QueryTest with BeforeAndAfterAll {
+
+ override def beforeAll {
+ dropTable
+
+ sql("CREATE TABLE origintable1 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format'")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ }
+
+ test("create table with no dictionary sort_columns") {
+ sql("CREATE TABLE sorttable1 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='empno')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select empno from sorttable1"), sql("select empno from sorttable1 order by empno"))
+ }
+
+ test("create table with dictionary sort_columns") {
+ sql("CREATE TABLE sorttable2 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='empname')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable2 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select empname from sorttable2"),sql("select empname from origintable1"))
+ }
+
+ test("create table with direct-dictioanry sort_columns") {
+ sql("CREATE TABLE sorttable3 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='doj')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable3 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select doj from sorttable3"), sql("select doj from sorttable3 order by doj"))
+ }
+
+ test("create table with multi-sort_columns and data loading with offheap safe") {
+ try {
+ setLoadingProperties("true", "false", "false")
+ sql("CREATE TABLE sorttable4_offheap_safe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='workgroupcategory, empname')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable4_offheap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select workgroupcategory, empname from sorttable4_offheap_safe"), sql("select workgroupcategory, empname from origintable1 order by workgroupcategory"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("create table with multi-sort_columns and data loading with offheap and unsafe sort") {
+ try {
+ setLoadingProperties("true", "true", "false")
+ sql(
+ "CREATE TABLE sorttable4_offheap_unsafe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='workgroupcategory, empname')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable4_offheap_unsafe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select workgroupcategory, empname from sorttable4_offheap_unsafe"),
+ sql("select workgroupcategory, empname from origintable1 order by workgroupcategory"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("create table with multi-sort_columns and data loading with offheap and inmemory sort") {
+ try {
+ setLoadingProperties("true", "false", "true")
+ sql(
+ "CREATE TABLE sorttable4_offheap_inmemory (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='workgroupcategory, empname')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable4_offheap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select workgroupcategory, empname from sorttable4_offheap_inmemory"),
+ sql("select workgroupcategory, empname from origintable1 order by workgroupcategory"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("create table with multi-sort_columns and data loading with heap") {
+ try {
+ setLoadingProperties("false", "false", "false")
+ sql(
+ "CREATE TABLE sorttable4_heap_safe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='workgroupcategory, empname')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable4_heap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select workgroupcategory, empname from sorttable4_heap_safe"),
+ sql("select workgroupcategory, empname from origintable1 order by workgroupcategory"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("create table with multi-sort_columns and data loading with heap and unsafe sort") {
+ try {
+ setLoadingProperties("false", "true", "false")
+ sql(
+ "CREATE TABLE sorttable4_heap_unsafe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='workgroupcategory, empname')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable4_heap_unsafe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select workgroupcategory, empname from sorttable4_heap_unsafe"),
+ sql("select workgroupcategory, empname from origintable1 order by workgroupcategory"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("create table with multi-sort_columns and data loading with heap and inmemory sort") {
+ try {
+ setLoadingProperties("false", "false", "true")
+ sql(
+ "CREATE TABLE sorttable4_heap_inmemory (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='workgroupcategory, empname')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable4_heap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select workgroupcategory, empname from sorttable4_heap_inmemory"),
+ sql("select workgroupcategory, empname from origintable1 order by workgroupcategory"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("compaction on sort_columns table") {
+ sql("CREATE TABLE origintable2 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format'")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable2 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable2 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable2 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE origintable2 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql("alter table origintable2 compact 'minor'")
+
+ sql("CREATE TABLE sorttable5 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='empno')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable5 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable5 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable5 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable5 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ sql("alter table sorttable5 compact 'minor'")
+
+ checkAnswer(sql("select empno from sorttable5"), sql("select empno from origintable2 order by empno"))
+ }
+
+ test("filter on sort_columns include no-dictionary, direct-dictionary and dictioanry") {
+ sql("CREATE TABLE sorttable6 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='workgroupcategory, doj, empname')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable6 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ // no dictionary
+ checkAnswer(sql("select * from sorttable6 where workgroupcategory = 1"), sql("select * from origintable1 where workgroupcategory = 1 order by doj"))
+ // direct dictionary
+ checkAnswer(sql("select * from sorttable6 where doj = '2007-01-17 00:00:00'"), sql("select * from origintable1 where doj = '2007-01-17 00:00:00'"))
+ // dictionary
+ checkAnswer(sql("select * from sorttable6 where empname = 'madhan'"), sql("select * from origintable1 where empname = 'madhan'"))
+ }
+
+ test("unsorted table creation, query data loading with heap and safe sort config") {
+ try {
+ setLoadingProperties("false", "false", "false")
+ sql("CREATE TABLE unsortedtable_heap_safe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select * from unsortedtable_heap_safe where empno = 11"), sql("select * from origintable1 where empno = 11"))
+ checkAnswer(sql("select * from unsortedtable_heap_safe order by empno"), sql("select * from origintable1 order by empno"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("unsorted table creation, query and data loading with heap and unsafe sort config") {
+ try {
+ setLoadingProperties("false", "true", "false")
+ sql("CREATE TABLE unsortedtable_heap_unsafe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_unsafe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select * from unsortedtable_heap_unsafe where empno = 11"), sql("select * from origintable1 where empno = 11"))
+ checkAnswer(sql("select * from unsortedtable_heap_unsafe order by empno"), sql("select * from origintable1 order by empno"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("unsorted table creation, query and loading with heap and inmemory sort config") {
+ try {
+ setLoadingProperties("false", "false", "true")
+ sql("CREATE TABLE unsortedtable_heap_inmemory (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select * from unsortedtable_heap_inmemory where empno = 11"), sql("select * from origintable1 where empno = 11"))
+ checkAnswer(sql("select * from unsortedtable_heap_inmemory order by empno"), sql("select * from origintable1 order by empno"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("unsorted table creation, query and data loading with offheap and safe sort config") {
+ try {
+ setLoadingProperties("true", "false", "false")
+ sql("CREATE TABLE unsortedtable_offheap_safe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select * from unsortedtable_offheap_safe where empno = 11"), sql("select * from origintable1 where empno = 11"))
+ checkAnswer(sql("select * from unsortedtable_offheap_safe order by empno"), sql("select * from origintable1 order by empno"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("unsorted table creation, query and data loading with offheap and unsafe sort config") {
+ try {
+ setLoadingProperties("true", "true", "false")
+ sql("CREATE TABLE unsortedtable_offheap_unsafe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_unsafe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select * from unsortedtable_offheap_unsafe where empno = 11"), sql("select * from origintable1 where empno = 11"))
+ checkAnswer(sql("select * from unsortedtable_offheap_unsafe order by empno"), sql("select * from origintable1 order by empno"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("unsorted table creation, query and data loading with offheap and inmemory sort config") {
+ try {
+ setLoadingProperties("true", "false", "true")
+ sql("CREATE TABLE unsortedtable_offheap_inmemory (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno = 11"), sql("select * from origintable1 where empno = 11"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory order by empno"), sql("select * from origintable1 order by empno"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+
+ override def afterAll = {
+ dropTable
+ }
+
+ def dropTable = {
+ sql("drop table if exists origintable1")
+ sql("drop table if exists origintable2")
+ sql("drop table if exists sorttable1")
+ sql("drop table if exists sorttable2")
+ sql("drop table if exists sorttable3")
+ sql("drop table if exists sorttable4_offheap_safe")
+ sql("drop table if exists sorttable4_offheap_unsafe")
+ sql("drop table if exists sorttable4_offheap_inmemory")
+ sql("drop table if exists sorttable4_heap_safe")
+ sql("drop table if exists sorttable4_heap_unsafe")
+ sql("drop table if exists sorttable4_heap_inmemory")
+ sql("drop table if exists sorttable5")
+ sql("drop table if exists sorttable6")
+ sql("drop table if exists unsortedtable_offheap_safe")
+ sql("drop table if exists unsortedtable_offheap_unsafe")
+ sql("drop table if exists unsortedtable_offheap_inmemory")
+ sql("drop table if exists unsortedtable_heap_safe")
+ sql("drop table if exists unsortedtable_heap_unsafe")
+ sql("drop table if exists unsortedtable_heap_inmemory")
+ }
+
+ def setLoadingProperties(offheap: String, unsafe: String, useBatch: String): Unit = {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, offheap)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT, unsafe)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.LOAD_USE_BATCH_SORT, useBatch)
+ }
+
+ def defaultLoadingProperties = {
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT, CarbonCommonConstants.ENABLE_OFFHEAP_SORT)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT, CarbonCommonConstants.ENABLE_UNSAFE_SORT_DEFAULT)
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.LOAD_USE_BATCH_SORT, CarbonCommonConstants.LOAD_USE_BATCH_SORT_DEFAULT)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index dad5716..23d1b58 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -33,7 +33,7 @@ import org.apache.spark.sql.execution.command._
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.datatype.DataType
-import org.apache.carbondata.core.util.DataTypeUtil
+import org.apache.carbondata.core.util.{CarbonUtil, DataTypeUtil}
import org.apache.carbondata.processing.constants.LoggerAction
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.util.CommonUtil
@@ -240,21 +240,15 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
fields.zipWithIndex.foreach { x =>
x._1.schemaOrdinal = x._2
}
- val (dims: Seq[Field], noDictionaryDims: Seq[String]) = extractDimColsAndNoDictionaryFields(
+ val (dims, msrs, noDictionaryDims, sortKeyDims) = extractDimAndMsrFields(
fields, tableProperties)
if (dims.isEmpty && !isAlterFlow) {
- throw new MalformedCarbonCommandException(s"Table ${
- dbName.getOrElse(
- CarbonCommonConstants.DATABASE_DEFAULT_NAME)
- }.$tableName"
- +
- " can not be created without key columns. Please " +
- "use DICTIONARY_INCLUDE or " +
- "DICTIONARY_EXCLUDE to set at least one key " +
- "column " +
- "if all specified columns are numeric types")
+ throw new MalformedCarbonCommandException(
+ s"Table ${dbName.getOrElse(CarbonCommonConstants.DATABASE_DEFAULT_NAME)}.$tableName " +
+ "can not be created without key columns. Please use DICTIONARY_INCLUDE or " +
+ "DICTIONARY_EXCLUDE to set at least one key column " +
+ "if all specified columns are numeric types")
}
- val msrs: Seq[Field] = extractMsrColsFromFields(fields, tableProperties)
// column properties
val colProps = extractColumnProperties(fields, tableProperties)
@@ -276,6 +270,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
tableProperties,
reorderDimensions(dims.map(f => normalizeType(f)).map(f => addParent(f))),
msrs.map(f => normalizeType(f)),
+ Option(sortKeyDims),
Option(noDictionaryDims),
Option(noInvertedIdxCols),
groupCols,
@@ -485,14 +480,50 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
* @param tableProperties
* @return
*/
- protected def extractDimColsAndNoDictionaryFields(fields: Seq[Field],
- tableProperties: Map[String, String]):
- (Seq[Field], Seq[String]) = {
+ protected def extractDimAndMsrFields(fields: Seq[Field],
+ tableProperties: Map[String, String]): (Seq[Field], Seq[Field], Seq[String], Seq[String]) = {
var dimFields: LinkedHashSet[Field] = LinkedHashSet[Field]()
+ var msrFields: Seq[Field] = Seq[Field]()
var dictExcludeCols: Array[String] = Array[String]()
var noDictionaryDims: Seq[String] = Seq[String]()
var dictIncludeCols: Seq[String] = Seq[String]()
+ // All columns in sortkey should be there in create table cols
+ val sortKeyOption = tableProperties.get(CarbonCommonConstants.SORT_COLUMNS)
+ var sortKeyDimsTmp: Seq[String] = Seq[String]()
+ val sortKeyString: String = if (sortKeyOption.isDefined) {
+ CarbonUtil.unquoteChar(sortKeyOption.get) trim
+ } else {
+ ""
+ }
+ if (!sortKeyString.isEmpty) {
+ val sortKey = sortKeyString.split(',').map(_.trim)
+ sortKey.foreach { column =>
+ if (!fields.exists(x => x.column.equalsIgnoreCase(column))) {
+ val errormsg = "sort_columns: " + column +
+ " does not exist in table. Please check create table statement."
+ throw new MalformedCarbonCommandException(errormsg)
+ } else {
+ val dataType = fields.find(x =>
+ x.column.equalsIgnoreCase(column)).get.dataType.get
+ if (isComplexDimDictionaryExclude(dataType)) {
+ val errormsg = "sort_columns is unsupported for complex datatype column: " + column
+ throw new MalformedCarbonCommandException(errormsg)
+ }
+ }
+ }
+
+ sortKey.foreach { dimension =>
+ if (!sortKeyDimsTmp.exists(dimension.equalsIgnoreCase(_))) {
+ fields.foreach { field =>
+ if (field.column.equalsIgnoreCase(dimension)) {
+ sortKeyDimsTmp :+= field.column
+ }
+ }
+ }
+ }
+ }
+
// All excluded cols should be there in create table cols
if (tableProperties.get(CarbonCommonConstants.DICTIONARY_EXCLUDE).isDefined) {
dictExcludeCols =
@@ -531,7 +562,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
}
}
- // include cols should contain exclude cols
+ // include cols should not contain exclude cols
dictExcludeCols.foreach { dicExcludeCol =>
if (dictIncludeCols.exists(x => x.equalsIgnoreCase(dicExcludeCol))) {
val errormsg = "DICTIONARY_EXCLUDE can not contain the same column: " + dicExcludeCol +
@@ -553,10 +584,30 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
dimFields += field
} else if (isDetectAsDimentionDatatype(field.dataType.get)) {
dimFields += field
+ } else if (sortKeyDimsTmp.exists(x => x.equalsIgnoreCase(field.column))) {
+ noDictionaryDims :+= field.column
+ dimFields += field
+ } else {
+ msrFields :+= field
}
}
- (dimFields.toSeq, noDictionaryDims)
+ var sortKeyDims = sortKeyDimsTmp
+ if (sortKeyOption.isEmpty) {
+ // if SORT_COLUMNS was not defined, add all dimension to SORT_COLUMNS.
+ dimFields.foreach { field =>
+ if (!isComplexDimDictionaryExclude(field.dataType.get)) {
+ sortKeyDims :+= field.column
+ }
+ }
+ }
+ if (sortKeyDims.isEmpty) {
+ // no SORT_COLUMNS
+ tableProperties.put(CarbonCommonConstants.SORT_COLUMNS, "")
+ } else {
+ tableProperties.put(CarbonCommonConstants.SORT_COLUMNS, sortKeyDims.mkString(","))
+ }
+ (dimFields.toSeq, msrFields, noDictionaryDims, sortKeyDims)
}
/**
@@ -603,44 +654,6 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
}
/**
- * Extract the Measure Cols fields. By default all non string cols will be measures.
- *
- * @param fields
- * @param tableProperties
- * @return
- */
- protected def extractMsrColsFromFields(fields: Seq[Field],
- tableProperties: Map[String, String]): Seq[Field] = {
- var msrFields: Seq[Field] = Seq[Field]()
- var dictIncludedCols: Array[String] = Array[String]()
- var dictExcludedCols: Array[String] = Array[String]()
-
- // get all included cols
- if (tableProperties.get(CarbonCommonConstants.DICTIONARY_INCLUDE).isDefined) {
- dictIncludedCols =
- tableProperties.get(CarbonCommonConstants.DICTIONARY_INCLUDE).get.split(',').map(_.trim)
- }
-
- // get all excluded cols
- if (tableProperties.get(CarbonCommonConstants.DICTIONARY_EXCLUDE).isDefined) {
- dictExcludedCols =
- tableProperties.get(CarbonCommonConstants.DICTIONARY_EXCLUDE).get.split(',').map(_.trim)
- }
-
- // by default consider all non string cols as msrs. consider all include/ exclude cols as dims
- fields.foreach(field => {
- if (!isDetectAsDimentionDatatype(field.dataType.get)) {
- if (!dictIncludedCols.exists(x => x.equalsIgnoreCase(field.column)) &&
- !dictExcludedCols.exists(x => x.equalsIgnoreCase(field.column))) {
- msrFields :+= field
- }
- }
- })
-
- msrFields
- }
-
- /**
* Extract the DbName and table name.
*
* @param tableNameParts
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index e03cbe5..187512d 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -54,6 +54,7 @@ case class TableModel(
tableProperties: Map[String, String],
dimCols: Seq[Field],
msrCols: Seq[Field],
+ sortKeyDims: Option[Seq[String]],
highcardinalitydims: Option[Seq[String]],
noInvertedIdxCols: Option[Seq[String]],
columnGroups: Seq[String],
@@ -348,6 +349,7 @@ class TableNewProcessor(cm: TableModel) {
columnSchema.setPrecision(precision)
columnSchema.setScale(scale)
columnSchema.setSchemaOrdinal(schemaOrdinal)
+ columnSchema.setSortColumn(false)
// TODO: Need to fill RowGroupID, converted type
// & Number of Children after DDL finalization
columnSchema
@@ -358,7 +360,11 @@ class TableNewProcessor(cm: TableModel) {
val LOGGER = LogServiceFactory.getLogService(TableNewProcessor.getClass.getName)
var allColumns = Seq[ColumnSchema]()
var index = 0
- cm.dimCols.foreach(field => {
+ var measureCount = 0
+
+ // Sort columns should be at the begin of all columns
+ cm.sortKeyDims.get.foreach { keyDim =>
+ val field = cm.dimCols.find(keyDim equals _.column).get
val encoders = new java.util.ArrayList[Encoding]()
encoders.add(Encoding.DICTIONARY)
val columnSchema: ColumnSchema = getColumnSchema(
@@ -372,11 +378,33 @@ class TableNewProcessor(cm: TableModel) {
field.precision,
field.scale,
field.schemaOrdinal)
- allColumns ++= Seq(columnSchema)
+ columnSchema.setSortColumn(true)
+ allColumns :+= columnSchema
index = index + 1
- if (field.children.isDefined && field.children.get != null) {
- columnSchema.setNumberOfChild(field.children.get.size)
- allColumns ++= getAllChildren(field.children)
+ }
+
+ cm.dimCols.foreach(field => {
+ val sortField = cm.sortKeyDims.get.find(field.column equals _)
+ if (sortField.isEmpty) {
+ val encoders = new java.util.ArrayList[Encoding]()
+ encoders.add(Encoding.DICTIONARY)
+ val columnSchema: ColumnSchema = getColumnSchema(
+ DataTypeConverterUtil.convertToCarbonType(field.dataType.getOrElse("")),
+ field.name.getOrElse(field.column),
+ index,
+ isCol = true,
+ encoders,
+ isDimensionCol = true,
+ -1,
+ field.precision,
+ field.scale,
+ field.schemaOrdinal)
+ allColumns :+= columnSchema
+ index = index + 1
+ if (field.children.isDefined && field.children.get != null) {
+ columnSchema.setNumberOfChild(field.children.get.size)
+ allColumns ++= getAllChildren(field.children)
+ }
}
})
@@ -393,10 +421,9 @@ class TableNewProcessor(cm: TableModel) {
field.precision,
field.scale,
field.schemaOrdinal)
- val measureCol = columnSchema
-
- allColumns ++= Seq(measureCol)
+ allColumns :+= columnSchema
index = index + 1
+ measureCount += 1
})
// Check if there is any duplicate measures or dimensions.
@@ -417,22 +444,6 @@ class TableNewProcessor(cm: TableModel) {
updateColumnGroupsInFields(cm.columnGroups, allColumns)
- var newOrderedDims = scala.collection.mutable.ListBuffer[ColumnSchema]()
- val complexDims = scala.collection.mutable.ListBuffer[ColumnSchema]()
- val measures = scala.collection.mutable.ListBuffer[ColumnSchema]()
- for (column <- allColumns) {
- if (highCardinalityDims.contains(column.getColumnName)) {
- newOrderedDims += column
- } else if (column.isComplex) {
- complexDims += column
- } else if (column.isDimensionColumn) {
- newOrderedDims += column
- } else {
- measures += column
- }
-
- }
-
// Setting the boolean value of useInvertedIndex in column schema
val noInvertedIndexCols = cm.noInvertedIdxCols.getOrElse(Seq())
for (column <- allColumns) {
@@ -447,7 +458,7 @@ class TableNewProcessor(cm: TableModel) {
}
// Adding dummy measure if no measure is provided
- if (measures.size < 1) {
+ if (measureCount == 0) {
val encoders = new java.util.ArrayList[Encoding]()
val columnSchema: ColumnSchema = getColumnSchema(DataType.DOUBLE,
CarbonCommonConstants.DEFAULT_INVISIBLE_DUMMY_MEASURE,
@@ -457,13 +468,10 @@ class TableNewProcessor(cm: TableModel) {
false,
-1, 0, 0, schemaOrdinal = -1)
columnSchema.setInvisible(true)
- val measureColumn = columnSchema
- measures += measureColumn
- allColumns = allColumns ++ measures
+ allColumns :+= columnSchema
}
val columnValidator = CarbonSparkFactory.getCarbonColumnValidator()
columnValidator.validateColumns(allColumns)
- newOrderedDims = newOrderedDims ++ complexDims ++ measures
val tableInfo = new TableInfo()
val tableSchema = new TableSchema()
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala b/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
index 8588868..b8f0a7c 100644
--- a/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
+++ b/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
@@ -32,23 +32,15 @@ private class TestCarbonSqlParserStub extends CarbonSqlParser {
def updateColumnGroupsInFieldTest(fields: Seq[Field], tableProperties: Map[String, String]): Seq[String] = {
- var (dims: Seq[Field], noDictionaryDims: Seq[String]) = extractDimColsAndNoDictionaryFields(
- fields, tableProperties)
- val msrs: Seq[Field] = extractMsrColsFromFields(fields, tableProperties)
+ var (dims, msrs, noDictionaryDims, sortkey) = extractDimAndMsrFields(fields, tableProperties)
updateColumnGroupsInField(tableProperties,
noDictionaryDims, msrs, dims)
}
- def extractDimColsAndNoDictionaryFieldsTest(fields: Seq[Field], tableProperties: Map[String, String]): (Seq[Field],
- Seq[String]) = {
-
- extractDimColsAndNoDictionaryFields(fields, tableProperties)
- }
-
- def extractMsrColsFromFieldsTest(fields: Seq[Field], tableProperties: Map[String, String]): (Seq[Field]) = {
-
- extractMsrColsFromFields(fields, tableProperties)
+ def extractDimAndMsrFieldsTest(fields: Seq[Field],
+ tableProperties: Map[String, String]): (Seq[Field], Seq[Field], Seq[String], Seq[String]) = {
+ extractDimAndMsrFields(fields, tableProperties)
}
@@ -199,7 +191,7 @@ class TestCarbonSqlParser extends QueryTest {
val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- val (dimCols, noDictionary) = stub.extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
+ val (dimCols, _, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
// testing col
@@ -219,9 +211,7 @@ class TestCarbonSqlParser extends QueryTest {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col1")
val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- val (dimCols, noDictionary) = stub
- .extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
//below fields should be available in dimensions list
assert(dimCols.size == 7)
@@ -242,9 +232,7 @@ class TestCarbonSqlParser extends QueryTest {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_INCLUDE -> "col1")
val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- val (dimCols, noDictionary) = stub
- .extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
//below dimension fields should be available in dimensions list
assert(dimCols.size == 7)
@@ -264,9 +252,8 @@ class TestCarbonSqlParser extends QueryTest {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col1", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col4")
val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- val (dimCols, noDictionary) = stub
- .extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields,
+ tableProperties)
//below dimension fields should be available in dimensions list
assert(dimCols.size == 8)
@@ -287,9 +274,7 @@ class TestCarbonSqlParser extends QueryTest {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col3", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col2")
val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- val (dimCols, noDictionary) = stub
- .extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
//below dimension fields should be available in dimensions list
assert(dimCols.size == 7)
@@ -310,9 +295,7 @@ class TestCarbonSqlParser extends QueryTest {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col1", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col2")
val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- val (dimCols, noDictionary) = stub
- .extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
//below dimension fields should be available in dimensions list
assert(dimCols.size == 7)
@@ -333,9 +316,7 @@ class TestCarbonSqlParser extends QueryTest {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col2", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col1")
val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- val (dimCols, noDictionary) = stub
- .extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
//below dimension fields should be available in dimensions list
assert(dimCols.size == 7)
@@ -358,9 +339,7 @@ class TestCarbonSqlParser extends QueryTest {
)
val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- val (dimCols, noDictionary) = stub
- .extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
//below dimension fields should be available in dimensions list
assert(dimCols.size == 8)
@@ -382,9 +361,7 @@ class TestCarbonSqlParser extends QueryTest {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE-> "col2", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col3")
val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- val (dimCols, noDictionary) = stub
- .extractDimColsAndNoDictionaryFieldsTest(fields, tableProperties)
- val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val (dimCols, msrCols, noDictionary, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
//below dimension fields should be available in dimensions list
assert(dimCols.size == 7)
@@ -402,10 +379,11 @@ class TestCarbonSqlParser extends QueryTest {
// Testing the extracting of measures
test("Test-extractMsrColsFromFields") {
- val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col2", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col1")
+ val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col2",
+ CarbonCommonConstants.DICTIONARY_INCLUDE -> "col1")
val fields: Seq[Field] = loadAllFields
val stub = new TestCarbonSqlParserStub()
- val msrCols = stub.extractMsrColsFromFieldsTest(fields, tableProperties)
+ val (_, msrCols, _, _) = stub.extractDimAndMsrFieldsTest(fields, tableProperties)
// testing col
assert(msrCols.lift(0).get.column.equalsIgnoreCase("col4"))
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java b/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
index b848543..b4b462a 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
@@ -28,18 +28,25 @@ import org.apache.spark.sql.catalyst.expressions.GenericInternalRow;
public class SparkRowReadSupportImpl extends DictionaryDecodeReadSupport<InternalRow> {
+ boolean[] isMeasure;
+
@Override public void initialize(CarbonColumn[] carbonColumns,
AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException {
- super.initialize(carbonColumns, absoluteTableIdentifier);
//can initialize and generate schema here.
+ isMeasure = new boolean[carbonColumns.length];
+ dataTypes = new DataType[carbonColumns.length];
+ for (int i = 0; i < carbonColumns.length; i++) {
+ isMeasure[i] = !carbonColumns[i].isDimesion();
+ dataTypes[i] = carbonColumns[i].getDataType();
+ }
}
@Override public InternalRow readRow(Object[] data) {
- for (int i = 0; i < dictionaries.length; i++) {
+ for (int i = 0; i < isMeasure.length; i++) {
if (data[i] == null) {
continue;
}
- if (dictionaries[i] == null) {
+ if (isMeasure[i]) {
if (dataTypes[i].equals(DataType.INT)) {
data[i] = ((Long)(data[i])).intValue();
} else if (dataTypes[i].equals(DataType.SHORT)) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
index f94c0b2..ad5e01f 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
@@ -20,6 +20,7 @@ package org.apache.carbondata.spark.vectorreader;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.spark.sql.execution.vectorized.ColumnVector;
+import org.apache.spark.sql.types.DataType;
import org.apache.spark.sql.types.Decimal;
class ColumnarVectorWrapper implements CarbonColumnVector {
@@ -30,6 +31,14 @@ class ColumnarVectorWrapper implements CarbonColumnVector {
this.columnVector = columnVector;
}
+ @Override public void putBoolean(int rowId, boolean value) {
+ columnVector.putBoolean(rowId, value);
+ }
+
+ @Override public void putFloat(int rowId, float value) {
+ columnVector.putFloat(rowId, value);
+ }
+
@Override public void putShort(int rowId, short value) {
columnVector.putShort(rowId, value);
}
@@ -110,4 +119,8 @@ class ColumnarVectorWrapper implements CarbonColumnVector {
@Override public void reset() {
// columnVector.reset();
}
+
+ @Override public DataType getType() {
+ return columnVector.dataType();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
index f8bdcf8..9e14f16 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.execution.command._
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.datatype.DataType
-import org.apache.carbondata.core.util.DataTypeUtil
+import org.apache.carbondata.core.util.{CarbonUtil, DataTypeUtil}
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.util.CommonUtil
@@ -51,18 +51,54 @@ object TableCreator {
dimensionType.exists(x => x.equalsIgnoreCase(dimensionDatatype))
}
- protected def extractDimColsAndNoDictionaryFields(fields: Seq[Field],
- tableProperties: Map[String, String]):
- (Seq[Field], Seq[String]) = {
+ protected def extractDimAndMsrFields(fields: Seq[Field],
+ tableProperties: Map[String, String]): (Seq[Field], Seq[Field], Seq[String], Seq[String]) = {
var dimFields: LinkedHashSet[Field] = LinkedHashSet[Field]()
+ var msrFields: Seq[Field] = Seq[Field]()
var dictExcludeCols: Array[String] = Array[String]()
var noDictionaryDims: Seq[String] = Seq[String]()
var dictIncludeCols: Seq[String] = Seq[String]()
+ // All columns in sortkey should be there in create table cols
+ val sortKeyOption = tableProperties.get(CarbonCommonConstants.SORT_COLUMNS)
+ var sortKeyDimsTmp: Seq[String] = Seq[String]()
+ val sortKeyString: String = if (sortKeyOption.isDefined) {
+ CarbonUtil.unquoteChar(sortKeyOption.get) trim
+ } else {
+ ""
+ }
+ if (!sortKeyString.isEmpty) {
+ val sortKey = sortKeyString.split(',').map(_.trim)
+ sortKey.foreach { column =>
+ if (!fields.exists(x => x.column.equalsIgnoreCase(column))) {
+ val errormsg = "sort_columns: " + column +
+ " does not exist in table. Please check create table statement."
+ throw new MalformedCarbonCommandException(errormsg)
+ } else {
+ val dataType = fields.find(x =>
+ x.column.equalsIgnoreCase(column)).get.dataType.get
+ if (isComplexDimDictionaryExclude(dataType)) {
+ val errormsg = "sort_columns is unsupported for complex datatype column: " + column
+ throw new MalformedCarbonCommandException(errormsg)
+ }
+ }
+ }
+
+ sortKey.foreach { dimension =>
+ if (!sortKeyDimsTmp.exists(dimension.equalsIgnoreCase(_))) {
+ fields.foreach { field =>
+ if (field.column.equalsIgnoreCase(dimension)) {
+ sortKeyDimsTmp :+= field.column
+ }
+ }
+ }
+ }
+ }
+
// All excluded cols should be there in create table cols
if (tableProperties.get(CarbonCommonConstants.DICTIONARY_EXCLUDE).isDefined) {
dictExcludeCols =
- tableProperties(CarbonCommonConstants.DICTIONARY_EXCLUDE).split(',').map(_.trim)
+ tableProperties.get(CarbonCommonConstants.DICTIONARY_EXCLUDE).get.split(',').map(_.trim)
dictExcludeCols
.foreach { dictExcludeCol =>
if (!fields.exists(x => x.column.equalsIgnoreCase(dictExcludeCol))) {
@@ -97,7 +133,7 @@ object TableCreator {
}
}
- // include cols should contain exclude cols
+ // include cols should not contain exclude cols
dictExcludeCols.foreach { dicExcludeCol =>
if (dictIncludeCols.exists(x => x.equalsIgnoreCase(dicExcludeCol))) {
val errormsg = "DICTIONARY_EXCLUDE can not contain the same column: " + dicExcludeCol +
@@ -108,11 +144,10 @@ object TableCreator {
// by default consider all String cols as dims and if any dictionary exclude is present then
// add it to noDictionaryDims list. consider all dictionary excludes/include cols as dims
- fields.foreach(field => {
-
+ fields.foreach { field =>
if (dictExcludeCols.toSeq.exists(x => x.equalsIgnoreCase(field.column))) {
- if (DataTypeUtil.getDataType(field.dataType.get.toUpperCase()) != DataType.TIMESTAMP &&
- DataTypeUtil.getDataType(field.dataType.get.toUpperCase()) != DataType.DATE) {
+ val dataType = DataTypeUtil.getDataType(field.dataType.get.toUpperCase())
+ if (dataType != DataType.TIMESTAMP && dataType != DataType.DATE) {
noDictionaryDims :+= field.column
}
dimFields += field
@@ -120,49 +155,30 @@ object TableCreator {
dimFields += field
} else if (isDetectAsDimentionDatatype(field.dataType.get)) {
dimFields += field
+ } else if (sortKeyDimsTmp.exists(x => x.equalsIgnoreCase(field.column))) {
+ noDictionaryDims :+= field.column
+ dimFields += field
+ } else {
+ msrFields :+= field
}
}
- )
-
- (dimFields.toSeq, noDictionaryDims)
- }
-
- /**
- * Extract the Measure Cols fields. By default all non string cols will be measures.
- *
- * @param fields
- * @param tableProperties
- * @return
- */
- protected def extractMsrColsFromFields(fields: Seq[Field],
- tableProperties: Map[String, String]): Seq[Field] = {
- var msrFields: Seq[Field] = Seq[Field]()
- var dictIncludedCols: Array[String] = Array[String]()
- var dictExcludedCols: Array[String] = Array[String]()
-
- // get all included cols
- if (tableProperties.get(CarbonCommonConstants.DICTIONARY_INCLUDE).isDefined) {
- dictIncludedCols =
- tableProperties(CarbonCommonConstants.DICTIONARY_INCLUDE).split(',').map(_.trim)
- }
-
- // get all excluded cols
- if (tableProperties.get(CarbonCommonConstants.DICTIONARY_EXCLUDE).isDefined) {
- dictExcludedCols =
- tableProperties(CarbonCommonConstants.DICTIONARY_EXCLUDE).split(',').map(_.trim)
- }
- // by default consider all non string cols as msrs. consider all include/ exclude cols as dims
- fields.foreach(field => {
- if (!isDetectAsDimentionDatatype(field.dataType.get)) {
- if (!dictIncludedCols.exists(x => x.equalsIgnoreCase(field.column)) &&
- !dictExcludedCols.exists(x => x.equalsIgnoreCase(field.column))) {
- msrFields :+= field
+ var sortKeyDims = sortKeyDimsTmp
+ if (sortKeyOption.isEmpty) {
+ // if SORT_COLUMNS was not defined, add all dimension to SORT_COLUMNS.
+ dimFields.foreach { field =>
+ if (!isComplexDimDictionaryExclude(field.dataType.get)) {
+ sortKeyDims :+= field.column
}
}
- })
-
- msrFields
+ }
+ if (sortKeyDims.isEmpty) {
+ // no SORT_COLUMNS
+ tableProperties.put(CarbonCommonConstants.SORT_COLUMNS, "")
+ } else {
+ tableProperties.put(CarbonCommonConstants.SORT_COLUMNS, sortKeyDims.mkString(","))
+ }
+ (dimFields.toSeq, msrFields, noDictionaryDims, sortKeyDims)
}
def getKey(parentColumnName: Option[String],
@@ -440,27 +456,24 @@ object TableCreator {
}
def prepareTableModel(ifNotExistPresent: Boolean, dbName: Option[String]
- , tableName: String, fields: Seq[Field],
- partitionCols: Seq[PartitionerField],
- bucketFields: Option[BucketFields],
- tableProperties: Map[String, String]): TableModel
+ , tableName: String, fields: Seq[Field],
+ partitionCols: Seq[PartitionerField],
+ bucketFields: Option[BucketFields],
+ tableProperties: Map[String, String]): TableModel
= {
- val (dims: Seq[Field], noDictionaryDims: Seq[String]) = extractDimColsAndNoDictionaryFields(
+ fields.zipWithIndex.foreach { x =>
+ x._1.schemaOrdinal = x._2
+ }
+ val (dims, msrs, noDictionaryDims, sortKeyDims) = extractDimAndMsrFields(
fields, tableProperties)
if (dims.isEmpty) {
- throw new MalformedCarbonCommandException(s"Table ${
- dbName.getOrElse(
- CarbonCommonConstants.DATABASE_DEFAULT_NAME)
- }.$tableName"
- +
- " can not be created without key columns. Please " +
- "use DICTIONARY_INCLUDE or " +
- "DICTIONARY_EXCLUDE to set at least one key " +
- "column " +
+ throw new MalformedCarbonCommandException(
+ s"Table ${dbName.getOrElse(CarbonCommonConstants.DATABASE_DEFAULT_NAME)}.$tableName " +
+ "can not be created without key columns. Please use DICTIONARY_INCLUDE or " +
+ "DICTIONARY_EXCLUDE to set at least one key column " +
"if all specified columns are numeric types")
}
- val msrs: Seq[Field] = extractMsrColsFromFields(fields, tableProperties)
// column properties
val colProps = extractColumnProperties(fields, tableProperties)
@@ -474,18 +487,20 @@ object TableCreator {
// validate the tableBlockSize from table properties
CommonUtil.validateTableBlockSize(tableProperties)
- TableModel(ifNotExistPresent,
+ TableModel(
+ ifNotExistPresent,
dbName.getOrElse(CarbonCommonConstants.DATABASE_DEFAULT_NAME),
dbName,
tableName,
tableProperties,
reorderDimensions(dims.map(f => normalizeType(f)).map(f => addParent(f))),
msrs.map(f => normalizeType(f)),
+ Option(sortKeyDims),
Option(noDictionaryDims),
Option(noInvertedIdxCols),
groupCols,
Some(colProps),
- bucketFields)
+ bucketFields: Option[BucketFields])
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
index a76cc0a..a8f9520 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CompactionResultSortProcessor.java
@@ -359,10 +359,18 @@ public class CompactionResultSortProcessor extends AbstractResultProcessor {
private void initializeFinalThreadMergerForMergeSort() {
String sortTempFileLocation = tempStoreLocation + CarbonCommonConstants.FILE_SEPARATOR
+ CarbonCommonConstants.SORT_TEMP_FILE_LOCATION;
+ boolean[] noDictionarySortColumnMapping = null;
+ if (noDictionaryColMapping.length == this.segmentProperties.getNumberOfSortColumns()) {
+ noDictionarySortColumnMapping = noDictionaryColMapping;
+ } else {
+ noDictionarySortColumnMapping = new boolean[this.segmentProperties.getNumberOfSortColumns()];
+ System.arraycopy(noDictionaryColMapping, 0,
+ noDictionarySortColumnMapping, 0, noDictionarySortColumnMapping.length);
+ }
finalMerger =
new SingleThreadFinalSortFilesMerger(sortTempFileLocation, tableName, dimensionColumnCount,
segmentProperties.getComplexDimensions().size(), measureCount, noDictionaryCount,
- aggType, noDictionaryColMapping);
+ aggType, noDictionaryColMapping, noDictionarySortColumnMapping);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
index b7aa32c..ce85227 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/RowResultMergerProcessor.java
@@ -183,6 +183,19 @@ public class RowResultMergerProcessor extends AbstractResultProcessor {
* Comparator class for comparing 2 raw row result.
*/
private class CarbonMdkeyComparator implements Comparator<RawResultIterator> {
+ int[] columnValueSizes = segprop.getEachDimColumnValueSize();
+ public CarbonMdkeyComparator() {
+ initSortColumns();
+ }
+
+ private void initSortColumns() {
+ int numberOfSortColumns = segprop.getNumberOfSortColumns();
+ if (numberOfSortColumns != columnValueSizes.length) {
+ int[] sortColumnValueSizes = new int[numberOfSortColumns];
+ System.arraycopy(columnValueSizes, 0, sortColumnValueSizes, 0, numberOfSortColumns);
+ this.columnValueSizes = sortColumnValueSizes;
+ }
+ }
@Override public int compare(RawResultIterator o1, RawResultIterator o2) {
@@ -200,7 +213,6 @@ public class RowResultMergerProcessor extends AbstractResultProcessor {
ByteArrayWrapper key1 = (ByteArrayWrapper) row1[0];
ByteArrayWrapper key2 = (ByteArrayWrapper) row2[0];
int compareResult = 0;
- int[] columnValueSizes = segprop.getEachDimColumnValueSize();
int dictionaryKeyOffset = 0;
byte[] dimCols1 = key1.getDictionaryKey();
byte[] dimCols2 = key2.getDictionaryKey();
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
index 12be777..e1eb071 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
@@ -80,6 +80,10 @@ public class CarbonDataLoadConfiguration {
private DictionaryCardinalityFinder cardinalityFinder;
+ private int numberOfSortColumns;
+
+ private int numberOfNoDictSortColumns;
+
public CarbonDataLoadConfiguration() {
}
@@ -125,6 +129,26 @@ public class CarbonDataLoadConfiguration {
return dimCount;
}
+ public void setNumberOfSortColumns(int numberOfSortColumns) {
+ this.numberOfSortColumns = numberOfSortColumns;
+ }
+
+ public int getNumberOfSortColumns() {
+ return this.numberOfSortColumns;
+ }
+
+ public boolean isSortTable() {
+ return this.numberOfSortColumns > 0;
+ }
+
+ public void setNumberOfNoDictSortColumns(int numberOfNoDictSortColumns) {
+ this.numberOfNoDictSortColumns = numberOfNoDictSortColumns;
+ }
+
+ public int getNumberOfNoDictSortColumns() {
+ return this.numberOfNoDictSortColumns;
+ }
+
public int getComplexDimensionCount() {
int dimCount = 0;
for (int i = 0; i < dataFields.length; i++) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
index 8865518..11e5db1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
@@ -40,6 +40,7 @@ import org.apache.carbondata.processing.newflow.steps.DataConverterProcessorWith
import org.apache.carbondata.processing.newflow.steps.DataWriterBatchProcessorStepImpl;
import org.apache.carbondata.processing.newflow.steps.DataWriterProcessorStepImpl;
import org.apache.carbondata.processing.newflow.steps.InputProcessorStepImpl;
+import org.apache.carbondata.processing.newflow.steps.NoSortProcessorStepImpl;
import org.apache.carbondata.processing.newflow.steps.SortProcessorStepImpl;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
@@ -76,9 +77,10 @@ public final class DataLoadProcessBuilder {
// data types and configurations.
AbstractDataLoadProcessorStep converterProcessorStep =
new DataConverterProcessorStepImpl(configuration, inputProcessorStep);
- // 3. Sorts the data which are part of key (all dimensions except complex types)
- AbstractDataLoadProcessorStep sortProcessorStep =
- new SortProcessorStepImpl(configuration, converterProcessorStep);
+ // 3. Sorts the data by SortColumn or not
+ AbstractDataLoadProcessorStep sortProcessorStep = configuration.isSortTable() ?
+ new SortProcessorStepImpl(configuration, converterProcessorStep) :
+ new NoSortProcessorStepImpl(configuration, converterProcessorStep);
// 4. Writes the sorted data in carbondata format.
AbstractDataLoadProcessorStep writerProcessorStep =
new DataWriterProcessorStepImpl(configuration, sortProcessorStep);
@@ -94,9 +96,10 @@ public final class DataLoadProcessBuilder {
// data types and configurations.
AbstractDataLoadProcessorStep converterProcessorStep =
new DataConverterProcessorStepImpl(configuration, inputProcessorStep);
- // 3. Sorts the data which are part of key (all dimensions except complex types)
- AbstractDataLoadProcessorStep sortProcessorStep =
- new SortProcessorStepImpl(configuration, converterProcessorStep);
+ // 3. Sorts the data by SortColumn or not
+ AbstractDataLoadProcessorStep sortProcessorStep = configuration.isSortTable() ?
+ new SortProcessorStepImpl(configuration, converterProcessorStep) :
+ new NoSortProcessorStepImpl(configuration, converterProcessorStep);
// 4. Writes the sorted data in carbondata format.
AbstractDataLoadProcessorStep writerProcessorStep =
new DataWriterBatchProcessorStepImpl(configuration, sortProcessorStep);
@@ -112,9 +115,10 @@ public final class DataLoadProcessBuilder {
// data types and configurations.
AbstractDataLoadProcessorStep converterProcessorStep =
new DataConverterProcessorWithBucketingStepImpl(configuration, inputProcessorStep);
- // 3. Sorts the data which are part of key (all dimensions except complex types)
- AbstractDataLoadProcessorStep sortProcessorStep =
- new SortProcessorStepImpl(configuration, converterProcessorStep);
+ // 3. Sorts the data by SortColumn or not
+ AbstractDataLoadProcessorStep sortProcessorStep = configuration.isSortTable() ?
+ new SortProcessorStepImpl(configuration, converterProcessorStep) :
+ new NoSortProcessorStepImpl(configuration, converterProcessorStep);
// 4. Writes the sorted data in carbondata format.
AbstractDataLoadProcessorStep writerProcessorStep =
new DataWriterProcessorStepImpl(configuration, sortProcessorStep);
@@ -193,6 +197,8 @@ public final class DataLoadProcessBuilder {
configuration.setDictionaryServerHost(loadModel.getDictionaryServerHost());
configuration.setDictionaryServerPort(loadModel.getDictionaryServerPort());
configuration.setPreFetch(loadModel.isPreFetch());
+ configuration.setNumberOfSortColumns(carbonTable.getNumberOfSortColumns());
+ configuration.setNumberOfNoDictSortColumns(carbonTable.getNumberOfNoDictSortColumns());
return configuration;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java
index 9e4b50d..3accb0b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java
@@ -16,8 +16,6 @@
*/
package org.apache.carbondata.processing.newflow.converter.impl;
-import java.nio.charset.Charset;
-
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
@@ -48,23 +46,24 @@ public class NonDictionaryFieldConverterImpl implements FieldConverter {
this.isEmptyBadRecord = isEmptyBadRecord;
}
- @Override
- public void convert(CarbonRow row, BadRecordLogHolder logHolder) {
+ @Override public void convert(CarbonRow row, BadRecordLogHolder logHolder) {
String dimensionValue = row.getString(index);
if (dimensionValue == null || dimensionValue.equals(nullformat)) {
- dimensionValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL;
- }
- if (dataType != DataType.STRING) {
- if (null == DataTypeUtil.normalizeIntAndLongValues(dimensionValue, dataType)) {
- if ((dimensionValue.length() > 0) || (dimensionValue.length() == 0 && isEmptyBadRecord)) {
+ row.update(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, index);
+ } else {
+ try {
+ row.update(DataTypeUtil
+ .getBytesBasedOnDataTypeForNoDictionaryColumn(dimensionValue, dataType), index);
+ } catch (Throwable ex) {
+ if (dimensionValue.length() != 0 || isEmptyBadRecord) {
logHolder.setReason(
"The value " + " \"" + dimensionValue + "\"" + " with column name " + column
.getColName() + " and column data type " + dataType + " is not a valid "
+ dataType + " type.");
+ } else {
+ row.update(new byte[0], index);
}
}
}
- row.update(dimensionValue.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)),
- index);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java
index 856b6ac..0e14660 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterImpl.java
@@ -83,7 +83,8 @@ public class ParallelReadMergeSorterImpl extends AbstractMergeSorter {
sortParameters.getDimColCount(),
sortParameters.getComplexDimColCount(), sortParameters.getMeasureColCount(),
sortParameters.getNoDictionaryCount(), sortParameters.getAggType(),
- sortParameters.getNoDictionaryDimnesionColumn());
+ sortParameters.getNoDictionaryDimnesionColumn(),
+ sortParameters.getNoDictionarySortColumn());
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
index e5af1c6..19b1cfe 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
@@ -143,7 +143,8 @@ public class ParallelReadMergeSorterWithBucketingImpl extends AbstractMergeSorte
new SingleThreadFinalSortFilesMerger(dataFolderLocation, sortParameters.getTableName(),
sortParameters.getDimColCount(), sortParameters.getComplexDimColCount(),
sortParameters.getMeasureColCount(), sortParameters.getNoDictionaryCount(),
- sortParameters.getAggType(), sortParameters.getNoDictionaryDimnesionColumn());
+ sortParameters.getAggType(), sortParameters.getNoDictionaryDimnesionColumn(),
+ this.sortParameters.getNoDictionarySortColumn());
return finalMerger;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
index e682263..44f11f7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
@@ -34,6 +34,8 @@ public class UnsafeCarbonRowPage {
private boolean[] noDictionaryDimensionMapping;
+ private boolean[] noDictionarySortColumnMapping;
+
private int dimensionSize;
private int measureSize;
@@ -52,9 +54,11 @@ public class UnsafeCarbonRowPage {
private boolean saveToDisk;
- public UnsafeCarbonRowPage(boolean[] noDictionaryDimensionMapping, int dimensionSize,
- int measureSize, char[] aggType, MemoryBlock memoryBlock, boolean saveToDisk) {
+ public UnsafeCarbonRowPage(boolean[] noDictionaryDimensionMapping,
+ boolean[] noDictionarySortColumnMapping, int dimensionSize, int measureSize, char[] aggType,
+ MemoryBlock memoryBlock, boolean saveToDisk) {
this.noDictionaryDimensionMapping = noDictionaryDimensionMapping;
+ this.noDictionarySortColumnMapping = noDictionarySortColumnMapping;
this.dimensionSize = dimensionSize;
this.measureSize = measureSize;
this.aggType = aggType;
@@ -324,4 +328,7 @@ public class UnsafeCarbonRowPage {
return noDictionaryDimensionMapping;
}
+ public boolean[] getNoDictionarySortColumnMapping() {
+ return noDictionarySortColumnMapping;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
index cb24968..40608fa 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
@@ -113,6 +113,7 @@ public class UnsafeSortDataRows {
public void initialize() throws CarbonSortKeyAndGroupByException {
MemoryBlock baseBlock = getMemoryBlock(inMemoryChunkSize);
this.rowPage = new UnsafeCarbonRowPage(parameters.getNoDictionaryDimnesionColumn(),
+ parameters.getNoDictionarySortColumn(),
parameters.getDimColCount() + parameters.getComplexDimColCount(),
parameters.getMeasureColCount(), parameters.getAggType(), baseBlock,
!UnsafeMemoryManager.INSTANCE.isMemoryAvailable());
@@ -178,6 +179,7 @@ public class UnsafeSortDataRows {
MemoryBlock memoryBlock = getMemoryBlock(inMemoryChunkSize);
boolean saveToDisk = !UnsafeMemoryManager.INSTANCE.isMemoryAvailable();
rowPage = new UnsafeCarbonRowPage(parameters.getNoDictionaryDimnesionColumn(),
+ parameters.getNoDictionarySortColumn(),
parameters.getDimColCount() + parameters.getComplexDimColCount(),
parameters.getMeasureColCount(), parameters.getAggType(), memoryBlock, saveToDisk);
bytesAdded += rowPage.addRow(rowBatch[i]);
@@ -205,12 +207,12 @@ public class UnsafeSortDataRows {
if (this.rowPage.getUsedSize() > 0) {
TimSort<UnsafeCarbonRow, IntPointerBuffer> timSort = new TimSort<>(
new UnsafeIntSortDataFormat(rowPage));
- if (parameters.getNoDictionaryCount() > 0) {
+ if (parameters.getNumberOfNoDictSortColumns() > 0) {
timSort.sort(rowPage.getBuffer(), 0, rowPage.getBuffer().getActualSize(),
new UnsafeRowComparator(rowPage));
} else {
timSort.sort(rowPage.getBuffer(), 0, rowPage.getBuffer().getActualSize(),
- new UnsafeRowComparatorForNormalDIms(parameters.getDimColCount(), rowPage));
+ new UnsafeRowComparatorForNormalDIms(rowPage));
}
unsafeInMemoryIntermediateFileMerger.addDataChunkToMerge(rowPage);
} else {
@@ -295,12 +297,13 @@ public class UnsafeSortDataRows {
long startTime = System.currentTimeMillis();
TimSort<UnsafeCarbonRow, IntPointerBuffer> timSort = new TimSort<>(
new UnsafeIntSortDataFormat(page));
- if (parameters.getNoDictionaryCount() > 0) {
+ // if sort_columns is not none, sort by sort_columns
+ if (parameters.getNumberOfNoDictSortColumns() > 0) {
timSort.sort(page.getBuffer(), 0, page.getBuffer().getActualSize(),
new UnsafeRowComparator(page));
} else {
timSort.sort(page.getBuffer(), 0, page.getBuffer().getActualSize(),
- new UnsafeRowComparatorForNormalDIms(parameters.getDimColCount(), page));
+ new UnsafeRowComparatorForNormalDIms(page));
}
if (rowPage.isSaveToDisk()) {
// create a new file every time
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparator.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparator.java
index 8f048bd..1d60b11 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparator.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparator.java
@@ -27,14 +27,14 @@ import org.apache.carbondata.processing.newflow.sort.unsafe.holder.UnsafeCarbonR
public class UnsafeRowComparator implements Comparator<UnsafeCarbonRow> {
/**
- * noDictionaryColMaping mapping of dictionary dimensions and no dictionary dimensions.
+ * mapping of dictionary and no dictionary of sort_columns.
*/
- private boolean[] noDictionaryColMaping;
+ private boolean[] noDictionarySortColumnMaping;
private Object baseObject;
public UnsafeRowComparator(UnsafeCarbonRowPage rowPage) {
- this.noDictionaryColMaping = rowPage.getNoDictionaryDimensionMapping();
+ this.noDictionarySortColumnMaping = rowPage.getNoDictionarySortColumnMapping();
this.baseObject = rowPage.getDataBlock().getBaseObject();
}
@@ -47,7 +47,7 @@ public class UnsafeRowComparator implements Comparator<UnsafeCarbonRow> {
long rowB = rowR.address;
int sizeA = 0;
int sizeB = 0;
- for (boolean isNoDictionary : noDictionaryColMaping) {
+ for (boolean isNoDictionary : noDictionarySortColumnMaping) {
if (isNoDictionary) {
short aShort1 = CarbonUnsafe.unsafe.getShort(baseObject, rowA + sizeA);
byte[] byteArr1 = new byte[aShort1];
@@ -92,7 +92,7 @@ public class UnsafeRowComparator implements Comparator<UnsafeCarbonRow> {
long rowB = rowR.address;
int sizeA = 0;
int sizeB = 0;
- for (boolean isNoDictionary : noDictionaryColMaping) {
+ for (boolean isNoDictionary : noDictionarySortColumnMaping) {
if (isNoDictionary) {
short aShort1 = CarbonUnsafe.unsafe.getShort(baseObjectL, rowA + sizeA);
byte[] byteArr1 = new byte[aShort1];
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java
index 7448aee..4fd245f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/comparator/UnsafeRowComparatorForNormalDIms.java
@@ -27,11 +27,11 @@ public class UnsafeRowComparatorForNormalDIms implements Comparator<UnsafeCarbon
private Object baseObject;
- private int dimCount;
+ private int numberOfSortColumns;
- public UnsafeRowComparatorForNormalDIms(int dimCount, UnsafeCarbonRowPage rowPage) {
+ public UnsafeRowComparatorForNormalDIms(UnsafeCarbonRowPage rowPage) {
this.baseObject = rowPage.getDataBlock().getBaseObject();
- this.dimCount = dimCount;
+ this.numberOfSortColumns = rowPage.getNoDictionarySortColumnMapping().length;
}
/**
@@ -43,7 +43,7 @@ public class UnsafeRowComparatorForNormalDIms implements Comparator<UnsafeCarbon
long rowB = rowR.address;
int sizeA = 0;
int sizeB = 0;
- for (int i = 0; i < dimCount; i++) {
+ for (int i = 0; i < numberOfSortColumns; i++) {
int dimFieldA = CarbonUnsafe.unsafe.getInt(baseObject, rowA + sizeA);
sizeA += 4;
int dimFieldB = CarbonUnsafe.unsafe.getInt(baseObject, rowB + sizeB);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeFinalMergePageHolder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
index ed9e0a6..397de63 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
@@ -45,13 +45,13 @@ public class UnsafeFinalMergePageHolder implements SortTempChunkHolder {
private int columnSize;
public UnsafeFinalMergePageHolder(UnsafeInMemoryIntermediateDataMerger merger,
- boolean[] noDictMapping, int columnSize) {
+ boolean[] noDictSortColumnMapping, int columnSize) {
this.actualSize = merger.getEntryCount();
this.mergedAddresses = merger.getMergedAddresses();
this.rowPageIndexes = merger.getRowPageIndexes();
this.rowPages = merger.getUnsafeCarbonRowPages();
LOGGER.audit("Processing unsafe inmemory rows page with size : " + actualSize);
- this.comparator = new NewRowComparator(noDictMapping);
+ this.comparator = new NewRowComparator(noDictSortColumnMapping);
this.columnSize = columnSize;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeInmemoryHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeInmemoryHolder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeInmemoryHolder.java
index f491623..048f4f8 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeInmemoryHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeInmemoryHolder.java
@@ -41,11 +41,12 @@ public class UnsafeInmemoryHolder implements SortTempChunkHolder {
private int columnSize;
- public UnsafeInmemoryHolder(UnsafeCarbonRowPage rowPage, int columnSize) {
+ public UnsafeInmemoryHolder(UnsafeCarbonRowPage rowPage, int columnSize,
+ int numberOfSortColumns) {
this.actualSize = rowPage.getBuffer().getActualSize();
this.rowPage = rowPage;
LOGGER.audit("Processing unsafe inmemory rows page with size : " + actualSize);
- this.comparator = new NewRowComparator(rowPage.getNoDictionaryDimensionMapping());
+ this.comparator = new NewRowComparator(rowPage.getNoDictionarySortColumnMapping());
this.columnSize = columnSize;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
index de2b874..84aa029 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
@@ -153,7 +153,7 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
this.aggType = parameters.getAggType();
this.isNoDictionaryDimensionColumn = parameters.getNoDictionaryDimnesionColumn();
this.nullSetWordsLength = ((measureCount - 1) >> 6) + 1;
- comparator = new NewRowComparator(isNoDictionaryDimensionColumn);
+ comparator = new NewRowComparator(parameters.getNoDictionarySortColumn());
initialize();
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
index 95a337a..c7d2ce4 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeSingleThreadFinalSortFilesMerger.java
@@ -131,7 +131,7 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
SortTempChunkHolder sortTempFileChunkHolder = new UnsafeInmemoryHolder(rowPage,
parameters.getDimColCount() + parameters.getComplexDimColCount() + parameters
- .getMeasureColCount());
+ .getMeasureColCount(), parameters.getNumberOfSortColumns());
// initialize
sortTempFileChunkHolder.readRow();
@@ -142,7 +142,7 @@ public class UnsafeSingleThreadFinalSortFilesMerger extends CarbonIterator<Objec
for (final UnsafeInMemoryIntermediateDataMerger merger : merges) {
SortTempChunkHolder sortTempFileChunkHolder =
- new UnsafeFinalMergePageHolder(merger, parameters.getNoDictionaryDimnesionColumn(),
+ new UnsafeFinalMergePageHolder(merger, parameters.getNoDictionarySortColumn(),
parameters.getDimColCount() + parameters.getComplexDimColCount() + parameters
.getMeasureColCount());
[44/50] [abbrv] incubator-carbondata git commit: add SORT_COLUMNS
options
Posted by ja...@apache.org.
add SORT_COLUMNS options
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/7dafae7f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/7dafae7f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/7dafae7f
Branch: refs/heads/12-dev
Commit: 7dafae7f3770728c360885881884c10b69141a58
Parents: 1edabc8
Author: QiangCai <qi...@qq.com>
Authored: Tue Apr 18 17:12:14 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Tue May 2 09:42:20 2017 +0800
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 1 +
.../core/datastore/block/SegmentProperties.java | 42 +++
.../impl/VariableLengthDimensionDataChunk.java | 3 +-
.../impl/btree/BTreeDataRefNodeFinder.java | 24 +-
.../core/keygenerator/mdkey/Bits.java | 6 +-
.../ThriftWrapperSchemaConverterImpl.java | 18 +
.../core/metadata/schema/table/CarbonTable.java | 24 ++
.../schema/table/column/ColumnSchema.java | 10 +
.../impl/DictionaryBasedResultCollector.java | 12 +-
...structureBasedDictionaryResultCollector.java | 4 +-
.../scan/filter/FilterExpressionProcessor.java | 4 +-
.../carbondata/core/scan/filter/FilterUtil.java | 98 ++++--
...velRangeLessThanEqualFilterExecuterImpl.java | 2 +-
.../RowLevelRangeLessThanFiterExecuterImpl.java | 2 +-
.../visitor/NoDictionaryTypeVisitor.java | 3 +-
.../visitor/RangeNoDictionaryTypeVisitor.java | 3 +-
.../AbstractDetailQueryResultIterator.java | 4 +-
.../scan/result/vector/CarbonColumnVector.java | 7 +
.../util/AbstractDataFileFooterConverter.java | 7 +
.../apache/carbondata/core/util/ByteUtil.java | 336 ++++++++++++++++++-
.../apache/carbondata/core/util/CarbonUtil.java | 18 +
.../carbondata/core/util/DataTypeUtil.java | 70 ++++
.../impl/btree/BTreeBlockFinderTest.java | 10 +-
.../core/scan/filter/FilterUtilTest.java | 7 +-
.../carbondata/hadoop/CarbonInputFormat.java | 4 +-
.../internal/index/impl/InMemoryBTreeIndex.java | 4 +-
.../impl/DictionaryDecodeReadSupport.java | 3 +
.../hadoop/util/BlockLevelTraverser.java | 4 +-
.../hadoop/test/util/StoreCreator.java | 5 +
.../testsuite/sortcolumns/TestSortColumns.scala | 267 +++++++++++++++
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 125 +++----
.../execution/command/carbonTableSchema.scala | 66 ++--
.../apache/spark/sql/TestCarbonSqlParser.scala | 56 +---
.../readsupport/SparkRowReadSupportImpl.java | 13 +-
.../vectorreader/ColumnarVectorWrapper.java | 13 +
.../org/apache/spark/sql/TableCreator.scala | 145 ++++----
.../merger/CompactionResultSortProcessor.java | 10 +-
.../merger/RowResultMergerProcessor.java | 14 +-
.../newflow/CarbonDataLoadConfiguration.java | 24 ++
.../newflow/DataLoadProcessBuilder.java | 24 +-
.../impl/NonDictionaryFieldConverterImpl.java | 21 +-
.../sort/impl/ParallelReadMergeSorterImpl.java | 3 +-
...arallelReadMergeSorterWithBucketingImpl.java | 3 +-
.../sort/unsafe/UnsafeCarbonRowPage.java | 11 +-
.../newflow/sort/unsafe/UnsafeSortDataRows.java | 11 +-
.../unsafe/comparator/UnsafeRowComparator.java | 10 +-
.../UnsafeRowComparatorForNormalDIms.java | 8 +-
.../holder/UnsafeFinalMergePageHolder.java | 4 +-
.../unsafe/holder/UnsafeInmemoryHolder.java | 5 +-
.../holder/UnsafeSortTempFileChunkHolder.java | 2 +-
.../UnsafeSingleThreadFinalSortFilesMerger.java | 4 +-
.../newflow/steps/NoSortProcessorStepImpl.java | 153 +++++++++
.../sortdata/IntermediateFileMerger.java | 3 +-
.../sortdata/NewRowComparator.java | 12 +-
.../sortdata/NewRowComparatorForNormalDims.java | 10 +-
.../sortandgroupby/sortdata/RowComparator.java | 10 +-
.../sortdata/RowComparatorForNormalDims.java | 10 +-
.../sortandgroupby/sortdata/SortDataRows.java | 13 +-
.../sortandgroupby/sortdata/SortParameters.java | 43 +++
.../sortdata/SortTempFileChunkHolder.java | 11 +-
.../store/CarbonFactDataHandlerColumnar.java | 45 ++-
.../store/CarbonFactDataHandlerModel.java | 7 +
.../store/SingleThreadFinalSortFilesMerger.java | 8 +-
63 files changed, 1549 insertions(+), 350 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index ee18321..4a79e0c 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -805,6 +805,7 @@ public final class CarbonCommonConstants {
public static final String COLUMN_GROUPS = "column_groups";
public static final String DICTIONARY_EXCLUDE = "dictionary_exclude";
public static final String DICTIONARY_INCLUDE = "dictionary_include";
+ public static final String SORT_COLUMNS = "sort_columns";
public static final String PARTITIONCLASS = "partitionclass";
public static final String PARTITIONCOUNT = "partitioncount";
public static final String COLUMN_PROPERTIES = "columnproperties";
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
index 3bc208d..38e3c02 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
@@ -55,6 +55,14 @@ public class SegmentProperties {
private KeyGenerator dimensionKeyGenerator;
/**
+ * key generator which was used to generate the mdkey for dimensions in SORT_COLUMNS
+ * if SORT_COLUMNS contains all dimensions, it is same with dimensionKeyGenerator
+ * otherwise, it is different with dimensionKeyGenerator, the number of its dimensions is less
+ * than dimensionKeyGenerator.
+ */
+ private KeyGenerator sortColumnsGenerator;
+
+ /**
* list of dimension present in the block
*/
private List<CarbonDimension> dimensions;
@@ -151,6 +159,10 @@ public class SegmentProperties {
*/
private ColumnGroupModel colGroupModel;
+ private int numberOfSortColumns = 0;
+
+ private int numberOfNoDictSortColumns = 0;
+
public SegmentProperties(List<ColumnSchema> columnsInTable, int[] columnCardinality) {
dimensions = new ArrayList<CarbonDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
complexDimensions =
@@ -336,6 +348,9 @@ public class SegmentProperties {
if (CarbonUtil.hasEncoding(columnSchema.getEncodingList(), Encoding.DICTIONARY)
&& !isComplexDimensionStarted && columnSchema.getNumberOfChild() == 0) {
cardinalityIndexForNormalDimensionColumn.add(tableOrdinal);
+ if (columnSchema.isSortColumn()) {
+ this.numberOfSortColumns++;
+ }
if (columnSchema.isColumnar()) {
// if it is a columnar dimension participated in mdkey then added
// key ordinal and dimension ordinal
@@ -385,6 +400,10 @@ public class SegmentProperties {
// for no dictionary dimension
carbonDimension = new CarbonDimension(columnSchema, dimensonOrdinal++, -1, -1, -1);
numberOfNoDictionaryDimension++;
+ if (columnSchema.isSortColumn()) {
+ this.numberOfSortColumns++;
+ this.numberOfNoDictSortColumns++;
+ }
}
dimensions.add(carbonDimension);
} else {
@@ -511,6 +530,14 @@ public class SegmentProperties {
int[] bitLength = CarbonUtil.getDimensionBitLength(dimColumnsCardinality, dimensionPartitions);
// create a key generator
this.dimensionKeyGenerator = new MultiDimKeyVarLengthGenerator(bitLength);
+ if (this.getNumberOfDictSortColumns() == bitLength.length) {
+ this.sortColumnsGenerator = this.dimensionKeyGenerator;
+ } else {
+ int numberOfDictSortColumns = this.getNumberOfDictSortColumns();
+ int [] sortColumnBitLength = new int[numberOfDictSortColumns];
+ System.arraycopy(bitLength, 0, sortColumnBitLength, 0, numberOfDictSortColumns);
+ this.sortColumnsGenerator = new MultiDimKeyVarLengthGenerator(sortColumnBitLength);
+ }
this.fixedLengthKeySplitter =
new MultiDimKeyVarLengthVariableSplitGenerator(bitLength, dimensionPartitions);
// get the size of each value in file block
@@ -636,6 +663,10 @@ public class SegmentProperties {
return dimensionKeyGenerator;
}
+ public KeyGenerator getSortColumnsGenerator() {
+ return sortColumnsGenerator;
+ }
+
/**
* @return the dimensions
*/
@@ -797,4 +828,15 @@ public class SegmentProperties {
return CarbonUtil.getMeasureFromCurrentBlock(this.measures, columnId);
}
+ public int getNumberOfSortColumns() {
+ return numberOfSortColumns;
+ }
+
+ public int getNumberOfNoDictSortColumns() {
+ return numberOfNoDictSortColumns;
+ }
+
+ public int getNumberOfDictSortColumns() {
+ return this.numberOfSortColumns - this.numberOfNoDictSortColumns;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
index 6c47bf5..0ac5655 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
@@ -18,7 +18,8 @@ package org.apache.carbondata.core.datastore.chunk.impl;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory
+ .DimensionStoreType;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
index 2e1709b..2f8aadf 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
@@ -49,18 +49,20 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
private int[] eachColumnValueSize;
/**
- * this will be used during search for no dictionary column
+ * the number of no dictionary columns in SORT_COLUMNS
*/
- private int numberOfNoDictionaryColumns;
+ private int numberOfNoDictSortColumns;
- public BTreeDataRefNodeFinder(int[] eachColumnValueSize) {
- this.eachColumnValueSize = eachColumnValueSize;
+ /**
+ * the number of columns in SORT_COLUMNS
+ */
+ private int numberOfSortColumns;
- for (int i = 0; i < eachColumnValueSize.length; i++) {
- if (eachColumnValueSize[i] == -1) {
- numberOfNoDictionaryColumns++;
- }
- }
+ public BTreeDataRefNodeFinder(int[] eachColumnValueSize, int numberOfSortColumns,
+ int numberOfNoDictSortColumns) {
+ this.eachColumnValueSize = eachColumnValueSize;
+ this.numberOfNoDictSortColumns = numberOfNoDictSortColumns;
+ this.numberOfSortColumns = numberOfSortColumns;
}
/**
@@ -213,7 +215,7 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
int dictionaryKeyOffset = 0;
int nonDictionaryKeyOffset = 0;
int compareResult = 0;
- int processedNoDictionaryColumn = numberOfNoDictionaryColumns;
+ int processedNoDictionaryColumn = numberOfNoDictSortColumns;
ByteBuffer firstNoDictionaryKeyBuffer = ByteBuffer.wrap(first.getNoDictionaryKeys());
ByteBuffer secondNoDictionaryKeyBuffer = ByteBuffer.wrap(second.getNoDictionaryKeys());
int actualOffset = 0;
@@ -221,7 +223,7 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
int firstNoDcitionaryLength = 0;
int secondNodeDictionaryLength = 0;
- for (int i = 0; i < eachColumnValueSize.length; i++) {
+ for (int i = 0; i < numberOfSortColumns; i++) {
if (eachColumnValueSize[i] != NO_DCITIONARY_COLUMN_VALUE) {
compareResult = ByteUtil.UnsafeComparer.INSTANCE
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
index 7bd338d..044a77b 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
@@ -119,7 +119,8 @@ public class Bits implements Serializable {
protected long[] get(long[] keys) {
long[] words = new long[wsize];
int ll = 0;
- for (int i = lens.length - 1; i >= 0; i--) {
+ int minLength = Math.min(lens.length, keys.length);
+ for (int i = minLength - 1; i >= 0; i--) {
long val = keys[i];
@@ -155,7 +156,8 @@ public class Bits implements Serializable {
protected long[] get(int[] keys) {
long[] words = new long[wsize];
int ll = 0;
- for (int i = lens.length - 1; i >= 0; i--) {
+ int minLength = Math.min(lens.length, keys.length);
+ for (int i = minLength - 1; i >= 0; i--) {
long val = keys[i];
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index 974cc81..3d114ec 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -17,8 +17,11 @@
package org.apache.carbondata.core.metadata.converter;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.BucketingInfo;
@@ -172,6 +175,13 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
thriftColumnSchema.setInvisible(wrapperColumnSchema.isInvisible());
thriftColumnSchema.setColumnReferenceId(wrapperColumnSchema.getColumnReferenceId());
thriftColumnSchema.setSchemaOrdinal(wrapperColumnSchema.getSchemaOrdinal());
+
+ if (wrapperColumnSchema.isSortColumn()) {
+ Map<String, String> properties = new HashMap<String, String>();
+ properties.put(CarbonCommonConstants.SORT_COLUMNS, "true");
+ thriftColumnSchema.setColumnProperties(properties);
+ }
+
return thriftColumnSchema;
}
@@ -360,6 +370,14 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
wrapperColumnSchema.setInvisible(externalColumnSchema.isInvisible());
wrapperColumnSchema.setColumnReferenceId(externalColumnSchema.getColumnReferenceId());
wrapperColumnSchema.setSchemaOrdinal(externalColumnSchema.getSchemaOrdinal());
+ wrapperColumnSchema.setSortColumn(false);
+ Map<String, String> properties = externalColumnSchema.getColumnProperties();
+ if (properties != null) {
+ String sortColumns = properties.get(CarbonCommonConstants.SORT_COLUMNS);
+ if (sortColumns != null) {
+ wrapperColumnSchema.setSortColumn(true);
+ }
+ }
return wrapperColumnSchema;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index dbc8836..fae9960 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -116,6 +116,16 @@ public class CarbonTable implements Serializable {
*/
private int blockSize;
+ /**
+ * the number of columns in SORT_COLUMNS
+ */
+ private int numberOfSortColumns;
+
+ /**
+ * the number of no dictionary columns in SORT_COLUMNS
+ */
+ private int numberOfNoDictSortColumns;
+
public CarbonTable() {
this.tableDimensionsMap = new HashMap<String, List<CarbonDimension>>();
this.tableImplicitDimensionsMap = new HashMap<String, List<CarbonDimension>>();
@@ -238,10 +248,16 @@ public class CarbonTable implements Serializable {
i = dimensionOrdinal - 1;
complexTypeOrdinal = assignComplexOrdinal(complexDimension, complexTypeOrdinal);
} else {
+ if (!columnSchema.isInvisible() && columnSchema.isSortColumn()) {
+ this.numberOfSortColumns++;
+ }
if (!columnSchema.getEncodingList().contains(Encoding.DICTIONARY)) {
CarbonDimension dimension =
new CarbonDimension(columnSchema, dimensionOrdinal++,
columnSchema.getSchemaOrdinal(), -1, -1, -1);
+ if (!columnSchema.isInvisible() && columnSchema.isSortColumn()) {
+ this.numberOfNoDictSortColumns++;
+ }
allDimensions.add(dimension);
primitiveDimensions.add(dimension);
} else if (columnSchema.getEncodingList().contains(Encoding.DICTIONARY)
@@ -639,4 +655,12 @@ public class CarbonTable implements Serializable {
}
tableMeasuresMap.put(tableName, visibleMeasures);
}
+
+ public int getNumberOfSortColumns() {
+ return numberOfSortColumns;
+ }
+
+ public int getNumberOfNoDictSortColumns() {
+ return numberOfNoDictSortColumns;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
index abe065c..6250dba 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
@@ -109,6 +109,8 @@ public class ColumnSchema implements Serializable {
*/
private boolean invisible = false;
+ private boolean isSortColumn = false;
+
/**
* @return the columnName
*/
@@ -411,4 +413,12 @@ public class ColumnSchema implements Serializable {
public void setSchemaOrdinal(int schemaOrdinal) {
this.schemaOrdinal = schemaOrdinal;
}
+
+ public boolean isSortColumn() {
+ return isSortColumn;
+ }
+
+ public void setSortColumn(boolean sortColumn) {
+ isSortColumn = sortColumn;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
index e5e4b3c..a7c93bb 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
@@ -88,7 +88,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
List<Object[]> listBasedResult = new ArrayList<>(batchSize);
int rowCounter = 0;
int[] surrogateResult;
- String[] noDictionaryKeys;
+ byte[][] noDictionaryKeys;
byte[][] complexTypeKeyArray;
BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache =
scannedResult.getDeleteDeltaDataCache();
@@ -96,7 +96,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
Object[] row = new Object[queryDimensions.length + queryMeasures.length];
if (isDimensionExists) {
surrogateResult = scannedResult.getDictionaryKeyIntegerArray();
- noDictionaryKeys = scannedResult.getNoDictionaryKeyStringArray();
+ noDictionaryKeys = scannedResult.getNoDictionaryKeyArray();
complexTypeKeyArray = scannedResult.getComplexTypeKeyArray();
dictionaryColumnIndex = 0;
noDictionaryColumnIndex = 0;
@@ -120,7 +120,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
}
protected void fillDimensionData(AbstractScannedResult scannedResult, int[] surrogateResult,
- String[] noDictionaryKeys, byte[][] complexTypeKeyArray,
+ byte[][] noDictionaryKeys, byte[][] complexTypeKeyArray,
Map<Integer, GenericQueryType> comlexDimensionInfoMap, Object[] row, int i) {
if (!dictionaryEncodingArray[i]) {
if (implictColumnArray[i]) {
@@ -134,9 +134,9 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
DataTypeUtil.getDataBasedOnDataType(scannedResult.getBlockletId(), DataType.STRING);
}
} else {
- row[order[i]] = DataTypeUtil
- .getDataBasedOnDataType(noDictionaryKeys[noDictionaryColumnIndex++],
- queryDimensions[i].getDimension().getDataType());
+ row[order[i]] = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(
+ noDictionaryKeys[noDictionaryColumnIndex++],
+ queryDimensions[i].getDimension().getDataType());
}
} else if (directDictionaryEncodingArray[i]) {
if (directDictionaryGenerators[i] != null) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
index 71045ff..7c07923 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
@@ -48,7 +48,7 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe
List<Object[]> listBasedResult = new ArrayList<>(batchSize);
int rowCounter = 0;
int[] surrogateResult;
- String[] noDictionaryKeys;
+ byte[][] noDictionaryKeys;
byte[][] complexTypeKeyArray;
BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache =
scannedResult.getDeleteDeltaDataCache();
@@ -58,7 +58,7 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe
Object[] row = new Object[queryDimensions.length + queryMeasures.length];
if (isDimensionExists) {
surrogateResult = scannedResult.getDictionaryKeyIntegerArray();
- noDictionaryKeys = scannedResult.getNoDictionaryKeyStringArray();
+ noDictionaryKeys = scannedResult.getNoDictionaryKeyArray();
complexTypeKeyArray = scannedResult.getComplexTypeKeyArray();
dictionaryColumnIndex = 0;
noDictionaryColumnIndex = 0;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index 264c0ae..b5341a0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -116,7 +116,9 @@ public class FilterExpressionProcessor implements FilterProcessor {
+ searchEndKey.getNoDictionaryKeys());
long startTimeInMillis = System.currentTimeMillis();
DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(
- tableSegment.getSegmentProperties().getEachDimColumnValueSize());
+ tableSegment.getSegmentProperties().getEachDimColumnValueSize(),
+ tableSegment.getSegmentProperties().getNumberOfSortColumns(),
+ tableSegment.getSegmentProperties().getNumberOfNoDictSortColumns());
DataRefNode startBlock = blockFinder.findFirstDataBlock(btreeNode, searchStartKey);
DataRefNode endBlock = blockFinder.findLastDataBlock(btreeNode, searchEndKey);
FilterExecuter filterExecuter =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 7799b6a..d816c69 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -355,10 +355,23 @@ public final class FilterUtil {
* @return DimColumnFilterInfo
*/
public static DimColumnFilterInfo getNoDictionaryValKeyMemberForFilter(
- List<String> evaluateResultListFinal, boolean isIncludeFilter) {
+ List<String> evaluateResultListFinal, boolean isIncludeFilter, DataType dataType)
+ throws FilterUnsupportedException {
List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
- for (String result : evaluateResultListFinal) {
- filterValuesList.add(result.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
+ String result = null;
+ try {
+ int length = evaluateResultListFinal.size();
+ for (int i = 0; i < length; i++) {
+ result = evaluateResultListFinal.get(i);
+ if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(result)) {
+ filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY);
+ continue;
+ }
+ filterValuesList.add(
+ DataTypeUtil.getBytesBasedOnDataTypeForNoDictionaryColumn(result, dataType));
+ }
+ } catch (Throwable ex) {
+ throw new FilterUnsupportedException("Unsupported Filter condition: " + result, ex);
}
Comparator<byte[]> filterNoDictValueComaparator = new Comparator<byte[]>() {
@@ -879,8 +892,10 @@ public final class FilterUtil {
}
int keyOrdinalOfDimensionFromCurrentBlock = dimensionFromCurrentBlock.getKeyOrdinal();
for (DimColumnFilterInfo info : values) {
- if (startKey[keyOrdinalOfDimensionFromCurrentBlock] < info.getFilterList().get(0)) {
- startKey[keyOrdinalOfDimensionFromCurrentBlock] = info.getFilterList().get(0);
+ if (keyOrdinalOfDimensionFromCurrentBlock < startKey.length) {
+ if (startKey[keyOrdinalOfDimensionFromCurrentBlock] < info.getFilterList().get(0)) {
+ startKey[keyOrdinalOfDimensionFromCurrentBlock] = info.getFilterList().get(0);
+ }
}
}
long[] newStartKey = new long[startKey.length];
@@ -944,10 +959,12 @@ public final class FilterUtil {
}
int keyOrdinalOfDimensionFromCurrentBlock = dimensionFromCurrentBlock.getKeyOrdinal();
for (DimColumnFilterInfo info : values) {
- if (endKey[keyOrdinalOfDimensionFromCurrentBlock] > info.getFilterList()
- .get(info.getFilterList().size() - 1)) {
- endKey[keyOrdinalOfDimensionFromCurrentBlock] =
- info.getFilterList().get(info.getFilterList().size() - 1);
+ if (keyOrdinalOfDimensionFromCurrentBlock < endKey.length) {
+ if (endKey[keyOrdinalOfDimensionFromCurrentBlock] > info.getFilterList()
+ .get(info.getFilterList().size() - 1)) {
+ endKey[keyOrdinalOfDimensionFromCurrentBlock] =
+ info.getFilterList().get(info.getFilterList().size() - 1);
+ }
}
}
long[] newEndKey = new long[endKey.length];
@@ -1040,44 +1057,43 @@ public final class FilterUtil {
*/
public static IndexKey prepareDefaultEndIndexKey(SegmentProperties segmentProperties)
throws KeyGenException {
- long[] dictionarySurrogateKey =
- new long[segmentProperties.getDimensions().size() - segmentProperties
- .getNumberOfNoDictionaryDimension()];
+ long[] dictionarySurrogateKey = new long[segmentProperties.getNumberOfDictSortColumns()];
int index = 0;
int[] dimColumnsCardinality = segmentProperties.getDimColumnsCardinality();
- for (int i = 0; i < dimColumnsCardinality.length; i++) {
+ for (int i = 0; i < dictionarySurrogateKey.length; i++) {
dictionarySurrogateKey[index++] = dimColumnsCardinality[i];
}
IndexKey endIndexKey;
byte[] dictionaryendMdkey =
- segmentProperties.getDimensionKeyGenerator().generateKey(dictionarySurrogateKey);
+ segmentProperties.getSortColumnsGenerator().generateKey(dictionarySurrogateKey);
byte[] noDictionaryEndKeyBuffer = getNoDictionaryDefaultEndKey(segmentProperties);
endIndexKey = new IndexKey(dictionaryendMdkey, noDictionaryEndKeyBuffer);
return endIndexKey;
}
public static byte[] getNoDictionaryDefaultEndKey(SegmentProperties segmentProperties) {
+
+ int numberOfNoDictionaryDimension = segmentProperties.getNumberOfNoDictSortColumns();
// in case of non filter query when no dictionary columns are present we
// need to set the default end key, as for non filter query
// we need to get the last
// block of the btree so we are setting the max byte value in the end key
ByteBuffer noDictionaryEndKeyBuffer = ByteBuffer.allocate(
- (segmentProperties.getNumberOfNoDictionaryDimension()
- * CarbonCommonConstants.SHORT_SIZE_IN_BYTE) + segmentProperties
- .getNumberOfNoDictionaryDimension());
+ (numberOfNoDictionaryDimension * CarbonCommonConstants.SHORT_SIZE_IN_BYTE)
+ + numberOfNoDictionaryDimension);
// end key structure will be
//<Offset of first No Dictionary key in 2 Bytes><Offset of second No Dictionary key in 2 Bytes>
//<Offset of n No Dictionary key in 2 Bytes><first no dictionary column value>
// <second no dictionary column value> <N no dictionary column value>
//example if we have 2 no dictionary column
//<[0,4,0,5,127,127]>
- short startPoint = (short) (segmentProperties.getNumberOfNoDictionaryDimension()
- * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
- for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+ short startPoint =
+ (short) (numberOfNoDictionaryDimension * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+ for (int i = 0; i < numberOfNoDictionaryDimension; i++) {
noDictionaryEndKeyBuffer.putShort((startPoint));
startPoint++;
}
- for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+ for (int i = 0; i < numberOfNoDictionaryDimension; i++) {
noDictionaryEndKeyBuffer.put((byte) 127);
}
return noDictionaryEndKeyBuffer.array();
@@ -1094,11 +1110,9 @@ public final class FilterUtil {
public static IndexKey prepareDefaultStartIndexKey(SegmentProperties segmentProperties)
throws KeyGenException {
IndexKey startIndexKey;
- long[] dictionarySurrogateKey =
- new long[segmentProperties.getDimensions().size() - segmentProperties
- .getNumberOfNoDictionaryDimension()];
+ long[] dictionarySurrogateKey = new long[segmentProperties.getNumberOfDictSortColumns()];
byte[] dictionaryStartMdkey =
- segmentProperties.getDimensionKeyGenerator().generateKey(dictionarySurrogateKey);
+ segmentProperties.getSortColumnsGenerator().generateKey(dictionarySurrogateKey);
byte[] noDictionaryStartKeyArray = getNoDictionaryDefaultStartKey(segmentProperties);
startIndexKey = new IndexKey(dictionaryStartMdkey, noDictionaryStartKeyArray);
@@ -1106,26 +1120,27 @@ public final class FilterUtil {
}
public static byte[] getNoDictionaryDefaultStartKey(SegmentProperties segmentProperties) {
+
+ int numberOfNoDictionaryDimension = segmentProperties.getNumberOfNoDictSortColumns();
// in case of non filter query when no dictionary columns are present we
// need to set the default start key, as for non filter query we need to get the first
// block of the btree so we are setting the least byte value in the start key
ByteBuffer noDictionaryStartKeyBuffer = ByteBuffer.allocate(
- (segmentProperties.getNumberOfNoDictionaryDimension()
- * CarbonCommonConstants.SHORT_SIZE_IN_BYTE) + segmentProperties
- .getNumberOfNoDictionaryDimension());
+ (numberOfNoDictionaryDimension * CarbonCommonConstants.SHORT_SIZE_IN_BYTE)
+ + numberOfNoDictionaryDimension);
// end key structure will be
//<Offset of first No Dictionary key in 2 Bytes><Offset of second No Dictionary key in 2 Bytes>
//<Offset of n No Dictionary key in 2 Bytes><first no dictionary column value>
// <second no dictionary column value> <N no dictionary column value>
//example if we have 2 no dictionary column
//<[0,4,0,5,0,0]>
- short startPoint = (short) (segmentProperties.getNumberOfNoDictionaryDimension()
- * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
- for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+ short startPoint =
+ (short) (numberOfNoDictionaryDimension * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+ for (int i = 0; i < numberOfNoDictionaryDimension; i++) {
noDictionaryStartKeyBuffer.putShort((startPoint));
startPoint++;
}
- for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+ for (int i = 0; i < numberOfNoDictionaryDimension; i++) {
noDictionaryStartKeyBuffer.put((byte) 0);
}
return noDictionaryStartKeyBuffer.array();
@@ -1181,8 +1196,8 @@ public final class FilterUtil {
FilterResolverIntf filterResolver, List<IndexKey> listOfStartEndKeys) {
IndexKey searchStartKey = null;
IndexKey searchEndKey = null;
- long[] startKey = new long[segmentProperties.getDimensionKeyGenerator().getDimCount()];
- long[] endKey = new long[segmentProperties.getDimensionKeyGenerator().getDimCount()];
+ long[] startKey = new long[segmentProperties.getNumberOfDictSortColumns()];
+ long[] endKey = new long[segmentProperties.getNumberOfDictSortColumns()];
List<byte[]> listOfStartKeyByteArray =
new ArrayList<byte[]>(segmentProperties.getNumberOfNoDictionaryDimension());
List<byte[]> listOfEndKeyByteArray =
@@ -1234,12 +1249,23 @@ public final class FilterUtil {
pruneStartAndEndKeys(setOfStartKeyByteArray, listOfStartKeyByteArray);
pruneStartAndEndKeys(setOfEndKeyByteArray, listOfEndKeyByteArray);
+ if (segmentProperties.getNumberOfNoDictSortColumns() == 0) {
+ listOfStartKeyByteArray = new ArrayList<byte[]>();
+ listOfEndKeyByteArray = new ArrayList<byte[]>();
+ } else if (segmentProperties.getNumberOfNoDictSortColumns() < listOfStartKeyByteArray
+ .size()) {
+ while (segmentProperties.getNumberOfNoDictSortColumns() < listOfStartKeyByteArray.size()) {
+ listOfStartKeyByteArray.remove(listOfStartKeyByteArray.size() - 1);
+ listOfEndKeyByteArray.remove(listOfEndKeyByteArray.size() - 1);
+ }
+ }
+
searchStartKey = FilterUtil
- .createIndexKeyFromResolvedFilterVal(startKey, segmentProperties.getDimensionKeyGenerator(),
+ .createIndexKeyFromResolvedFilterVal(startKey, segmentProperties.getSortColumnsGenerator(),
FilterUtil.getKeyWithIndexesAndValues(listOfStartKeyByteArray));
searchEndKey = FilterUtil
- .createIndexKeyFromResolvedFilterVal(endKey, segmentProperties.getDimensionKeyGenerator(),
+ .createIndexKeyFromResolvedFilterVal(endKey, segmentProperties.getSortColumnsGenerator(),
FilterUtil.getKeyWithIndexesAndValues(listOfEndKeyByteArray));
listOfStartEndKeys.add(searchStartKey);
listOfStartEndKeys.add(searchEndKey);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index 306becf..bd83eec 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -157,7 +157,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
CarbonDimension currentBlockDimension =
segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension,
- this.segmentProperties.getDimensionKeyGenerator());
+ this.segmentProperties.getSortColumnsGenerator());
}
if (dimensionColumnDataChunk.isExplicitSorted()
&& dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index f2a49d9..a439228 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -157,7 +157,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
CarbonDimension currentBlockDimension =
segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension,
- this.segmentProperties.getDimensionKeyGenerator());
+ this.segmentProperties.getSortColumnsGenerator());
}
if (dimensionColumnDataChunk.isExplicitSorted()
&& dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
index 351d2c0..efedd93 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
@@ -67,7 +67,8 @@ public class NoDictionaryTypeVisitor implements ResolvedFilterInfoVisitorIntf {
throw new FilterUnsupportedException(e);
}
resolvedFilterObject = FilterUtil
- .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter());
+ .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(),
+ metadata.getColumnExpression().getDataType());
visitableObj.setFilterValues(resolvedFilterObject);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
index 8d0a8b4..d703ed0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
@@ -72,7 +72,8 @@ public class RangeNoDictionaryTypeVisitor extends NoDictionaryTypeVisitor
throw new FilterUnsupportedException(e);
}
resolvedFilterObject = FilterUtil
- .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter());
+ .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(),
+ metadata.getColumnExpression().getDataType());
visitableObj.setFilterValues(resolvedFilterObject);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
index 4e5681c..cdab4e4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@ -104,7 +104,9 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
private void intialiseInfos() {
for (BlockExecutionInfo blockInfo : blockExecutionInfos) {
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(blockInfo.getEachColumnValueSize());
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(blockInfo.getEachColumnValueSize(),
+ blockInfo.getDataBlock().getSegmentProperties().getNumberOfSortColumns(),
+ blockInfo.getDataBlock().getSegmentProperties().getNumberOfNoDictSortColumns());
DataRefNode startDataBlock = finder
.findFirstDataBlock(blockInfo.getDataBlock().getDataRefNode(), blockInfo.getStartKey());
while (startDataBlock.nodeNumber() < blockInfo.getStartBlockletIndex()) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
index 4952e07..82a0b45 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
@@ -17,10 +17,15 @@
package org.apache.carbondata.core.scan.result.vector;
+import org.apache.spark.sql.types.DataType;
import org.apache.spark.sql.types.Decimal;
public interface CarbonColumnVector {
+ void putBoolean(int rowId, boolean value);
+
+ void putFloat(int rowId, float value);
+
void putShort(int rowId, short value);
void putShorts(int rowId, int count, short value);
@@ -59,4 +64,6 @@ public interface CarbonColumnVector {
void reset();
+ DataType getType();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index 48a081d..52f2f31 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -21,6 +21,7 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.List;
+import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.BlockInfo;
@@ -195,6 +196,12 @@ public abstract class AbstractDataFileFooterConverter {
wrapperColumnSchema.setColumnGroup(externalColumnSchema.getColumn_group_id());
wrapperColumnSchema.setScale(externalColumnSchema.getScale());
wrapperColumnSchema.setDefaultValue(externalColumnSchema.getDefault_value());
+ Map<String, String> properties = externalColumnSchema.getColumnProperties();
+ if (properties != null) {
+ if (properties.get(CarbonCommonConstants.SORT_COLUMNS) != null) {
+ wrapperColumnSchema.setSortColumn(true);
+ }
+ }
return wrapperColumnSchema;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
index 16c6760..f3f0fcd 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
@@ -17,7 +17,11 @@
package org.apache.carbondata.core.util;
+import java.io.UnsupportedEncodingException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.memory.CarbonUnsafe;
@@ -27,7 +31,13 @@ import org.apache.carbondata.core.memory.CarbonUnsafe;
*/
public final class ByteUtil {
- private static final int SIZEOF_LONG = 8;
+ public static final int SIZEOF_LONG = 8;
+
+ public static final int SIZEOF_INT = 4;
+
+ public static final int SIZEOF_SHORT = 2;
+
+ public static final String UTF8_CSN = StandardCharsets.UTF_8.name();
private ByteUtil() {
@@ -326,4 +336,328 @@ public final class ByteUtil {
}
+ /**
+ * Stirng => byte[]
+ *
+ * @param s
+ * @return
+ */
+ public static byte[] toBytes(String s) {
+ try {
+ return s.getBytes(UTF8_CSN);
+ } catch (UnsupportedEncodingException e) {
+ // should never happen!
+ throw new IllegalArgumentException("UTF8 decoding is not supported", e);
+ }
+ }
+
+ /**
+ * byte[] => String
+ *
+ * @param b
+ * @param off
+ * @param len
+ * @return
+ */
+ public static String toString(final byte[] b, int off, int len) {
+ if (b == null) {
+ return null;
+ }
+ if (len == 0) {
+ return "";
+ }
+ try {
+ return new String(b, off, len, UTF8_CSN);
+ } catch (UnsupportedEncodingException e) {
+ // should never happen!
+ throw new IllegalArgumentException("UTF8 encoding is not supported", e);
+ }
+ }
+
+ /**
+ * boolean => byte[]
+ *
+ * @param b
+ * @return
+ */
+ public static byte[] toBytes(final boolean b) {
+ return new byte[] { b ? (byte) -1 : (byte) 0 };
+ }
+
+ /**
+ * byte[] => boolean
+ *
+ * @param b
+ * @return
+ */
+ public static boolean toBoolean(final byte[] b) {
+ if (b.length != 1) {
+ throw new IllegalArgumentException("Array has wrong size: " + b.length);
+ }
+ return b[0] != (byte) 0;
+ }
+
+ /**
+ * short => byte[]
+ *
+ * @param val
+ * @return
+ */
+ public static byte[] toBytes(short val) {
+ byte[] b = new byte[SIZEOF_SHORT];
+ b[1] = (byte) val;
+ val >>= 8;
+ b[0] = (byte) val;
+ return b;
+ }
+
+ /**
+ * byte[] => short
+ *
+ * @param bytes
+ * @param offset
+ * @param length
+ * @return
+ */
+ public static short toShort(byte[] bytes, int offset, final int length) {
+ if (length != SIZEOF_SHORT || offset + length > bytes.length) {
+ throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
+ }
+ if (CarbonUnsafe.unsafe != null) {
+ if (CarbonUnsafe.ISLITTLEENDIAN) {
+ return Short.reverseBytes(
+ CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
+ } else {
+ return CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+ }
+ } else {
+ short n = 0;
+ n ^= bytes[offset] & 0xFF;
+ n <<= 8;
+ n ^= bytes[offset + 1] & 0xFF;
+ return n;
+ }
+ }
+
+ /**
+ * int => byte[]
+ *
+ * @param val
+ * @return
+ */
+ public static byte[] toBytes(int val) {
+ byte[] b = new byte[4];
+ for (int i = 3; i > 0; i--) {
+ b[i] = (byte) val;
+ val >>>= 8;
+ }
+ b[0] = (byte) val;
+ return b;
+ }
+
+ /**
+ * byte[] => int
+ *
+ * @param bytes
+ * @param offset
+ * @param length
+ * @return
+ */
+ public static int toInt(byte[] bytes, int offset, final int length) {
+ if (length != SIZEOF_INT || offset + length > bytes.length) {
+ throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
+ }
+ if (CarbonUnsafe.unsafe != null) {
+ if (CarbonUnsafe.ISLITTLEENDIAN) {
+ return Integer.reverseBytes(
+ CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
+ } else {
+ return CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+ }
+ } else {
+ int n = 0;
+ for (int i = offset; i < (offset + length); i++) {
+ n <<= 8;
+ n ^= bytes[i] & 0xFF;
+ }
+ return n;
+ }
+ }
+
+ /**
+ * float => byte[]
+ *
+ * @param f
+ * @return
+ */
+ public static byte[] toBytes(final float f) {
+ // Encode it as int
+ return toBytes(Float.floatToRawIntBits(f));
+ }
+
+ /**
+ * byte[] => float
+ *
+ * @param bytes
+ * @param offset
+ * @return
+ */
+ public static float toFloat(byte[] bytes, int offset) {
+ return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT));
+ }
+
+ /**
+ * long => byte[]
+ *
+ * @param val
+ * @return
+ */
+ public static byte[] toBytes(long val) {
+ byte[] b = new byte[8];
+ for (int i = 7; i > 0; i--) {
+ b[i] = (byte) val;
+ val >>>= 8;
+ }
+ b[0] = (byte) val;
+ return b;
+ }
+
+ /**
+ * byte[] => long
+ */
+ public static long toLong(byte[] bytes, int offset, final int length) {
+ if (length != SIZEOF_LONG || offset + length > bytes.length) {
+ throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
+ }
+ if (CarbonUnsafe.unsafe != null) {
+ if (CarbonUnsafe.ISLITTLEENDIAN) {
+ return Long.reverseBytes(
+ CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
+ } else {
+ return CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+ }
+ } else {
+ long l = 0;
+ for (int i = offset; i < offset + length; i++) {
+ l <<= 8;
+ l ^= bytes[i] & 0xFF;
+ }
+ return l;
+ }
+ }
+
+ /**
+ * doube => byte[]
+ *
+ * @param d
+ * @return
+ */
+ public static byte[] toBytes(final double d) {
+ // Encode it as a long
+ return toBytes(Double.doubleToRawLongBits(d));
+ }
+
+ /**
+ * byte[] => double
+ *
+ * @param bytes
+ * @param offset
+ * @return
+ */
+ public static double toDouble(final byte[] bytes, final int offset) {
+ return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG));
+ }
+
+ /**
+ * BigDecimal => byte[]
+ *
+ * @param val
+ * @return
+ */
+ public static byte[] toBytes(BigDecimal val) {
+ byte[] valueBytes = val.unscaledValue().toByteArray();
+ byte[] result = new byte[valueBytes.length + SIZEOF_INT];
+ int offset = putInt(result, 0, val.scale());
+ putBytes(result, offset, valueBytes, 0, valueBytes.length);
+ return result;
+ }
+
+ /**
+ * byte[] => BigDecimal
+ *
+ * @param bytes
+ * @param offset
+ * @param length
+ * @return
+ */
+ public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) {
+ if (bytes == null || length < SIZEOF_INT + 1 || (offset + length > bytes.length)) {
+ return null;
+ }
+
+ int scale = toInt(bytes, offset, bytes.length);
+ byte[] tcBytes = new byte[length - SIZEOF_INT];
+ System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT);
+ return new BigDecimal(new BigInteger(tcBytes), scale);
+ }
+
+ private static IllegalArgumentException explainWrongLengthOrOffset(final byte[] bytes,
+ final int offset, final int length, final int expectedLength) {
+ String reason;
+ if (length != expectedLength) {
+ reason = "Wrong length: " + length + ", expected " + expectedLength;
+ } else {
+ reason = "offset (" + offset + ") + length (" + length + ") exceed the"
+ + " capacity of the array: " + bytes.length;
+ }
+ return new IllegalArgumentException(reason);
+ }
+
+ /**
+ * Put an int value out to the specified byte array position.
+ *
+ * @param bytes the byte array
+ * @param offset position in the array
+ * @param val int to write out
+ * @return incremented offset
+ * @throws IllegalArgumentException if the byte array given doesn't have
+ * enough room at the offset specified.
+ */
+ public static int putInt(byte[] bytes, int offset, int val) {
+ if (bytes.length - offset < SIZEOF_INT) {
+ throw new IllegalArgumentException(
+ "Not enough room to put an int at" + " offset " + offset + " in a " + bytes.length
+ + " byte array");
+ }
+ if (CarbonUnsafe.unsafe != null) {
+ if (CarbonUnsafe.ISLITTLEENDIAN) {
+ val = Integer.reverseBytes(val);
+ }
+ CarbonUnsafe.unsafe.putInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET, val);
+ return offset + ByteUtil.SIZEOF_INT;
+ } else {
+ for (int i = offset + 3; i > offset; i--) {
+ bytes[i] = (byte) val;
+ val >>>= 8;
+ }
+ bytes[offset] = (byte) val;
+ return offset + SIZEOF_INT;
+ }
+ }
+
+ /**
+ * Put bytes at the specified byte array position.
+ *
+ * @param tgtBytes the byte array
+ * @param tgtOffset position in the array
+ * @param srcBytes array to write out
+ * @param srcOffset source offset
+ * @param srcLength source length
+ * @return incremented offset
+ */
+ public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes, int srcOffset,
+ int srcLength) {
+ System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength);
+ return tgtOffset + srcLength;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 8956549..95ee94b 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -641,6 +641,24 @@ public final class CarbonUtil {
}
/**
+ * remove the quote char for a string, e.g. "abc" => abc, 'abc' => abc
+ * @param parseStr
+ * @return
+ */
+ public static String unquoteChar(String parseStr) {
+ if (parseStr == null) {
+ return null;
+ }
+ if (parseStr.startsWith("'") && parseStr.endsWith("'")) {
+ return parseStr.substring(1, parseStr.length() - 1);
+ } else if (parseStr.startsWith("\"") && parseStr.endsWith("\"")) {
+ return parseStr.substring(1, parseStr.length() - 1);
+ } else {
+ return parseStr;
+ }
+ }
+
+ /**
* special char delimiter Converter
*
* @param delimiter
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index d8e0c82..2e16b0f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -323,6 +323,76 @@ public final class DataTypeUtil {
}
+ public static byte[] getBytesBasedOnDataTypeForNoDictionaryColumn(String dimensionValue,
+ DataType actualDataType) throws Throwable {
+ switch (actualDataType) {
+ case STRING:
+ return ByteUtil.toBytes(dimensionValue);
+ case BOOLEAN:
+ return ByteUtil.toBytes(Boolean.parseBoolean(dimensionValue));
+ case SHORT:
+ return ByteUtil.toBytes(Short.parseShort(dimensionValue));
+ case INT:
+ return ByteUtil.toBytes(Integer.parseInt(dimensionValue));
+ case FLOAT:
+ return ByteUtil.toBytes(Float.parseFloat(dimensionValue));
+ case LONG:
+ return ByteUtil.toBytes(Long.parseLong(dimensionValue));
+ case DOUBLE:
+ return ByteUtil.toBytes(Double.parseDouble(dimensionValue));
+ case DECIMAL:
+ return ByteUtil.toBytes(new BigDecimal(dimensionValue));
+ default:
+ return ByteUtil.toBytes(dimensionValue);
+ }
+ }
+
+
+ /**
+ * Below method will be used to convert the data passed to its actual data
+ * type
+ *
+ * @param dataInBytes data
+ * @param actualDataType actual data type
+ * @return actual data after conversion
+ */
+ public static Object getDataBasedOnDataTypeForNoDictionaryColumn(byte[] dataInBytes,
+ DataType actualDataType) {
+ if (null == dataInBytes || Arrays
+ .equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, dataInBytes)) {
+ return null;
+ }
+ try {
+ switch (actualDataType) {
+ case STRING:
+ return UTF8String.fromBytes(dataInBytes);
+ case BOOLEAN:
+ return ByteUtil.toBoolean(dataInBytes);
+ case SHORT:
+ return ByteUtil.toShort(dataInBytes, 0, dataInBytes.length);
+ case INT:
+ return ByteUtil.toInt(dataInBytes, 0, dataInBytes.length);
+ case FLOAT:
+ return ByteUtil.toFloat(dataInBytes, 0);
+ case LONG:
+ return ByteUtil.toLong(dataInBytes, 0, dataInBytes.length);
+ case DOUBLE:
+ return ByteUtil.toDouble(dataInBytes, 0);
+ case DECIMAL:
+ return ByteUtil.toBigDecimal(dataInBytes, 0, dataInBytes.length);
+ default:
+ return ByteUtil.toString(dataInBytes, 0, dataInBytes.length);
+ }
+ } catch (Throwable ex) {
+ String data = new String(dataInBytes, CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
+ LOGGER.error("Cannot convert" + data + " to " + actualDataType.getName() + " type value" + ex
+ .getMessage());
+ LOGGER.error("Problem while converting data type" + data);
+ return null;
+ }
+ }
+
+
/**
* Below method will be used to convert the data passed to its actual data
* type
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
index 54bbe06..d6081a3 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
@@ -68,7 +68,7 @@ public class BTreeBlockFinderTest extends TestCase {
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 });
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }, 1, 1);
ByteBuffer buffer = ByteBuffer.allocate(4 + 2);
buffer.rewind();
buffer.putShort((short) 1);
@@ -88,7 +88,7 @@ public class BTreeBlockFinderTest extends TestCase {
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 });
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }, 1, 1);
ByteBuffer buffer = ByteBuffer.allocate(4 + 1);
buffer.rewind();
buffer.put((byte) 1);
@@ -109,7 +109,7 @@ public class BTreeBlockFinderTest extends TestCase {
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 });
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
int[] dimensionBitLength =
CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
KeyGenerator multiDimKeyVarLengthGenerator =
@@ -131,7 +131,7 @@ public class BTreeBlockFinderTest extends TestCase {
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 });
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
int[] dimensionBitLength =
CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
KeyGenerator multiDimKeyVarLengthGenerator =
@@ -159,7 +159,7 @@ public class BTreeBlockFinderTest extends TestCase {
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 });
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
int[] dimensionBitLength =
CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
KeyGenerator multiDimKeyVarLengthGenerator =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
index fee3d3d..078479f 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
@@ -37,6 +37,7 @@ import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.LiteralExpression;
import org.apache.carbondata.core.scan.expression.conditional.ListExpression;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.intf.RowImpl;
import org.apache.carbondata.core.util.BitSetGroup;
@@ -363,7 +364,7 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest {
assertFalse(result);
}
- @Test public void testGetNoDictionaryValKeyMemberForFilter() {
+ @Test public void testGetNoDictionaryValKeyMemberForFilter() throws FilterUnsupportedException {
boolean isIncludeFilter = true;
AbsoluteTableIdentifier absoluteTableIdentifier =
new AbsoluteTableIdentifier(this.carbonStorePath, carbonTableIdentifier);
@@ -371,7 +372,9 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest {
List<String> evaluateResultListFinal = new ArrayList<>();
evaluateResultListFinal.add("test1");
evaluateResultListFinal.add("test2");
- assertTrue(FilterUtil.getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, isIncludeFilter) instanceof DimColumnFilterInfo);
+ assertTrue(FilterUtil
+ .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, isIncludeFilter,
+ DataType.STRING) instanceof DimColumnFilterInfo);
}
@Test public void testPrepareDefaultStartIndexKey() throws KeyGenException {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index cda34e4..05f2b67 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -598,7 +598,9 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
// Add all blocks of btree into result
DataRefNodeFinder blockFinder =
- new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize());
+ new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize(),
+ segmentProperties.getNumberOfSortColumns(),
+ segmentProperties.getNumberOfNoDictSortColumns());
DataRefNode startBlock =
blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), startIndexKey);
DataRefNode endBlock =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
index 7ba6133..d476f74 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
@@ -199,7 +199,9 @@ class InMemoryBTreeIndex implements Index {
// Add all blocks of btree into result
DataRefNodeFinder blockFinder =
- new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize());
+ new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize(),
+ segmentProperties.getNumberOfSortColumns(),
+ segmentProperties.getNumberOfNoDictSortColumns());
DataRefNode startBlock =
blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), startIndexKey);
DataRefNode endBlock =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java
index 79a1559..0239bce 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java
@@ -87,6 +87,9 @@ public class DictionaryDecodeReadSupport<T> implements CarbonReadSupport<T> {
* threshold is reached
*/
@Override public void close() {
+ if (dictionaries == null) {
+ return;
+ }
for (int i = 0; i < dictionaries.length; i++) {
CarbonUtil.clearDictionaryCache(dictionaries[i]);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
index e39d214..b814468 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
@@ -53,7 +53,9 @@ public class BlockLevelTraverser {
FilterUtil.prepareDefaultStartIndexKey(abstractIndex.getSegmentProperties());
DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(
- abstractIndex.getSegmentProperties().getEachDimColumnValueSize());
+ abstractIndex.getSegmentProperties().getEachDimColumnValueSize(),
+ abstractIndex.getSegmentProperties().getNumberOfSortColumns(),
+ abstractIndex.getSegmentProperties().getNumberOfNoDictSortColumns());
DataRefNode currentBlock =
blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), searchStartKey);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
index c90b34c..1448957 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
@@ -203,6 +203,7 @@ public class StoreCreator {
date.setColumnUniqueId(UUID.randomUUID().toString());
date.setDimensionColumn(true);
date.setColumnGroup(2);
+ date.setSortColumn(true);
columnSchemas.add(date);
ColumnSchema country = new ColumnSchema();
@@ -213,6 +214,7 @@ public class StoreCreator {
country.setColumnUniqueId(UUID.randomUUID().toString());
country.setDimensionColumn(true);
country.setColumnGroup(3);
+ country.setSortColumn(true);
columnSchemas.add(country);
ColumnSchema name = new ColumnSchema();
@@ -223,6 +225,7 @@ public class StoreCreator {
name.setColumnUniqueId(UUID.randomUUID().toString());
name.setDimensionColumn(true);
name.setColumnGroup(4);
+ name.setSortColumn(true);
columnSchemas.add(name);
ColumnSchema phonetype = new ColumnSchema();
@@ -233,6 +236,7 @@ public class StoreCreator {
phonetype.setColumnUniqueId(UUID.randomUUID().toString());
phonetype.setDimensionColumn(true);
phonetype.setColumnGroup(5);
+ phonetype.setSortColumn(true);
columnSchemas.add(phonetype);
ColumnSchema serialname = new ColumnSchema();
@@ -243,6 +247,7 @@ public class StoreCreator {
serialname.setColumnUniqueId(UUID.randomUUID().toString());
serialname.setDimensionColumn(true);
serialname.setColumnGroup(6);
+ serialname.setSortColumn(true);
columnSchemas.add(serialname);
ColumnSchema salary = new ColumnSchema();
[24/50] [abbrv] incubator-carbondata git commit: [CARBONDATA-978]
Range Filter Evaluation Rectification This closes #843
Posted by ja...@apache.org.
[CARBONDATA-978] Range Filter Evaluation Rectification This closes #843
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/53a0a072
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/53a0a072
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/53a0a072
Branch: refs/heads/12-dev
Commit: 53a0a0725642c0e3539482f2a55d8f50d3653c64
Parents: 2acb310 105e3a0
Author: ravipesala <ra...@gmail.com>
Authored: Thu Apr 27 15:17:36 2017 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Apr 27 15:17:36 2017 +0530
----------------------------------------------------------------------
.../executer/RangeValueFilterExecuterImpl.java | 43 ++-----
.../apache/carbondata/core/util/ByteUtil.java | 19 ++-
.../src/test/resources/rangedatasample.csv | 12 +-
.../test/resources/rangenodictionarycompare.csv | 21 ++++
.../RangeFilterAllDataTypesTestCases.scala | 116 ++++++++++++++++++-
.../detailquery/RangeFilterTestCase.scala | 80 ++++++++++++-
6 files changed, 245 insertions(+), 46 deletions(-)
----------------------------------------------------------------------