You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2018/09/18 12:00:39 UTC
carbondata git commit: [CARBONDATA-2940]Fix BufferUnderFlowException
for ComplexPushDown
Repository: carbondata
Updated Branches:
refs/heads/master 02a89a9c6 -> 61fcdf286
[CARBONDATA-2940]Fix BufferUnderFlowException for ComplexPushDown
Problem:
BufferUnderFlow Exception was thrown, as buffer data was filled with wrong values.
Solution:
Check for if column is of Primitive Dimension column with ComplexParentOrdinal = -1 and not Complex and increment Buffer index for NoDictionaryKeys.
This closes #2727
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/61fcdf28
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/61fcdf28
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/61fcdf28
Branch: refs/heads/master
Commit: 61fcdf286cabfce199b402b63ffc55bbf5cc64de
Parents: 02a89a9
Author: Indhumathi27 <in...@gmail.com>
Authored: Mon Sep 17 12:32:49 2018 +0530
Committer: kunal642 <ku...@gmail.com>
Committed: Tue Sep 18 17:29:49 2018 +0530
----------------------------------------------------------------------
.../core/datastore/blocklet/BlockletEncodedColumnPage.java | 2 +-
.../core/scan/collector/impl/DictionaryBasedResultCollector.java | 3 +++
.../spark/testsuite/complexType/TestComplexDataType.scala | 3 +++
3 files changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/61fcdf28/core/src/main/java/org/apache/carbondata/core/datastore/blocklet/BlockletEncodedColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/blocklet/BlockletEncodedColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/blocklet/BlockletEncodedColumnPage.java
index 135b1e2..da4e211 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/blocklet/BlockletEncodedColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/blocklet/BlockletEncodedColumnPage.java
@@ -213,7 +213,7 @@ public class BlockletEncodedColumnPage {
new DecoderBasedFallbackEncoder(encodedColumnPage, pageIndex, localDictionaryGenerator)));
} else {
fallbackFutureQueue.add(fallbackExecutorService.submit(
- new ActualDataBasedFallbackEncoder(encodedColumnPage, encodedColumnPageList.size())));
+ new ActualDataBasedFallbackEncoder(encodedColumnPage, pageIndex)));
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/61fcdf28/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
index 1faf2fd..69b142f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
@@ -231,6 +231,9 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
childColumnByteBuffer
.put(queryDimensions[i].getDimension(), buffer);
mergedComplexDimensionDataMap.put(complexParentOrdinal, childColumnByteBuffer);
+ } else if (!queryDimensions[i].getDimension().isComplex()) {
+ // If Dimension is not a Complex Column, then increment index for noDictionaryComplexColumn
+ noDictionaryComplexColumnIndex++;
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/61fcdf28/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
index 8527380..220451b 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
@@ -1015,6 +1015,9 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
"create table table1 (id int, name string, structField struct<intval:int, stringval:string>) stored by 'carbondata'")
sql("insert into table1 values(null,'aaa','23$bb')")
checkAnswer(sql("select * from table1"),Seq(Row(null,"aaa", Row(23,"bb"))))
+ checkAnswer(sql("select id,name,structField.intval,structField.stringval from table1"),Seq(Row(null,"aaa",23,"bb")))
+ checkAnswer(sql("select id,name,structField.intval,structField.stringval,name from table1"),Seq(Row(null,"aaa",23,"bb","aaa")))
+ checkAnswer(sql("select id,name,structField.intval,name,structField.stringval from table1"),Seq(Row(null,"aaa",23,"aaa","bb")))
}
}