You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by gv...@apache.org on 2016/09/08 08:48:29 UTC
[1/2] incubator-carbondata git commit: fixed query issue,
use different actualoffset
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 66f6e62b6 -> 2ca1b5691
fixed query issue, use different actualoffset
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/c5b45f44
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/c5b45f44
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/c5b45f44
Branch: refs/heads/master
Commit: c5b45f444344ad121dfd9704f519af9054474d0e
Parents: 66f6e62
Author: foryou2030 <fo...@126.com>
Authored: Wed Sep 7 22:46:28 2016 +0800
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Thu Sep 8 14:12:43 2016 +0530
----------------------------------------------------------------------
.../impl/btree/BTreeDataRefNodeFinder.java | 17 +++++++++++------
.../spark/src/test/resources/dataWithEmptyRows.csv | 2 ++
.../detailquery/NoDictionaryColumnTestCase.scala | 16 ++++++++++++++++
3 files changed, 29 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c5b45f44/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java
index e443182..31273ff 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java
@@ -223,6 +223,7 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
ByteBuffer firstNoDictionaryKeyBuffer = ByteBuffer.wrap(first.getNoDictionaryKeys());
ByteBuffer secondNoDictionaryKeyBuffer = ByteBuffer.wrap(second.getNoDictionaryKeys());
int actualOffset = 0;
+ int actualOffset1 = 0;
int firstNoDcitionaryLength = 0;
int secondNodeDictionaryLength = 0;
@@ -237,21 +238,25 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
if (processedNoDictionaryColumn > 1) {
actualOffset = firstNoDictionaryKeyBuffer.getShort(nonDictionaryKeyOffset);
firstNoDcitionaryLength =
- firstNoDictionaryKeyBuffer.getShort(nonDictionaryKeyOffset + SHORT_SIZE_IN_BYTES);
+ firstNoDictionaryKeyBuffer.getShort(nonDictionaryKeyOffset + SHORT_SIZE_IN_BYTES)
+ - actualOffset;
+ actualOffset1 = secondNoDictionaryKeyBuffer.getShort(nonDictionaryKeyOffset);
secondNodeDictionaryLength =
- secondNoDictionaryKeyBuffer.getShort(nonDictionaryKeyOffset + SHORT_SIZE_IN_BYTES);
+ secondNoDictionaryKeyBuffer.getShort(nonDictionaryKeyOffset + SHORT_SIZE_IN_BYTES)
+ - actualOffset1;
compareResult = ByteUtil.UnsafeComparer.INSTANCE
- .compareTo(first.getNoDictionaryKeys(), actualOffset, firstNoDcitionaryLength,
- second.getNoDictionaryKeys(), actualOffset, secondNodeDictionaryLength);
+ .compareTo(first.getNoDictionaryKeys(), actualOffset, firstNoDcitionaryLength,
+ second.getNoDictionaryKeys(), actualOffset1, secondNodeDictionaryLength);
nonDictionaryKeyOffset += SHORT_SIZE_IN_BYTES;
processedNoDictionaryColumn--;
} else {
actualOffset = firstNoDictionaryKeyBuffer.getShort(nonDictionaryKeyOffset);
+ actualOffset1 = secondNoDictionaryKeyBuffer.getShort(nonDictionaryKeyOffset);
firstNoDcitionaryLength = first.getNoDictionaryKeys().length - actualOffset;
- secondNodeDictionaryLength = second.getNoDictionaryKeys().length - actualOffset;
+ secondNodeDictionaryLength = second.getNoDictionaryKeys().length - actualOffset1;
compareResult = ByteUtil.UnsafeComparer.INSTANCE
.compareTo(first.getNoDictionaryKeys(), actualOffset, firstNoDcitionaryLength,
- second.getNoDictionaryKeys(), actualOffset, secondNodeDictionaryLength);
+ second.getNoDictionaryKeys(), actualOffset1, secondNodeDictionaryLength);
}
}
if (compareResult != 0) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c5b45f44/integration/spark/src/test/resources/dataWithEmptyRows.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/dataWithEmptyRows.csv b/integration/spark/src/test/resources/dataWithEmptyRows.csv
new file mode 100644
index 0000000..0e3a81f
--- /dev/null
+++ b/integration/spark/src/test/resources/dataWithEmptyRows.csv
@@ -0,0 +1,2 @@
+29000,cust_name_2000,active_emui_version_2000,2010-10-04 01:00:01,12345678
+,,,,0
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/c5b45f44/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala
index 1b3a4bc..b36da1f 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/NoDictionaryColumnTestCase.scala
@@ -33,10 +33,17 @@ class NoDictionaryColumnTestCase extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
sql("DROP TABLE IF EXISTS carbonTable")
sql("DROP TABLE IF EXISTS hiveTable")
+ sql("DROP TABLE IF EXISTS carbonEmpty")
+ sql("DROP TABLE IF EXISTS hiveEmpty")
sql("CREATE TABLE carbonTable (imei String, age Int, num BigInt) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='age,num')")
sql("LOAD DATA LOCAL INPATH './src/test/resources/datawithNegtiveNumber.csv' INTO TABLE carbonTable")
sql("CREATE TABLE hiveTable (imei String, age Int, num BigInt) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','")
sql("LOAD DATA LOCAL INPATH './src/test/resources/datawithNegeativewithoutHeader.csv' INTO TABLE hiveTable")
+
+ sql("CREATE TABLE carbonEmpty (cust_id int, cust_name String, active_emui_version String, bob timestamp, bigint_column bigint) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='cust_name,active_emui_version')")
+ sql("LOAD DATA LOCAL INPATH './src/test/resources/dataWithEmptyRows.csv' INTO TABLE carbonEmpty OPTIONS('FILEHEADER'='cust_id,cust_name,active_emui_version,bob,bigint_column')")
+ sql("CREATE TABLE hiveEmpty (cust_id int, cust_name String, active_emui_version String, bob timestamp, bigint_column bigint) ROW FORMAT DELIMITED FIELDS TERMINATED BY ','")
+ sql("LOAD DATA LOCAL INPATH './src/test/resources/dataWithEmptyRows.csv' INTO TABLE hiveEmpty")
}
test("SELECT IntType FROM carbonTable") {
@@ -53,8 +60,17 @@ class NoDictionaryColumnTestCase extends QueryTest with BeforeAndAfterAll {
)
}
+ test("test load data with one row that all no dictionary column values are empty") {
+ checkAnswer(
+ sql("SELECT cust_name,active_emui_version FROM carbonEmpty"),
+ sql("SELECT cust_name,active_emui_version FROM hiveEmpty")
+ )
+ }
+
override def afterAll {
sql("DROP TABLE IF EXISTS carbonTable")
sql("DROP TABLE IF EXISTS hiveTable")
+ sql("DROP TABLE IF EXISTS carbonEmpty")
+ sql("DROP TABLE IF EXISTS hiveEmpty")
}
}
\ No newline at end of file
[2/2] incubator-carbondata git commit: [CARBONDATA-222] No dictionary
column offset problem. This closes #137
Posted by gv...@apache.org.
[CARBONDATA-222] No dictionary column offset problem. This closes #137
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/2ca1b569
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/2ca1b569
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/2ca1b569
Branch: refs/heads/master
Commit: 2ca1b56913b405dda92ba92fd9dff9e742bf103e
Parents: 66f6e62 c5b45f4
Author: Venkata Ramana G <ra...@huawei.com>
Authored: Thu Sep 8 14:17:09 2016 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Thu Sep 8 14:17:09 2016 +0530
----------------------------------------------------------------------
.../impl/btree/BTreeDataRefNodeFinder.java | 17 +++++++++++------
.../spark/src/test/resources/dataWithEmptyRows.csv | 2 ++
.../detailquery/NoDictionaryColumnTestCase.scala | 16 ++++++++++++++++
3 files changed, 29 insertions(+), 6 deletions(-)
----------------------------------------------------------------------