You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2018/10/31 12:24:29 UTC
carbondata git commit: [HOTFIX-compatibility] Handle Lazy loading
with inverted index for ColumnarVectorWrapperDirectWithInvertedIndex
Repository: carbondata
Updated Branches:
refs/heads/master bcf3e0fd5 -> 94a4f8314
[HOTFIX-compatibility] Handle Lazy loading with inverted index for ColumnarVectorWrapperDirectWithInvertedIndex
Problem:
Create a store with 1.4 code with inverted index and read it with vector filling (latest master code).
below exception will be thrown from AbstractCarbonColumnarVector.
UnsupportedOperationException("Not allowed from here " + getClass().getName());
cause:
when the lazy loading with an inverted index,
getBlockDataType() was not implemented for ColumnarVectorWrapperDirectWithInvertedIndex.
So, Added implementation.
This closes #2870
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/94a4f831
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/94a4f831
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/94a4f831
Branch: refs/heads/master
Commit: 94a4f8314068ffd4c0743907752b58879578749b
Parents: bcf3e0f
Author: ajantha-bhat <aj...@gmail.com>
Authored: Mon Oct 29 12:33:55 2018 +0530
Committer: kumarvishal09 <ku...@gmail.com>
Committed: Wed Oct 31 17:54:03 2018 +0530
----------------------------------------------------------------------
.../encoding/adaptive/AdaptiveDeltaFloatingCodec.java | 10 ++++++++++
.../page/encoding/adaptive/AdaptiveFloatingCodec.java | 10 ++++++++++
.../ColumnarVectorWrapperDirectWithInvertedIndex.java | 6 ++++++
3 files changed, 26 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/94a4f831/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java
index d73318d..f91ede5 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java
@@ -272,6 +272,11 @@ public class AdaptiveDeltaFloatingCodec extends AdaptiveCodec {
int shortInt = ByteUtil.valueOf3Bytes(shortIntPage, i * 3);
vector.putFloat(i, (max - shortInt) / floatFactor);
}
+ } else if (pageDataType == DataTypes.INT) {
+ int[] intData = columnPage.getIntPage();
+ for (int i = 0; i < pageSize; i++) {
+ vector.putFloat(i, (max - intData[i]) / floatFactor);
+ }
} else {
throw new RuntimeException("internal error: " + this.toString());
}
@@ -298,6 +303,11 @@ public class AdaptiveDeltaFloatingCodec extends AdaptiveCodec {
for (int i = 0; i < pageSize; i++) {
vector.putDouble(i, (max - intData[i]) / factor);
}
+ } else if (pageDataType == DataTypes.LONG) {
+ long[] longData = columnPage.getLongPage();
+ for (int i = 0; i < pageSize; i++) {
+ vector.putDouble(i, (max - longData[i]) / factor);
+ }
} else {
throw new RuntimeException("Unsupported datatype : " + pageDataType);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/94a4f831/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
index b300ee1..49696eb 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
@@ -274,6 +274,11 @@ public class AdaptiveFloatingCodec extends AdaptiveCodec {
int shortInt = ByteUtil.valueOf3Bytes(shortIntPage, i * 3);
vector.putFloat(i, (shortInt / floatFactor));
}
+ } else if (pageDataType == DataTypes.INT) {
+ int[] intData = columnPage.getIntPage();
+ for (int i = 0; i < pageSize; i++) {
+ vector.putFloat(i, (intData[i] / floatFactor));
+ }
} else {
throw new RuntimeException("internal error: " + this.toString());
}
@@ -300,6 +305,11 @@ public class AdaptiveFloatingCodec extends AdaptiveCodec {
for (int i = 0; i < pageSize; i++) {
vector.putDouble(i, (intData[i] / factor));
}
+ } else if (pageDataType == DataTypes.LONG) {
+ long[] longData = columnPage.getLongPage();
+ for (int i = 0; i < pageSize; i++) {
+ vector.putDouble(i, (longData[i] / factor));
+ }
} else {
throw new RuntimeException("Unsupported datatype : " + pageDataType);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/94a4f831/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/ColumnarVectorWrapperDirectWithInvertedIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/ColumnarVectorWrapperDirectWithInvertedIndex.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/ColumnarVectorWrapperDirectWithInvertedIndex.java
index d95267f..f190d7d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/ColumnarVectorWrapperDirectWithInvertedIndex.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/directread/ColumnarVectorWrapperDirectWithInvertedIndex.java
@@ -19,6 +19,7 @@ package org.apache.carbondata.core.scan.result.vector.impl.directread;
import java.math.BigDecimal;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
/**
@@ -141,4 +142,9 @@ class ColumnarVectorWrapperDirectWithInvertedIndex extends AbstractCarbonColumna
columnVector.putByte(invertedIndex[rowId++], src[i]);
}
}
+
+ @Override
+ public DataType getBlockDataType() {
+ return columnVector.getBlockDataType();
+ }
}