You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/11/02 13:04:15 UTC
carbondata git commit: [HOTFIX] Fixed data loading failure with safe
column page
Repository: carbondata
Updated Branches:
refs/heads/master d62277696 -> 934216de1
[HOTFIX] Fixed data loading failure with safe column page
Note:
Currently In this PR I have disabling safe column page and local dictionary enable once all testcases passed i will revert.
Problem:
Data Loading is failing with safe column page.
Root cause:
This is because SafeFixedLengthColumnPage is not implementing getPageLengthInBytes because of this is it is going to default method and with local dictionary it is throwing exception
Solution:
Now added implementation getPageLengthInBytes in SafeFixedLengthColumnPage
This closes #2867
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/934216de
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/934216de
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/934216de
Branch: refs/heads/master
Commit: 934216de13669ac50dc55518c066124d9c8b564d
Parents: d622776
Author: kumarvishal09 <ku...@gmail.com>
Authored: Mon Oct 29 11:05:28 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Nov 2 18:34:04 2018 +0530
----------------------------------------------------------------------
.../core/datastore/page/SafeFixLengthColumnPage.java | 13 +++++++++++++
1 file changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/934216de/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java
index dd2ddf1..9dd05a5 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java
@@ -41,6 +41,7 @@ public class SafeFixLengthColumnPage extends ColumnPage {
private double[] doubleData;
private byte[] shortIntData;
private byte[][] fixedLengthdata;
+ private int totalLength;
// total number of entries in array
private int arrayElementCount = 0;
@@ -57,6 +58,7 @@ public class SafeFixLengthColumnPage extends ColumnPage {
ensureArraySize(rowId, DataTypes.BYTE);
byteData[rowId] = value;
arrayElementCount++;
+ totalLength += DataTypes.BYTE.getSizeInBytes();
}
/**
@@ -67,6 +69,7 @@ public class SafeFixLengthColumnPage extends ColumnPage {
ensureArraySize(rowId, DataTypes.SHORT);
shortData[rowId] = value;
arrayElementCount++;
+ totalLength += DataTypes.SHORT.getSizeInBytes();
}
/**
@@ -77,6 +80,7 @@ public class SafeFixLengthColumnPage extends ColumnPage {
ensureArraySize(rowId, DataTypes.INT);
intData[rowId] = value;
arrayElementCount++;
+ totalLength += DataTypes.INT.getSizeInBytes();
}
/**
@@ -87,6 +91,7 @@ public class SafeFixLengthColumnPage extends ColumnPage {
ensureArraySize(rowId, DataTypes.LONG);
longData[rowId] = value;
arrayElementCount++;
+ totalLength += DataTypes.LONG.getSizeInBytes();
}
/**
@@ -97,6 +102,7 @@ public class SafeFixLengthColumnPage extends ColumnPage {
ensureArraySize(rowId, DataTypes.DOUBLE);
doubleData[rowId] = value;
arrayElementCount++;
+ totalLength += DataTypes.DOUBLE.getSizeInBytes();
}
/**
@@ -107,6 +113,7 @@ public class SafeFixLengthColumnPage extends ColumnPage {
ensureArraySize(rowId, DataTypes.FLOAT);
floatData[rowId] = value;
arrayElementCount++;
+ totalLength += DataTypes.FLOAT.getSizeInBytes();
}
/**
@@ -117,6 +124,7 @@ public class SafeFixLengthColumnPage extends ColumnPage {
ensureArraySize(rowId, DataTypes.BYTE_ARRAY);
this.fixedLengthdata[rowId] = bytes;
arrayElementCount++;
+ totalLength += bytes.length;
}
@Override
@@ -125,6 +133,7 @@ public class SafeFixLengthColumnPage extends ColumnPage {
byte[] converted = ByteUtil.to3Bytes(value);
System.arraycopy(converted, 0, shortIntData, rowId * 3, 3);
arrayElementCount++;
+ totalLength += DataTypes.SHORT_INT.getSizeInBytes();
}
@Override
@@ -475,4 +484,8 @@ public class SafeFixLengthColumnPage extends ColumnPage {
return arrayElementCount;
}
+ @Override
+ public long getPageLengthInBytes() throws IOException {
+ return totalLength;
+ }
}