You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by gv...@apache.org on 2018/07/19 06:10:31 UTC
carbondata git commit: [CARBONDATA-2735] Fixed Performance issue for
complex array data type when number of elements in array is more
Repository: carbondata
Updated Branches:
refs/heads/master 63b930c89 -> 892594743
[CARBONDATA-2735] Fixed Performance issue for complex array data type when number of elements in array is more
GC issue due to repeated arraylist resize, is fixed
This closes #2487
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/89259474
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/89259474
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/89259474
Branch: refs/heads/master
Commit: 89259474374eda6ba8754b213eb401b70c093f3e
Parents: 63b930c
Author: kumarvishal09 <ku...@gmail.com>
Authored: Wed Jul 11 15:24:08 2018 +0530
Committer: Venkata Ramana G <ra...@huawei.com>
Committed: Thu Jul 19 11:37:00 2018 +0530
----------------------------------------------------------------------
.../core/datastore/page/ColumnPage.java | 6 +-
.../datastore/page/SafeDecimalColumnPage.java | 1 +
.../datastore/page/SafeFixLengthColumnPage.java | 6 +-
.../datastore/page/SafeVarLengthColumnPage.java | 1 +
.../datastore/page/UnsafeDecimalColumnPage.java | 13 +--
.../page/UnsafeFixLengthColumnPage.java | 27 ++++-
.../page/UnsafeVarLengthColumnPage.java | 17 +--
.../datastore/page/VarLengthColumnPageBase.java | 106 +++++++++++--------
8 files changed, 113 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
index 8745545..ea250cf 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
@@ -49,7 +49,7 @@ import static org.apache.carbondata.core.metadata.datatype.DataTypes.SHORT_INT;
public abstract class ColumnPage {
// number of row in this page
- protected final int pageSize;
+ protected int pageSize;
// data type of the page storage
protected final DataType dataType;
@@ -832,4 +832,8 @@ public abstract class ColumnPage {
public PageLevelDictionary getColumnPageDictionary() {
throw new UnsupportedOperationException("Operation not supported");
}
+
+ public int getActualRowCount() {
+ throw new UnsupportedOperationException("Operation not supported");
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeDecimalColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeDecimalColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeDecimalColumnPage.java
index 0e0ba85..c220fa4 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeDecimalColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeDecimalColumnPage.java
@@ -218,5 +218,6 @@ public class SafeDecimalColumnPage extends DecimalColumnPage {
@Override
public void freeMemory() {
byteArrayData = null;
+ super.freeMemory();
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java
index 8f0d934..74bb3fe 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java
@@ -388,7 +388,7 @@ public class SafeFixLengthColumnPage extends ColumnPage {
}
}
- protected void ensureArraySize(int requestSize, DataType dataType) {
+ private void ensureArraySize(int requestSize, DataType dataType) {
if (dataType == DataTypes.BYTE) {
if (requestSize >= byteData.length) {
byte[] newArray = new byte[arrayElementCount + 16];
@@ -436,4 +436,8 @@ public class SafeFixLengthColumnPage extends ColumnPage {
"not support value conversion on " + dataType + " page");
}
}
+ public int getActualRowCount() {
+ return arrayElementCount;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java
index c2eb40c..274b8a7 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java
@@ -40,6 +40,7 @@ public class SafeVarLengthColumnPage extends VarLengthColumnPageBase {
@Override
public void freeMemory() {
byteArrayData = null;
+ super.freeMemory();
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeDecimalColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeDecimalColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeDecimalColumnPage.java
index 7449da6..e4a949a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeDecimalColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeDecimalColumnPage.java
@@ -119,6 +119,7 @@ public class UnsafeDecimalColumnPage extends DecimalColumnPage {
memoryBlock = null;
baseAddress = null;
baseOffset = 0;
+ super.freeMemory();
}
}
@@ -168,7 +169,7 @@ public class UnsafeDecimalColumnPage extends DecimalColumnPage {
throw new RuntimeException(e);
}
CarbonUnsafe.getUnsafe().copyMemory(bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET + offset, baseAddress,
- baseOffset + rowOffset.get(rowId), length);
+ baseOffset + rowOffset.getInt(rowId), length);
}
@Override
@@ -193,9 +194,9 @@ public class UnsafeDecimalColumnPage extends DecimalColumnPage {
@Override
public byte[] getBytes(int rowId) {
- int length = rowOffset.get(rowId + 1) - rowOffset.get(rowId);
+ int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId);
byte[] bytes = new byte[length];
- CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.get(rowId),
+ CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.getInt(rowId),
bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
return bytes;
}
@@ -242,9 +243,9 @@ public class UnsafeDecimalColumnPage extends DecimalColumnPage {
} else if (dataType == DataTypes.LONG) {
value = getLong(rowId);
} else {
- int length = rowOffset.get(rowId + 1) - rowOffset.get(rowId);
+ int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId);
byte[] bytes = new byte[length];
- CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.get(rowId), bytes,
+ CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.getInt(rowId), bytes,
CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
return decimalConverter.getDecimal(bytes);
}
@@ -253,7 +254,7 @@ public class UnsafeDecimalColumnPage extends DecimalColumnPage {
@Override
void copyBytes(int rowId, byte[] dest, int destOffset, int length) {
- CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.get(rowId), dest,
+ CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.getInt(rowId), dest,
CarbonUnsafe.BYTE_ARRAY_OFFSET + destOffset, length);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
index a4cea5d..c5805d5 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
@@ -110,6 +110,12 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
}
}
+ private void updatePageSize(int rowId) {
+ if (pageSize < rowId) {
+ pageSize = rowId;
+ }
+ }
+
@Override
public void putByte(int rowId, byte value) {
try {
@@ -120,6 +126,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
long offset = ((long)rowId) << byteBits;
CarbonUnsafe.getUnsafe().putByte(baseAddress, baseOffset + offset, value);
totalLength += ByteUtil.SIZEOF_BYTE;
+ updatePageSize(rowId);
}
@@ -134,6 +141,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
long offset = ((long)rowId) << shortBits;
CarbonUnsafe.getUnsafe().putShort(baseAddress, baseOffset + offset, value);
totalLength += ByteUtil.SIZEOF_SHORT;
+ updatePageSize(rowId);
}
@Override
@@ -149,6 +157,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
CarbonUnsafe.getUnsafe().putByte(baseAddress, baseOffset + offset + 1, data[1]);
CarbonUnsafe.getUnsafe().putByte(baseAddress, baseOffset + offset + 2, data[2]);
totalLength += ByteUtil.SIZEOF_SHORT_INT;
+ updatePageSize(rowId);
}
@Override
@@ -161,6 +170,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
long offset = ((long)rowId) << intBits;
CarbonUnsafe.getUnsafe().putInt(baseAddress, baseOffset + offset, value);
totalLength += ByteUtil.SIZEOF_INT;
+ updatePageSize(rowId);
}
@Override
@@ -173,6 +183,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
long offset = ((long)rowId) << longBits;
CarbonUnsafe.getUnsafe().putLong(baseAddress, baseOffset + offset, value);
totalLength += ByteUtil.SIZEOF_LONG;
+ updatePageSize(rowId);
}
@Override
@@ -185,6 +196,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
long offset = ((long)rowId) << doubleBits;
CarbonUnsafe.getUnsafe().putDouble(baseAddress, baseOffset + offset, value);
totalLength += ByteUtil.SIZEOF_DOUBLE;
+ updatePageSize(rowId);
}
@Override
@@ -194,6 +206,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
CarbonUnsafe.getUnsafe()
.copyMemory(bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, memoryBlock.getBaseObject(),
baseOffset + offset, bytes.length);
+ updatePageSize(rowId);
}
@Override
@@ -274,7 +287,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
@Override
public byte[] getBytePage() {
- byte[] data = new byte[getPageSize()];
+ byte[] data = new byte[getEndLoop()];
for (long i = 0; i < data.length; i++) {
long offset = i << byteBits;
data[(int)i] = CarbonUnsafe.getUnsafe().getByte(baseAddress, baseOffset + offset);
@@ -284,7 +297,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
@Override
public short[] getShortPage() {
- short[] data = new short[getPageSize()];
+ short[] data = new short[getEndLoop()];
for (long i = 0; i < data.length; i++) {
long offset = i << shortBits;
data[(int)i] = CarbonUnsafe.getUnsafe().getShort(baseAddress, baseOffset + offset);
@@ -294,7 +307,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
@Override
public byte[] getShortIntPage() {
- byte[] data = new byte[pageSize * 3];
+ byte[] data = new byte[getEndLoop() * 3];
CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset,
data, CarbonUnsafe.BYTE_ARRAY_OFFSET, data.length);
return data;
@@ -302,7 +315,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
@Override
public int[] getIntPage() {
- int[] data = new int[getPageSize()];
+ int[] data = new int[getEndLoop()];
for (long i = 0; i < data.length; i++) {
long offset = i << intBits;
data[(int)i] = CarbonUnsafe.getUnsafe().getInt(baseAddress, baseOffset + offset);
@@ -312,7 +325,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
@Override
public long[] getLongPage() {
- long[] data = new long[getPageSize()];
+ long[] data = new long[getEndLoop()];
for (long i = 0; i < data.length; i++) {
long offset = i << longBits;
data[(int)i] = CarbonUnsafe.getUnsafe().getLong(baseAddress, baseOffset + offset);
@@ -514,4 +527,8 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
capacity = newSize;
}
}
+
+ public int getActualRowCount() {
+ return getEndLoop();
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
index f60e505..ae57dcd 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
@@ -49,6 +49,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase {
memoryBlock = null;
baseAddress = null;
baseOffset = 0;
+ super.freeMemory();
}
}
@@ -65,7 +66,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase {
throw new RuntimeException(e);
}
CarbonUnsafe.getUnsafe().copyMemory(bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET + offset,
- baseAddress, baseOffset + rowOffset.get(rowId), length);
+ baseAddress, baseOffset + rowOffset.getInt(rowId), length);
}
@Override
@@ -89,20 +90,20 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase {
@Override
public byte[] getBytes(int rowId) {
- int length = rowOffset.get(rowId + 1) - rowOffset.get(rowId);
+ int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId);
byte[] bytes = new byte[length];
- CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.get(rowId),
+ CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.getInt(rowId),
bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
return bytes;
}
@Override
public byte[][] getByteArrayPage() {
- byte[][] bytes = new byte[rowOffset.size() - 1][];
- for (int rowId = 0; rowId < rowOffset.size() - 1; rowId++) {
- int length = rowOffset.get(rowId + 1) - rowOffset.get(rowId);
+ byte[][] bytes = new byte[rowOffset.getActualRowCount() - 1][];
+ for (int rowId = 0; rowId < rowOffset.getActualRowCount() - 1; rowId++) {
+ int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId);
byte[] rowData = new byte[length];
- CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.get(rowId),
+ CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.getInt(rowId),
rowData, CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
bytes[rowId] = rowData;
}
@@ -111,7 +112,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase {
@Override
void copyBytes(int rowId, byte[] dest, int destOffset, int length) {
- CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.get(rowId),
+ CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset.getInt(rowId),
dest, CarbonUnsafe.BYTE_ARRAY_OFFSET + destOffset, length);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/89259474/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
index fa163bc..55877c3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
@@ -18,10 +18,10 @@
package org.apache.carbondata.core.datastore.page;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
+import org.apache.carbondata.core.datastore.ColumnType;
import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.memory.CarbonUnsafe;
import org.apache.carbondata.core.memory.MemoryBlock;
@@ -53,7 +53,7 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
Object baseAddress;
// the offset of row in the unsafe memory, its size is pageSize + 1
- List<Integer> rowOffset;
+ ColumnPage rowOffset;
// the length of bytes added in the page
int totalLength;
@@ -66,7 +66,14 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
VarLengthColumnPageBase(TableSpec.ColumnSpec columnSpec, DataType dataType, int pageSize) {
super(columnSpec, dataType, pageSize);
- rowOffset = new ArrayList<>();
+ TableSpec.ColumnSpec spec = TableSpec.ColumnSpec
+ .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE);
+ try {
+ rowOffset =
+ ColumnPage.newPage(spec, DataTypes.INT, pageSize);
+ } catch (MemoryException e) {
+ throw new RuntimeException(e);
+ }
totalLength = 0;
}
@@ -142,14 +149,19 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
private static ColumnPage getDecimalColumnPage(TableSpec.ColumnSpec columnSpec,
byte[] lvEncodedBytes, int size) throws MemoryException {
- List<Integer> rowOffset = new ArrayList<>();
+ TableSpec.ColumnSpec spec = TableSpec.ColumnSpec
+ .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE);
+ ColumnPage rowOffset = ColumnPage.newPage(spec, DataTypes.INT,
+ CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT);
int offset;
int rowId = 0;
+ int counter = 0;
for (offset = 0; offset < lvEncodedBytes.length; offset += size) {
- rowOffset.add(offset);
+ rowOffset.putInt(counter, offset);
rowId++;
+ counter++;
}
- rowOffset.add(offset);
+ rowOffset.putInt(counter, offset);
VarLengthColumnPageBase page;
if (unsafe) {
@@ -160,10 +172,7 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
// set total length and rowOffset in page
page.totalLength = offset;
- page.rowOffset = new ArrayList<>();
- for (int i = 0; i < rowOffset.size(); i++) {
- page.rowOffset.add(rowOffset.get(i));
- }
+ page.rowOffset = rowOffset;
for (int i = 0; i < rowId; i++) {
page.putBytes(i, lvEncodedBytes, i * size, size);
}
@@ -175,24 +184,26 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
throws MemoryException {
// extract length and data, set them to rowOffset and unsafe memory correspondingly
int rowId = 0;
- List<Integer> rowOffset = new ArrayList<>();
- List<Integer> rowLength = new ArrayList<>();
+ TableSpec.ColumnSpec spec = TableSpec.ColumnSpec
+ .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE);
+ ColumnPage rowOffset = ColumnPage.newPage(spec, DataTypes.INT,
+ CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT);
int length;
int offset;
int lvEncodedOffset = 0;
-
+ int counter = 0;
// extract Length field in input and calculate total length
for (offset = 0; lvEncodedOffset < lvEncodedBytes.length; offset += length) {
length = ByteUtil.toInt(lvEncodedBytes, lvEncodedOffset);
- rowOffset.add(offset);
- rowLength.add(length);
+ rowOffset.putInt(counter, offset);
lvEncodedOffset += lvLength + length;
rowId++;
+ counter++;
}
- rowOffset.add(offset);
+ rowOffset.putInt(counter, offset);
VarLengthColumnPageBase page =
getVarLengthColumnPage(columnSpec, lvEncodedBytes, dataType, lvLength, rowId, rowOffset,
- rowLength, offset);
+ offset);
return page;
}
@@ -201,31 +212,33 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
throws MemoryException {
// extract length and data, set them to rowOffset and unsafe memory correspondingly
int rowId = 0;
- List<Integer> rowOffset = new ArrayList<>();
- List<Integer> rowLength = new ArrayList<>();
+ TableSpec.ColumnSpec spec = TableSpec.ColumnSpec
+ .newInstance(columnSpec.getFieldName(), DataTypes.INT, ColumnType.MEASURE);
+ ColumnPage rowOffset = ColumnPage.newPage(spec, DataTypes.INT,
+ CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT);
int length;
int offset;
int lvEncodedOffset = 0;
-
+ int counter = 0;
// extract Length field in input and calculate total length
for (offset = 0; lvEncodedOffset < lvEncodedBytes.length; offset += length) {
length = ByteUtil.toShort(lvEncodedBytes, lvEncodedOffset);
- rowOffset.add(offset);
- rowLength.add(length);
+ rowOffset.putInt(counter, offset);
lvEncodedOffset += lvLength + length;
rowId++;
+ counter++;
}
- rowOffset.add(offset);
+ rowOffset.putInt(counter, offset);
VarLengthColumnPageBase page =
getVarLengthColumnPage(columnSpec, lvEncodedBytes, dataType, lvLength, rowId, rowOffset,
- rowLength, offset);
+ offset);
return page;
}
private static VarLengthColumnPageBase getVarLengthColumnPage(TableSpec.ColumnSpec columnSpec,
- byte[] lvEncodedBytes, DataType dataType, int lvLength, int rowId, List<Integer> rowOffset,
- List<Integer> rowLength, int offset) throws MemoryException {
+ byte[] lvEncodedBytes, DataType dataType, int lvLength, int rowId, ColumnPage rowOffset,
+ int offset) throws MemoryException {
int lvEncodedOffset;
int length;
int numRows = rowId;
@@ -240,15 +253,12 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
// set total length and rowOffset in page
page.totalLength = offset;
- page.rowOffset = new ArrayList<>();
- for (int i = 0; i < rowOffset.size(); i++) {
- page.rowOffset.add(rowOffset.get(i));
- }
+ page.rowOffset = rowOffset;
// set data in page
lvEncodedOffset = 0;
for (int i = 0; i < numRows; i++) {
- length = rowLength.get(i);
+ length = rowOffset.getInt(i + 1) - rowOffset.getInt(i);
page.putBytes(i, lvEncodedBytes, lvEncodedOffset + lvLength, length);
lvEncodedOffset += lvLength + length;
}
@@ -296,9 +306,9 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
+ " exceed this limit at rowId " + rowId);
}
if (rowId == 0) {
- rowOffset.add(0);
+ rowOffset.putInt(0, 0);
}
- rowOffset.add(rowOffset.get(rowId) + bytes.length);
+ rowOffset.putInt(rowId + 1, rowOffset.getInt(rowId) + bytes.length);
putBytesAtRow(rowId, bytes);
totalLength += bytes.length;
}
@@ -379,7 +389,7 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
int offset = 0;
byte[] data = new byte[totalLength];
for (int rowId = 0; rowId < pageSize; rowId++) {
- int length = rowOffset.get(rowId + 1) - rowOffset.get(rowId);
+ int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId);
copyBytes(rowId, data, offset, length);
offset += length;
}
@@ -395,9 +405,9 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
public byte[] getLVFlattenedBytePage() throws IOException {
// output LV encoded byte array
int offset = 0;
- byte[] data = new byte[totalLength + ((rowOffset.size() - 1) * 4)];
- for (int rowId = 0; rowId < rowOffset.size() - 1; rowId++) {
- int length = rowOffset.get(rowId + 1) - rowOffset.get(rowId);
+ byte[] data = new byte[totalLength + ((rowOffset.getActualRowCount() - 1) * 4)];
+ for (int rowId = 0; rowId < rowOffset.getActualRowCount() - 1; rowId++) {
+ int length = rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId);
ByteUtil.setInt(data, offset, length);
copyBytes(rowId, data, offset + 4, length);
offset += 4 + length;
@@ -408,9 +418,9 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
@Override public byte[] getComplexChildrenLVFlattenedBytePage() throws IOException {
// output LV encoded byte array
int offset = 0;
- byte[] data = new byte[totalLength + ((rowOffset.size() - 1) * 2)];
- for (int rowId = 0; rowId < rowOffset.size() - 1; rowId++) {
- short length = (short) (rowOffset.get(rowId + 1) - rowOffset.get(rowId));
+ byte[] data = new byte[totalLength + ((rowOffset.getActualRowCount() - 1) * 2)];
+ for (int rowId = 0; rowId < rowOffset.getActualRowCount() - 1; rowId++) {
+ short length = (short) (rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId));
ByteUtil.setShort(data, offset, length);
copyBytes(rowId, data, offset + 2, length);
offset += 2 + length;
@@ -423,8 +433,8 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
// output LV encoded byte array
int offset = 0;
byte[] data = new byte[totalLength];
- for (int rowId = 0; rowId < rowOffset.size() - 1; rowId++) {
- int length = (rowOffset.get(rowId + 1) - rowOffset.get(rowId));
+ for (int rowId = 0; rowId < rowOffset.getActualRowCount() - 1; rowId++) {
+ int length = (rowOffset.getInt(rowId + 1) - rowOffset.getInt(rowId));
copyBytes(rowId, data, offset, length);
offset += length;
}
@@ -452,4 +462,14 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
capacity = newSize;
}
}
+
+ /**
+ * free memory as needed
+ */
+ public void freeMemory() {
+ if (null != rowOffset) {
+ rowOffset.freeMemory();
+ rowOffset = null;
+ }
+ }
}