You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/11/21 17:59:56 UTC
[29/50] [abbrv] carbondata git commit: [CARBONDATA-3113] Fixed Local
Dictionary Query Performance and Added reusable buffer for direct flow
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
index e5312f3..51dfbf2 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
@@ -308,7 +308,8 @@ public abstract class ColumnPage {
private static ColumnPage newDecimalPage(ColumnPageEncoderMeta meta,
byte[] lvEncodedByteArray) throws MemoryException {
- return VarLengthColumnPageBase.newDecimalColumnPage(meta, lvEncodedByteArray);
+ return VarLengthColumnPageBase
+ .newDecimalColumnPage(meta, lvEncodedByteArray, lvEncodedByteArray.length);
}
private static ColumnPage newLVBytesPage(TableSpec.ColumnSpec columnSpec,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/datastore/page/DecoderBasedFallbackEncoder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/DecoderBasedFallbackEncoder.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/DecoderBasedFallbackEncoder.java
index 9bed89f..1de8201 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/DecoderBasedFallbackEncoder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/DecoderBasedFallbackEncoder.java
@@ -88,7 +88,8 @@ public class DecoderBasedFallbackEncoder implements Callable<FallbackEncodedColu
CarbonUtil.getIntArray(data, offset, encodedColumnPage.getPageMetadata().rle_page_length);
// uncompress the data with rle indexes
bytes = UnBlockIndexer
- .uncompressData(bytes, rlePage, CarbonCommonConstants.LOCAL_DICT_ENCODED_BYTEARRAY_SIZE);
+ .uncompressData(bytes, rlePage, CarbonCommonConstants.LOCAL_DICT_ENCODED_BYTEARRAY_SIZE,
+ bytes.length);
}
// disable encoding using local dictionary
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
index 81bb1b5..0f409f6 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
@@ -125,7 +125,7 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
* Create a new column page for decimal page
*/
public static ColumnPage newDecimalColumnPage(ColumnPageEncoderMeta meta,
- byte[] lvEncodedBytes) throws MemoryException {
+ byte[] lvEncodedBytes, int actualDataLength) throws MemoryException {
TableSpec.ColumnSpec columnSpec = meta.getColumnSpec();
DecimalConverterFactory.DecimalConverter decimalConverter =
DecimalConverterFactory.INSTANCE.getDecimalConverter(columnSpec.getPrecision(),
@@ -137,7 +137,7 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
CarbonCommonConstants.INT_SIZE_IN_BYTE, meta.getCompressorName());
} else {
// Here the size is always fixed.
- return getDecimalColumnPage(meta, lvEncodedBytes, size);
+ return getDecimalColumnPage(meta, lvEncodedBytes, size, actualDataLength);
}
}
@@ -160,7 +160,7 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
}
private static ColumnPage getDecimalColumnPage(ColumnPageEncoderMeta meta,
- byte[] lvEncodedBytes, int size) throws MemoryException {
+ byte[] lvEncodedBytes, int size, int actualDataLength) throws MemoryException {
TableSpec.ColumnSpec columnSpec = meta.getColumnSpec();
String compressorName = meta.getCompressorName();
TableSpec.ColumnSpec spec = TableSpec.ColumnSpec
@@ -171,7 +171,7 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
int offset;
int rowId = 0;
int counter = 0;
- for (offset = 0; offset < lvEncodedBytes.length; offset += size) {
+ for (offset = 0; offset < actualDataLength; offset += size) {
rowOffset.putInt(counter, offset);
rowId++;
counter++;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageDecoder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageDecoder.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageDecoder.java
index 6f36c08..b5dc502 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageDecoder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageDecoder.java
@@ -20,6 +20,7 @@ package org.apache.carbondata.core.datastore.page.encoding;
import java.io.IOException;
import java.util.BitSet;
+import org.apache.carbondata.core.datastore.ReusableDataBuffer;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
@@ -29,13 +30,15 @@ public interface ColumnPageDecoder {
/**
* Apply decoding algorithm on input byte array and return decoded column page
*/
- ColumnPage decode(byte[] input, int offset, int length) throws MemoryException, IOException;
+ ColumnPage decode(byte[] input, int offset, int length)
+ throws MemoryException, IOException;
/**
* Apply decoding algorithm on input byte array and fill the vector here.
*/
void decodeAndFillVector(byte[] input, int offset, int length, ColumnVectorInfo vectorInfo,
- BitSet nullBits, boolean isLVEncoded, int pageSize) throws MemoryException, IOException;
+ BitSet nullBits, boolean isLVEncoded, int pageSize, ReusableDataBuffer reusableDataBuffer)
+ throws MemoryException, IOException;
ColumnPage decode(byte[] input, int offset, int length, boolean isLVEncoded)
throws MemoryException, IOException;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java
index 735847e..fb53dba 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaFloatingCodec.java
@@ -23,6 +23,7 @@ import java.util.BitSet;
import java.util.List;
import java.util.Map;
+import org.apache.carbondata.core.datastore.ReusableDataBuffer;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -123,20 +124,26 @@ public class AdaptiveDeltaFloatingCodec extends AdaptiveCodec {
@Override
public ColumnPageDecoder createDecoder(final ColumnPageEncoderMeta meta) {
return new ColumnPageDecoder() {
- @Override
- public ColumnPage decode(byte[] input, int offset, int length)
+ @Override public ColumnPage decode(byte[] input, int offset, int length)
throws MemoryException, IOException {
ColumnPage page = ColumnPage.decompress(meta, input, offset, length, false);
return LazyColumnPage.newPage(page, converter);
}
- @Override
- public void decodeAndFillVector(byte[] input, int offset, int length,
- ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize)
+ @Override public void decodeAndFillVector(byte[] input, int offset, int length,
+ ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize,
+ ReusableDataBuffer reusableDataBuffer)
throws MemoryException, IOException {
Compressor compressor =
CompressorFactory.getInstance().getCompressor(meta.getCompressorName());
- byte[] unCompressData = compressor.unCompressByte(input, offset, length);
+ byte[] unCompressData;
+ if (null != reusableDataBuffer && compressor.supportReusableBuffer()) {
+ int uncompressedLength = compressor.unCompressedLength(input, offset, length);
+ unCompressData = reusableDataBuffer.getDataBuffer(uncompressedLength);
+ compressor.rawUncompress(input, offset, length, unCompressData);
+ } else {
+ unCompressData = compressor.unCompressByte(input, offset, length);
+ }
converter.decodeAndFillVector(unCompressData, vectorInfo, nullBits, meta.getStoreDataType(),
pageSize);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaIntegralCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaIntegralCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaIntegralCodec.java
index 578945b..9ed21f4 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaIntegralCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveDeltaIntegralCodec.java
@@ -24,6 +24,7 @@ import java.util.BitSet;
import java.util.List;
import java.util.Map;
+import org.apache.carbondata.core.datastore.ReusableDataBuffer;
import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
@@ -130,8 +131,7 @@ public class AdaptiveDeltaIntegralCodec extends AdaptiveCodec {
@Override
public ColumnPageDecoder createDecoder(final ColumnPageEncoderMeta meta) {
return new ColumnPageDecoder() {
- @Override
- public ColumnPage decode(byte[] input, int offset, int length)
+ @Override public ColumnPage decode(byte[] input, int offset, int length)
throws MemoryException, IOException {
ColumnPage page = null;
if (DataTypes.isDecimal(meta.getSchemaDataType())) {
@@ -142,13 +142,20 @@ public class AdaptiveDeltaIntegralCodec extends AdaptiveCodec {
return LazyColumnPage.newPage(page, converter);
}
- @Override
- public void decodeAndFillVector(byte[] input, int offset, int length,
- ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize)
+ @Override public void decodeAndFillVector(byte[] input, int offset, int length,
+ ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize,
+ ReusableDataBuffer reusableDataBuffer)
throws MemoryException, IOException {
Compressor compressor =
CompressorFactory.getInstance().getCompressor(meta.getCompressorName());
- byte[] unCompressData = compressor.unCompressByte(input, offset, length);
+ byte[] unCompressData;
+ if (null != reusableDataBuffer && compressor.supportReusableBuffer()) {
+ int uncompressedLength = compressor.unCompressedLength(input, offset, length);
+ unCompressData = reusableDataBuffer.getDataBuffer(uncompressedLength);
+ compressor.rawUncompress(input, offset, length, unCompressData);
+ } else {
+ unCompressData = compressor.unCompressByte(input, offset, length);
+ }
if (DataTypes.isDecimal(meta.getSchemaDataType())) {
TableSpec.ColumnSpec columnSpec = meta.getColumnSpec();
DecimalConverterFactory.DecimalConverter decimalConverter =
@@ -160,8 +167,7 @@ public class AdaptiveDeltaIntegralCodec extends AdaptiveCodec {
pageSize);
}
- @Override
- public ColumnPage decode(byte[] input, int offset, int length, boolean isLVEncoded)
+ @Override public ColumnPage decode(byte[] input, int offset, int length, boolean isLVEncoded)
throws MemoryException, IOException {
return decode(input, offset, length);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
index c66c065..b16e57d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveFloatingCodec.java
@@ -23,6 +23,7 @@ import java.util.BitSet;
import java.util.List;
import java.util.Map;
+import org.apache.carbondata.core.datastore.ReusableDataBuffer;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.page.ColumnPage;
@@ -111,26 +112,31 @@ public class AdaptiveFloatingCodec extends AdaptiveCodec {
@Override
public ColumnPageDecoder createDecoder(final ColumnPageEncoderMeta meta) {
return new ColumnPageDecoder() {
- @Override
- public ColumnPage decode(byte[] input, int offset, int length)
+ @Override public ColumnPage decode(byte[] input, int offset, int length)
throws MemoryException, IOException {
ColumnPage page = ColumnPage.decompress(meta, input, offset, length, false);
return LazyColumnPage.newPage(page, converter);
}
- @Override
- public void decodeAndFillVector(byte[] input, int offset, int length,
- ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize)
+ @Override public void decodeAndFillVector(byte[] input, int offset, int length,
+ ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize,
+ ReusableDataBuffer reusableDataBuffer)
throws MemoryException, IOException {
Compressor compressor =
CompressorFactory.getInstance().getCompressor(meta.getCompressorName());
- byte[] unCompressData = compressor.unCompressByte(input, offset, length);
+ byte[] unCompressData;
+ if (null != reusableDataBuffer && compressor.supportReusableBuffer()) {
+ int uncompressedLength = compressor.unCompressedLength(input, offset, length);
+ unCompressData = reusableDataBuffer.getDataBuffer(uncompressedLength);
+ compressor.rawUncompress(input, offset, length, unCompressData);
+ } else {
+ unCompressData = compressor.unCompressByte(input, offset, length);
+ }
converter.decodeAndFillVector(unCompressData, vectorInfo, nullBits, meta.getStoreDataType(),
pageSize);
}
- @Override
- public ColumnPage decode(byte[] input, int offset, int length, boolean isLVEncoded)
+ @Override public ColumnPage decode(byte[] input, int offset, int length, boolean isLVEncoded)
throws MemoryException, IOException {
return decode(input, offset, length);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveIntegralCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveIntegralCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveIntegralCodec.java
index d9db437..66639fe 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveIntegralCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/adaptive/AdaptiveIntegralCodec.java
@@ -23,6 +23,7 @@ import java.util.BitSet;
import java.util.List;
import java.util.Map;
+import org.apache.carbondata.core.datastore.ReusableDataBuffer;
import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
@@ -107,8 +108,7 @@ public class AdaptiveIntegralCodec extends AdaptiveCodec {
@Override
public ColumnPageDecoder createDecoder(final ColumnPageEncoderMeta meta) {
return new ColumnPageDecoder() {
- @Override
- public ColumnPage decode(byte[] input, int offset, int length)
+ @Override public ColumnPage decode(byte[] input, int offset, int length)
throws MemoryException, IOException {
ColumnPage page = null;
if (DataTypes.isDecimal(meta.getSchemaDataType())) {
@@ -119,13 +119,19 @@ public class AdaptiveIntegralCodec extends AdaptiveCodec {
return LazyColumnPage.newPage(page, converter);
}
- @Override
- public void decodeAndFillVector(byte[] input, int offset, int length,
- ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize)
- throws MemoryException, IOException {
+ @Override public void decodeAndFillVector(byte[] input, int offset, int length,
+ ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize,
+ ReusableDataBuffer reusableDataBuffer) throws MemoryException, IOException {
Compressor compressor =
CompressorFactory.getInstance().getCompressor(meta.getCompressorName());
- byte[] unCompressData = compressor.unCompressByte(input, offset, length);
+ byte[] unCompressData;
+ if (null != reusableDataBuffer && compressor.supportReusableBuffer()) {
+ int uncompressedLength = compressor.unCompressedLength(input, offset, length);
+ unCompressData = reusableDataBuffer.getDataBuffer(uncompressedLength);
+ compressor.rawUncompress(input, offset, length, unCompressData);
+ } else {
+ unCompressData = compressor.unCompressByte(input, offset, length);
+ }
if (DataTypes.isDecimal(meta.getSchemaDataType())) {
TableSpec.ColumnSpec columnSpec = meta.getColumnSpec();
DecimalConverterFactory.DecimalConverter decimalConverter =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java
index 7b7c0b6..60344d6 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java
@@ -23,6 +23,7 @@ import java.util.BitSet;
import java.util.List;
import java.util.Map;
+import org.apache.carbondata.core.datastore.ReusableDataBuffer;
import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
@@ -93,8 +94,8 @@ public class DirectCompressCodec implements ColumnPageCodec {
public ColumnPageDecoder createDecoder(final ColumnPageEncoderMeta meta) {
return new ColumnPageDecoder() {
- @Override
- public ColumnPage decode(byte[] input, int offset, int length) throws MemoryException {
+ @Override public ColumnPage decode(byte[] input, int offset, int length)
+ throws MemoryException {
ColumnPage decodedPage;
if (DataTypes.isDecimal(dataType)) {
decodedPage = ColumnPage.decompressDecimalPage(meta, input, offset, length);
@@ -106,11 +107,21 @@ public class DirectCompressCodec implements ColumnPageCodec {
@Override
public void decodeAndFillVector(byte[] input, int offset, int length,
- ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize)
+ ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize,
+ ReusableDataBuffer reusableDataBuffer)
throws MemoryException, IOException {
Compressor compressor =
CompressorFactory.getInstance().getCompressor(meta.getCompressorName());
- byte[] unCompressData = compressor.unCompressByte(input, offset, length);
+ int uncompressedLength;
+ byte[] unCompressData;
+ if (null != reusableDataBuffer && compressor.supportReusableBuffer()) {
+ uncompressedLength = compressor.unCompressedLength(input, offset, length);
+ unCompressData = reusableDataBuffer.getDataBuffer(uncompressedLength);
+ compressor.rawUncompress(input, offset, length, unCompressData);
+ } else {
+ unCompressData = compressor.unCompressByte(input, offset, length);
+ uncompressedLength = unCompressData.length;
+ }
if (DataTypes.isDecimal(dataType)) {
TableSpec.ColumnSpec columnSpec = meta.getColumnSpec();
DecimalConverterFactory.DecimalConverter decimalConverter =
@@ -118,8 +129,8 @@ public class DirectCompressCodec implements ColumnPageCodec {
.getDecimalConverter(columnSpec.getPrecision(), columnSpec.getScale());
vectorInfo.decimalConverter = decimalConverter;
if (DataTypes.isDecimal(meta.getStoreDataType())) {
- ColumnPage decimalColumnPage =
- VarLengthColumnPageBase.newDecimalColumnPage(meta, unCompressData);
+ ColumnPage decimalColumnPage = VarLengthColumnPageBase
+ .newDecimalColumnPage(meta, unCompressData, uncompressedLength);
decimalConverter.fillVector(decimalColumnPage.getByteArrayPage(), pageSize, vectorInfo,
nullBits, meta.getStoreDataType());
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java
index 2c940bb..836a8b1 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java
@@ -27,6 +27,7 @@ import java.util.BitSet;
import java.util.List;
import java.util.Map;
+import org.apache.carbondata.core.datastore.ReusableDataBuffer;
import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.datastore.page.encoding.ColumnPageCodec;
@@ -295,8 +296,7 @@ public class RLECodec implements ColumnPageCodec {
this.compressorName = compressorName;
}
- @Override
- public ColumnPage decode(byte[] input, int offset, int length)
+ @Override public ColumnPage decode(byte[] input, int offset, int length)
throws MemoryException, IOException {
DataType dataType = columnSpec.getSchemaDataType();
DataInputStream in = new DataInputStream(new ByteArrayInputStream(input, offset, length));
@@ -316,9 +316,9 @@ public class RLECodec implements ColumnPageCodec {
return resultPage;
}
- @Override
- public void decodeAndFillVector(byte[] input, int offset, int length,
- ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize)
+ @Override public void decodeAndFillVector(byte[] input, int offset, int length,
+ ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize,
+ ReusableDataBuffer reusableDataBuffer)
throws MemoryException, IOException {
throw new UnsupportedOperationException("Not supposed to be called here");
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index 710fbe3..40d04e2 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -35,6 +35,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
import org.apache.carbondata.core.datamap.Segment;
import org.apache.carbondata.core.datastore.IndexKey;
+import org.apache.carbondata.core.datastore.ReusableDataBuffer;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
@@ -404,19 +405,33 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
// fill all the block execution infos for all the blocks selected in
// query
// and query will be executed based on that infos
+ ReusableDataBuffer[] dimensionReusableDataBuffers = null;
+ ReusableDataBuffer[] measureReusableDataBuffers = null;
+
for (int i = 0; i < queryProperties.dataBlocks.size(); i++) {
AbstractIndex abstractIndex = queryProperties.dataBlocks.get(i);
BlockletDataRefNode dataRefNode =
(BlockletDataRefNode) abstractIndex.getDataRefNode();
- blockExecutionInfoList.add(
- getBlockExecutionInfoForBlock(
- queryModel,
- abstractIndex,
+ final BlockExecutionInfo blockExecutionInfoForBlock =
+ getBlockExecutionInfoForBlock(queryModel, abstractIndex,
dataRefNode.getBlockInfos().get(0).getBlockletInfos().getStartBlockletNumber(),
- dataRefNode.numberOfNodes(),
- dataRefNode.getBlockInfos().get(0).getFilePath(),
+ dataRefNode.numberOfNodes(), dataRefNode.getBlockInfos().get(0).getFilePath(),
dataRefNode.getBlockInfos().get(0).getDeletedDeltaFilePath(),
- dataRefNode.getBlockInfos().get(0).getSegment()));
+ dataRefNode.getBlockInfos().get(0).getSegment());
+ if (null == dimensionReusableDataBuffers || null == measureReusableDataBuffers) {
+ dimensionReusableDataBuffers = blockExecutionInfoForBlock.getDimensionResusableDataBuffer();
+ measureReusableDataBuffers = blockExecutionInfoForBlock.getMeasureResusableDataBuffer();
+ } else {
+ if (dimensionReusableDataBuffers.length == blockExecutionInfoForBlock
+ .getDimensionResusableDataBuffer().length) {
+ blockExecutionInfoForBlock.setDimensionResusableDataBuffer(dimensionReusableDataBuffers);
+ }
+ if (measureReusableDataBuffers.length == blockExecutionInfoForBlock
+ .getMeasureResusableDataBuffer().length) {
+ blockExecutionInfoForBlock.setMeasureResusableDataBuffer(measureReusableDataBuffers);
+ }
+ }
+ blockExecutionInfoList.add(blockExecutionInfoForBlock);
}
if (null != queryModel.getStatisticsRecorder()) {
QueryStatistic queryStatistic = new QueryStatistic();
@@ -534,6 +549,16 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
int[] dimensionChunkIndexes = QueryUtil.getDimensionChunkIndexes(projectDimensions,
segmentProperties.getDimensionOrdinalToChunkMapping(),
currentBlockFilterDimensions, allProjectionListDimensionIdexes);
+ int reusableBufferSize = segmentProperties.getDimensionOrdinalToChunkMapping().size()
+ < projectDimensions.size() ?
+ projectDimensions.size() :
+ segmentProperties.getDimensionOrdinalToChunkMapping().size();
+ ReusableDataBuffer[] dimensionBuffer =
+ new ReusableDataBuffer[reusableBufferSize];
+ for (int i = 0; i < dimensionBuffer.length; i++) {
+ dimensionBuffer[i] = new ReusableDataBuffer();
+ }
+ blockExecutionInfo.setDimensionResusableDataBuffer(dimensionBuffer);
int numberOfColumnToBeReadInOneIO = Integer.parseInt(CarbonProperties.getInstance()
.getProperty(CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO,
CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_DEFAULTVALUE));
@@ -558,6 +583,12 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
currentBlockQueryMeasures, expressionMeasures,
segmentProperties.getMeasuresOrdinalToChunkMapping(), filterMeasures,
allProjectionListMeasureIndexes);
+ ReusableDataBuffer[] measureBuffer =
+ new ReusableDataBuffer[segmentProperties.getMeasuresOrdinalToChunkMapping().size()];
+ for (int i = 0; i < measureBuffer.length; i++) {
+ measureBuffer[i] = new ReusableDataBuffer();
+ }
+ blockExecutionInfo.setMeasureResusableDataBuffer(measureBuffer);
if (measureChunkIndexes.length > 0) {
numberOfElementToConsider = measureChunkIndexes[measureChunkIndexes.length - 1]
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
index f0ef23b..f19b96b 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
@@ -20,6 +20,7 @@ import java.util.Map;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.IndexKey;
+import org.apache.carbondata.core.datastore.ReusableDataBuffer;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.mutate.DeleteDeltaVo;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
@@ -221,6 +222,10 @@ public class BlockExecutionInfo {
*/
private boolean isDirectVectorFill;
+ private ReusableDataBuffer[] dimensionResusableDataBuffer;
+
+ private ReusableDataBuffer[] measureResusableDataBuffer;
+
/**
* @param blockIndex the tableBlock to set
*/
@@ -638,4 +643,20 @@ public class BlockExecutionInfo {
public void setDirectVectorFill(boolean directVectorFill) {
isDirectVectorFill = directVectorFill;
}
+
+ public ReusableDataBuffer[] getDimensionResusableDataBuffer() {
+ return dimensionResusableDataBuffer;
+ }
+
+ public void setDimensionResusableDataBuffer(ReusableDataBuffer[] dimensionResusableDataBuffer) {
+ this.dimensionResusableDataBuffer = dimensionResusableDataBuffer;
+ }
+
+ public ReusableDataBuffer[] getMeasureResusableDataBuffer() {
+ return measureResusableDataBuffer;
+ }
+
+ public void setMeasureResusableDataBuffer(ReusableDataBuffer[] measureResusableDataBuffer) {
+ this.measureResusableDataBuffer = measureResusableDataBuffer;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
index 8217487..bb373eb 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
@@ -27,6 +27,7 @@ import java.util.Map;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.ReusableDataBuffer;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnPage;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
@@ -149,8 +150,14 @@ public abstract class BlockletScannedResult {
protected LazyBlockletLoader lazyBlockletLoader;
+ private ReusableDataBuffer[] dimensionReusableBuffer;
+
+ private ReusableDataBuffer[] measureReusableBuffer;
+
public BlockletScannedResult(BlockExecutionInfo blockExecutionInfo,
QueryStatisticsModel queryStatisticsModel) {
+ this.dimensionReusableBuffer = blockExecutionInfo.getDimensionResusableDataBuffer();
+ this.measureReusableBuffer = blockExecutionInfo.getMeasureResusableDataBuffer();
this.fixedLengthKeySize = blockExecutionInfo.getFixedLengthKeySize();
this.noDictionaryColumnChunkIndexes = blockExecutionInfo.getNoDictionaryColumnChunkIndexes();
this.dictionaryColumnChunkIndexes = blockExecutionInfo.getDictionaryColumnChunkIndex();
@@ -382,15 +389,15 @@ public abstract class BlockletScannedResult {
long startTime = System.currentTimeMillis();
for (int i = 0; i < dimensionColumnPages.length; i++) {
if (dimensionColumnPages[i][pageCounter] == null && dimRawColumnChunks[i] != null) {
- dimensionColumnPages[i][pageCounter] =
- dimRawColumnChunks[i].convertToDimColDataChunkWithOutCache(pageCounter);
+ dimensionColumnPages[i][pageCounter] = dimRawColumnChunks[i]
+ .convertToDimColDataChunkWithOutCache(pageCounter, null);
}
}
for (int i = 0; i < measureColumnPages.length; i++) {
if (measureColumnPages[i][pageCounter] == null && msrRawColumnChunks[i] != null) {
- measureColumnPages[i][pageCounter] =
- msrRawColumnChunks[i].convertToColumnPageWithOutCache(pageCounter);
+ measureColumnPages[i][pageCounter] = msrRawColumnChunks[i]
+ .convertToColumnPageWithOutCache(pageCounter, null);
}
}
QueryStatistic pageUncompressTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
@@ -412,18 +419,20 @@ public abstract class BlockletScannedResult {
for (int i = 0; i < this.dictionaryColumnChunkIndexes.length; i++) {
dictionaryInfo[i].vector.setLazyPage(
new LazyPageLoader(lazyBlockletLoader, dictionaryColumnChunkIndexes[i], false,
- pageIdFiltered[pageCounter], dictionaryInfo[i]));
+ pageIdFiltered[pageCounter], dictionaryInfo[i], dimensionReusableBuffer[i]));
}
+ int startIndex = dictionaryColumnChunkIndexes.length;
for (int i = 0; i < this.noDictionaryColumnChunkIndexes.length; i++) {
noDictionaryInfo[i].vector.setLazyPage(
new LazyPageLoader(lazyBlockletLoader, noDictionaryColumnChunkIndexes[i], false,
- pageIdFiltered[pageCounter], noDictionaryInfo[i]));
+ pageIdFiltered[pageCounter], noDictionaryInfo[i],
+ dimensionReusableBuffer[startIndex++]));
}
for (int i = 0; i < measuresOrdinal.length; i++) {
msrVectorInfo[i].vector.setLazyPage(
new LazyPageLoader(lazyBlockletLoader, measuresOrdinal[i], true,
- pageIdFiltered[pageCounter], msrVectorInfo[i]));
+ pageIdFiltered[pageCounter], msrVectorInfo[i], measureReusableBuffer[i]));
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonDictionary.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonDictionary.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonDictionary.java
index 84d52a6..298eadc 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonDictionary.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonDictionary.java
@@ -22,6 +22,10 @@ public interface CarbonDictionary {
int getDictionarySize();
+ boolean isDictionaryUsed();
+
+ void setDictionaryUsed();
+
byte[] getDictionaryValue(int index);
byte[][] getAllDictionaryValues();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/CarbonDictionaryImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/CarbonDictionaryImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/CarbonDictionaryImpl.java
index 962a772..135ec3f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/CarbonDictionaryImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/CarbonDictionaryImpl.java
@@ -30,6 +30,8 @@ public class CarbonDictionaryImpl implements CarbonDictionary {
private int actualSize;
+ private boolean isDictUsed;
+
public CarbonDictionaryImpl(byte[][] dictionary, int actualSize) {
this.dictionary = dictionary;
this.actualSize = actualSize;
@@ -43,6 +45,14 @@ public class CarbonDictionaryImpl implements CarbonDictionary {
return this.dictionary.length;
}
+ @Override public boolean isDictionaryUsed() {
+ return this.isDictUsed;
+ }
+
+ @Override public void setDictionaryUsed() {
+ this.isDictUsed = true;
+ }
+
@Override public byte[] getDictionaryValue(int index) {
return dictionary[index];
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/main/java/org/apache/carbondata/core/scan/scanner/LazyPageLoader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/LazyPageLoader.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/LazyPageLoader.java
index 42af6c3..0aca79a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/LazyPageLoader.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/LazyPageLoader.java
@@ -18,6 +18,7 @@ package org.apache.carbondata.core.scan.scanner;
import java.io.IOException;
+import org.apache.carbondata.core.datastore.ReusableDataBuffer;
import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
@@ -43,14 +44,17 @@ public class LazyPageLoader {
private QueryStatisticsModel queryStatisticsModel;
+ private ReusableDataBuffer reusableDataBuffer;
+
public LazyPageLoader(LazyBlockletLoader lazyBlockletLoader, int index, boolean isMeasure,
- int pageNumber, ColumnVectorInfo vectorInfo) {
+ int pageNumber, ColumnVectorInfo vectorInfo, ReusableDataBuffer reusableDataBuffer) {
this.lazyBlockletLoader = lazyBlockletLoader;
this.lazyChunkWrapper = lazyBlockletLoader.getLazyChunkWrapper(index, isMeasure);
this.isMeasure = isMeasure;
this.pageNumber = pageNumber;
this.vectorInfo = vectorInfo;
this.queryStatisticsModel = lazyBlockletLoader.getQueryStatisticsModel();
+ this.reusableDataBuffer = reusableDataBuffer;
}
public void loadPage() {
@@ -64,10 +68,10 @@ public class LazyPageLoader {
long startTime = System.currentTimeMillis();
if (isMeasure) {
((MeasureRawColumnChunk) lazyChunkWrapper.getRawColumnChunk())
- .convertToColumnPageAndFillVector(pageNumber, vectorInfo);
+ .convertToColumnPageAndFillVector(pageNumber, vectorInfo, reusableDataBuffer);
} else {
((DimensionRawColumnChunk) lazyChunkWrapper.getRawColumnChunk())
- .convertToDimColDataChunkAndFillVector(pageNumber, vectorInfo);
+ .convertToDimColDataChunkAndFillVector(pageNumber, vectorInfo, reusableDataBuffer);
}
if (queryStatisticsModel.isEnabled()) {
QueryStatistic pageUncompressTime = queryStatisticsModel.getStatisticsTypeAndObjMap()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java
index 8dabe4c..e6f3d9a 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java
@@ -38,7 +38,8 @@ public class FixedLengthDimensionDataChunkTest {
int invertedIndexReverse[] = { 1, 0, 5, 7, 8 };
fixedLengthDimensionDataChunk =
- new FixedLengthDimensionColumnPage(data, invertedIndex, invertedIndexReverse, 5, 4);
+ new FixedLengthDimensionColumnPage(data, invertedIndex, invertedIndexReverse, 5, 4,
+ data.length);
}
@Test public void fillChunkDataTest() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java
index 2662cee..ee3bc16 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImplTest.java
@@ -184,7 +184,7 @@ public class IncludeFilterExecuterImplTest extends TestCase {
dim.setFilterKeys(filterKeys);
dimensionColumnDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
- dataChunkSize, dimColumnSize);
+ dataChunkSize, dimColumnSize, dataChunk.length);
// repeat query and compare 2 result between old code and new optimized code
for (int j = 0; j < queryTimes; j++) {
@@ -304,7 +304,7 @@ public class IncludeFilterExecuterImplTest extends TestCase {
dim.setFilterKeys(filterKeys);
dimensionColumnDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
- dataChunk.length / dimColumnSize, dimColumnSize);
+ dataChunk.length / dimColumnSize, dimColumnSize, dataChunk.length);
// initial to run
BitSet bitOld = this.setFilterdIndexToBitSetWithColumnIndexOld(dimensionColumnDataChunk, dataChunkSize, filterKeys);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index 4a5d1b3..a82a8aa 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -247,7 +247,7 @@ public class CarbonUtilTest {
byte[] dataChunks = { 5, 6, 7, 8, 9 };
byte[] compareValues = { 7 };
FixedLengthDimensionColumnPage fixedLengthDataChunk =
- new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1, dataChunks.length);
int result = CarbonUtil.nextLesserValueToTarget(2, fixedLengthDataChunk, compareValues);
assertEquals(result, 1);
}
@@ -256,7 +256,7 @@ public class CarbonUtilTest {
byte[] dataChunks = { 7, 7, 7, 8, 9 };
byte[] compareValues = { 7 };
FixedLengthDimensionColumnPage fixedLengthDataChunk =
- new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1, dataChunks.length);
int result = CarbonUtil.nextLesserValueToTarget(2, fixedLengthDataChunk, compareValues);
assertEquals(result, -1);
}
@@ -265,7 +265,7 @@ public class CarbonUtilTest {
byte[] dataChunks = { 5, 6, 7, 8, 9 };
byte[] compareValues = { 7 };
FixedLengthDimensionColumnPage fixedLengthDataChunk =
- new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1, dataChunks.length);
int result = CarbonUtil.nextGreaterValueToTarget(2, fixedLengthDataChunk, compareValues, 5);
assertEquals(result, 3);
}
@@ -282,7 +282,7 @@ public class CarbonUtilTest {
byte[] dataChunks = { 5, 6, 7, 7, 7 };
byte[] compareValues = { 7 };
FixedLengthDimensionColumnPage fixedLengthDataChunk =
- new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1, dataChunks.length);
int result = CarbonUtil.nextGreaterValueToTarget(2, fixedLengthDataChunk, compareValues, 5);
assertEquals(result, 5);
}
@@ -772,7 +772,7 @@ public class CarbonUtilTest {
byte[] dataChunks = { 10, 20, 30, 40, 50, 60 };
byte[] compareValue = { 5 };
FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk =
- new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1);
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1, dataChunks.length);
int result = CarbonUtil
.getFirstIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 1, 3, compareValue, false);
assertEquals(-2, result);
@@ -782,7 +782,7 @@ public class CarbonUtilTest {
byte[] dataChunks = { 10, 20, 30, 40, 50, 60 };
byte[] compareValue = { 30 };
FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk =
- new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1);
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1, dataChunks.length);
int result = CarbonUtil
.getFirstIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 1, 3, compareValue, false);
assertEquals(2, result);
@@ -792,7 +792,7 @@ public class CarbonUtilTest {
byte[] dataChunks = { 10, 10, 10, 40, 50, 60 };
byte[] compareValue = { 10 };
FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk =
- new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1);
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1, dataChunks.length);
int result = CarbonUtil
.getFirstIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 1, 3, compareValue, false);
assertEquals(0, result);
@@ -802,7 +802,7 @@ public class CarbonUtilTest {
byte[] dataChunks = { 10, 10, 10, 40, 50, 60 };
byte[] compareValue = { 10 };
FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk =
- new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1);
+ new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1, dataChunks.length);
int result = CarbonUtil
.getFirstIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 1, 3, compareValue, true);
assertEquals(2, result);
@@ -819,7 +819,7 @@ public class CarbonUtilTest {
dataChunk = "abbcccddddeffgggh".getBytes();
byte[][] dataArr = new byte[dataChunk.length / keyWord.length][keyWord.length];
fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
- dataChunk.length / keyWord.length, keyWord.length);
+ dataChunk.length / keyWord.length, keyWord.length, dataChunk.length);
for (int ii = 0; ii < dataChunk.length / keyWord.length; ii++) {
dataArr[ii] = fixedLengthDimensionDataChunk.getChunkData(ii);
@@ -851,7 +851,7 @@ public class CarbonUtilTest {
dataChunk = "ab".getBytes();
fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
- dataChunk.length / keyWord.length, keyWord.length);
+ dataChunk.length / keyWord.length, keyWord.length, dataChunk.length);
keyWord[0] = Byte.valueOf("97");
range = CarbonUtil.getRangeIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 0, dataChunk.length - 1, keyWord);
@@ -865,7 +865,7 @@ public class CarbonUtilTest {
dataChunk = "aabb".getBytes();
fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
- dataChunk.length / keyWord.length, keyWord.length);
+ dataChunk.length / keyWord.length, keyWord.length, dataChunk.length);
keyWord[0] = Byte.valueOf("97");
range = CarbonUtil.getRangeIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 0, dataChunk.length - 1, keyWord);
@@ -879,7 +879,7 @@ public class CarbonUtilTest {
dataChunk = "a".getBytes();
fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
- dataChunk.length / keyWord.length, keyWord.length);
+ dataChunk.length / keyWord.length, keyWord.length, dataChunk.length);
keyWord[0] = Byte.valueOf("97");
range = CarbonUtil.getRangeIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 0, dataChunk.length - 1, keyWord);
@@ -888,7 +888,7 @@ public class CarbonUtilTest {
dataChunk = "aa".getBytes();
fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
- dataChunk.length / keyWord.length, keyWord.length);
+ dataChunk.length / keyWord.length, keyWord.length, dataChunk.length);
keyWord[0] = Byte.valueOf("97");
range = CarbonUtil.getRangeIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 0, dataChunk.length - 1, keyWord);
@@ -897,7 +897,7 @@ public class CarbonUtilTest {
dataChunk = "aabbbbbbbbbbcc".getBytes();
fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
- dataChunk.length / keyWord.length, keyWord.length);
+ dataChunk.length / keyWord.length, keyWord.length, dataChunk.length);
keyWord[0] = Byte.valueOf("98");
range = CarbonUtil.getRangeIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 0, dataChunk.length - 1, keyWord);
assertEquals(2, range[0]);
@@ -917,7 +917,7 @@ public class CarbonUtilTest {
byte[][] dataArr = new byte[dataChunk.length / keyWord.length][keyWord.length];
fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
- dataChunk.length / keyWord.length, keyWord.length);
+ dataChunk.length / keyWord.length, keyWord.length, dataChunk.length);
for (int ii = 0; ii < dataChunk.length / keyWord.length; ii++) {
dataArr[ii] = fixedLengthDimensionDataChunk.getChunkData(ii);
@@ -959,7 +959,7 @@ public class CarbonUtilTest {
byte[][] dataArr = new byte[dataChunk.length / keyWord.length][keyWord.length];
fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunk, null, null,
- dataChunk.length / keyWord.length, keyWord.length);
+ dataChunk.length / keyWord.length, keyWord.length, dataChunk.length);
for (int ii = 0; ii < dataChunk.length / keyWord.length; ii++) {
dataArr[ii] = fixedLengthDimensionDataChunk.getChunkData(ii);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithCompression.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithCompression.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithCompression.scala
index f5153fc..6ec44e7 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithCompression.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithCompression.scala
@@ -147,6 +147,21 @@ class CustomizeCompressor extends Compressor {
override def supportUnsafe(): Boolean = {
false
}
+
+ override def unCompressedLength(data: Array[Byte],
+ offset: Int,
+ length: Int): Int = {
+ throw new RuntimeException("Unsupported operation Exception")
+ }
+
+ override def rawUncompress(data: Array[Byte],
+ offset: Int,
+ length: Int,
+ output: Array[Byte]): Int = {
+ throw new RuntimeException("Unsupported operation Exception")
+ }
+
+ override def supportReusableBuffer(): Boolean = false
}
class TestLoadDataWithCompression extends QueryTest with BeforeAndAfterEach with BeforeAndAfterAll {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
----------------------------------------------------------------------
diff --git a/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java b/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
index 8082511..1f28b8c 100644
--- a/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
+++ b/integration/spark-datasource/src/main/scala/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
@@ -365,7 +365,6 @@ public class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> {
for (int i = 0; i < isNoDictStringField.length; i++) {
if (isNoDictStringField[i]) {
vectorProxy.resetDictionaryIds(i);
- vectorProxy.column(i).setDictionary(null);
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e7dd19be/tools/cli/src/main/java/org/apache/carbondata/tool/ScanBenchmark.java
----------------------------------------------------------------------
diff --git a/tools/cli/src/main/java/org/apache/carbondata/tool/ScanBenchmark.java b/tools/cli/src/main/java/org/apache/carbondata/tool/ScanBenchmark.java
index 12384fc..07682ed 100644
--- a/tools/cli/src/main/java/org/apache/carbondata/tool/ScanBenchmark.java
+++ b/tools/cli/src/main/java/org/apache/carbondata/tool/ScanBenchmark.java
@@ -205,7 +205,7 @@ class ScanBenchmark implements Command {
DimensionColumnPage[] pages = new DimensionColumnPage[numPages];
for (int i = 0; i < pages.length; i++) {
pages[i] = dimensionColumnChunkReader.decodeColumnPage(
- (DimensionRawColumnChunk) rawColumnChunk, i);
+ (DimensionRawColumnChunk) rawColumnChunk, i, null);
}
return pages;
}
@@ -215,7 +215,7 @@ class ScanBenchmark implements Command {
ColumnPage[] pages = new ColumnPage[numPages];
for (int i = 0; i < pages.length; i++) {
pages[i] = measureColumnChunkReader.decodeColumnPage(
- (MeasureRawColumnChunk) rawColumnChunk, i);
+ (MeasureRawColumnChunk) rawColumnChunk, i, null);
}
return pages;
}