You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/07/28 06:10:07 UTC
[7/7] carbondata git commit: Resolve rebase conflicts when rebasing
branch encoding_override onto master
Resolve rebase conflicts when rebasing branch encoding_override onto master
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/74226907
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/74226907
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/74226907
Branch: refs/heads/master
Commit: 74226907990cdee41a6ccbd69e2a813077792f89
Parents: a5af0ff
Author: Raghunandan S <ca...@gmail.com>
Authored: Wed Jul 26 21:59:05 2017 +0800
Committer: Raghunandan S <ca...@gmail.com>
Committed: Fri Jul 28 01:24:38 2017 +0800
----------------------------------------------------------------------
.../carbondata/core/datastore/TableSpec.java | 38 ++++++++++-------
.../AbstractMeasureChunkReaderV2V3Format.java | 1 -
...CompressedMeasureChunkFileBasedReaderV1.java | 2 +-
...CompressedMeasureChunkFileBasedReaderV2.java | 18 ++++++--
...CompressedMeasureChunkFileBasedReaderV3.java | 16 +++++--
.../core/datastore/page/ColumnPage.java | 5 +--
.../page/encoding/AdaptiveIntegralCodec.java | 3 +-
.../page/encoding/DeltaIntegralCodec.java | 3 +-
.../page/encoding/EncodingStrategy.java | 2 +-
.../core/datastore/page/key/TablePageKey.java | 4 +-
.../statistics/PrimitivePageStatsCollector.java | 44 +++++++++++++++++---
.../page/statistics/SimpleStatsResult.java | 4 ++
.../statistics/VarLengthPageStatsCollector.java | 8 ++++
.../core/metadata/ColumnPageCodecMeta.java | 20 +++++++--
.../core/metadata/ValueEncoderMeta.java | 20 +++++++++
.../apache/carbondata/core/util/CarbonUtil.java | 1 +
.../newflow/sort/SortStepRowUtil.java | 4 --
.../sortdata/SortTempFileChunkHolder.java | 1 +
.../carbondata/processing/store/TablePage.java | 12 +++---
.../store/writer/AbstractFactDataWriter.java | 12 +++---
20 files changed, 162 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java b/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
index f1d3546..818f46e 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
@@ -75,7 +75,8 @@ public class TableSpec {
private void addMeasures(List<CarbonMeasure> measures) {
for (int i = 0; i < measures.size(); i++) {
CarbonMeasure measure = measures.get(i);
- measureSpec[i] = new MeasureSpec(measure.getColName(), measure.getDataType());
+ measureSpec[i] = new MeasureSpec(measure.getColName(), measure.getDataType(), measure
+ .getScale(), measure.getPrecision());
}
}
@@ -95,20 +96,14 @@ public class TableSpec {
return dimensionSpec.length;
}
- public int getScale(int index) {
- assert (index >= 0 && index < precision.length);
- return scale[index];
- }
+ /**
+ * return number of measures
+ */
+ public int getNumMeasures() {
+ return measureSpec.length;
+ }
- public int getPrecision(int index) {
- assert (index >= 0 && index < precision.length);
- return precision[index];
- }/**
- * return number of measures
- */
- public int getNumMeasures() {
- return measureSpec.length;
- }public class ColumnSpec {
+ public class ColumnSpec {
// field name of this column
private String fieldName;
@@ -162,8 +157,21 @@ public class TableSpec {
public class MeasureSpec extends ColumnSpec {
- MeasureSpec(String fieldName, DataType dataType) {
+ private int scale;
+ private int precision;
+
+ MeasureSpec(String fieldName, DataType dataType, int scale, int precision) {
super(fieldName, dataType);
+ this.scale = scale;
+ this.precision = precision;
+ }
+
+ public int getScale() {
+ return scale;
+ }
+
+ public int getPrecision() {
+ return precision;
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
index dd61826..049aba9 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
@@ -17,7 +17,6 @@
package org.apache.carbondata.core.datastore.chunk.reader.measure;
import java.io.IOException;
-import java.nio.ByteBuffer;
import java.util.BitSet;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
index 6bf65da..7df18db 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
@@ -98,7 +98,7 @@ public class CompressedMeasureChunkFileBasedReaderV1 extends AbstractMeasureChun
DataChunk dataChunk = measureColumnChunks.get(blockIndex);
ValueEncoderMeta meta = dataChunk.getValueEncoderMeta().get(0);
- ColumnPageCodec codec = strategy.newCodec(meta);
+ ColumnPageCodec codec = strategy.newCodec(meta, -1, -1);
ColumnPage page = codec.decode(measureRawColumnChunk.getRawData().array(),
measureRawColumnChunk.getOffSet(), dataChunk.getDataPageLength());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
index 7511b6e..f2679ae 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
@@ -18,6 +18,7 @@ package org.apache.carbondata.core.datastore.chunk.reader.measure.v2;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.List;
import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
@@ -130,12 +131,21 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
protected ColumnPage decodeMeasure(MeasureRawColumnChunk measureRawColumnChunk,
DataChunk2 measureColumnChunk, int copyPoint) throws MemoryException, IOException {
- // for measure, it should have only one ValueEncoderMeta
- assert (measureColumnChunk.getEncoder_meta().size() == 1);
- byte[] encodedMeta = measureColumnChunk.getEncoder_meta().get(0).array();
+ assert (measureColumnChunk.getEncoder_meta().size() > 0);
+ List<ByteBuffer> encoder_meta = measureColumnChunk.getEncoder_meta();
+ byte[] encodedMeta = encoder_meta.get(0).array();
ValueEncoderMeta meta = CarbonUtil.deserializeEncoderMetaV3(encodedMeta);
- ColumnPageCodec codec = strategy.newCodec(meta);
+ int scale = -1;
+ int precision = -1;
+ if (encoder_meta.size() > 1) {
+ ByteBuffer decimalInfo = encoder_meta.get(1);
+ scale = decimalInfo.getInt();
+ precision = decimalInfo.getInt();
+ }
+
+
+ ColumnPageCodec codec = strategy.newCodec(meta, scale, precision);
byte[] rawData = measureRawColumnChunk.getRawData().array();
return codec.decode(rawData, copyPoint, measureColumnChunk.data_page_length);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
index 1881791..354aa38 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v3/CompressedMeasureChunkFileBasedReaderV3.java
@@ -18,6 +18,7 @@ package org.apache.carbondata.core.datastore.chunk.reader.measure.v3;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.List;
import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
@@ -226,13 +227,22 @@ public class CompressedMeasureChunkFileBasedReaderV3 extends AbstractMeasureChun
protected ColumnPage decodeMeasure(MeasureRawColumnChunk measureRawColumnChunk,
DataChunk2 measureColumnChunk, int copyPoint) throws MemoryException {
+ List<ByteBuffer> encoder_meta = measureColumnChunk.getEncoder_meta();
// for measure, it should have only one ValueEncoderMeta
- assert (measureColumnChunk.getEncoder_meta().size() == 1);
- byte[] encodedMeta = measureColumnChunk.getEncoder_meta().get(0).array();
+ assert (encoder_meta.size() > 0);
+ byte[] encodedMeta = encoder_meta.get(0).array();
+
+ int scale = -1;
+ int precision = -1;
+ if (encoder_meta.size() > 1) {
+ ByteBuffer decimalInfo = encoder_meta.get(1);
+ scale = decimalInfo.getInt();
+ precision = decimalInfo.getInt();
+ }
ColumnPageCodecMeta meta = new ColumnPageCodecMeta();
meta.deserialize(encodedMeta);
- ColumnPageCodec codec = strategy.newCodec(meta);
+ ColumnPageCodec codec = strategy.newCodec(meta, scale, precision);
byte[] rawData = measureRawColumnChunk.getRawData().array();
return codec.decode(rawData, copyPoint, measureColumnChunk.data_page_length);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
index 90300d9..245e95b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
@@ -59,8 +59,6 @@ public abstract class ColumnPage {
this.pageSize = pageSize;
this.scale = scale;
this.precision = precision;
- this.stats = new ColumnPageStatsVO(dataType);
- this.nullBitSet = new BitSet(pageSize);
if (dataType == DECIMAL) {
decimalConverter = DecimalConverterFactory.INSTANCE.getDecimalConverter(precision, scale);
}
@@ -184,7 +182,7 @@ public abstract class ColumnPage {
instance = newDecimalPage(new byte[pageSize][], scale, precision);
break;
case BYTE_ARRAY:
- instance = new SafeVarLengthColumnPage(dataType, pageSize);
+ instance = new SafeVarLengthColumnPage(dataType, pageSize, scale, precision);
break;
default:
throw new RuntimeException("Unsupported data dataType: " + dataType);
@@ -328,6 +326,7 @@ public abstract class ColumnPage {
break;
case DECIMAL:
putDecimal(rowId, (BigDecimal) value);
+ statsCollector.update(((BigDecimal) value).unscaledValue().longValue());
break;
case BYTE_ARRAY:
putBytes(rowId, (byte[]) value);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/AdaptiveIntegralCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/AdaptiveIntegralCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/AdaptiveIntegralCodec.java
index ed8d734..6480bcd 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/AdaptiveIntegralCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/AdaptiveIntegralCodec.java
@@ -65,7 +65,8 @@ class AdaptiveIntegralCodec extends AdaptiveCompressionCodec {
@Override
public ColumnPage decode(byte[] input, int offset, int length) throws MemoryException {
- ColumnPage page = ColumnPage.decompress(compressor, targetDataType, input, offset, length);
+ ColumnPage page = ColumnPage.decompress(compressor, targetDataType, input, offset, length,
+ stats.getScale(), stats.getPrecision());
return LazyColumnPage.newPage(page, codec);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java
index 53a8295..ba61cd3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java
@@ -87,7 +87,8 @@ public class DeltaIntegralCodec extends AdaptiveCompressionCodec {
@Override
public ColumnPage decode(byte[] input, int offset, int length) throws MemoryException {
- ColumnPage page = ColumnPage.decompress(compressor, targetDataType, input, offset, length);
+ ColumnPage page = ColumnPage.decompress(compressor, targetDataType, input, offset, length,
+ stats.getScale(), stats.getPrecision());
return LazyColumnPage.newPage(page, codec);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingStrategy.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingStrategy.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingStrategy.java
index 29219ea..53c565d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingStrategy.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingStrategy.java
@@ -76,7 +76,7 @@ public abstract class EncodingStrategy {
throw new RuntimeException("unsupported data type: " + stats.getDataType());
}
} else {
- SimpleStatsResult stats = PrimitivePageStatsCollector.newInstance(meta, scale, precision);
+ SimpleStatsResult stats = PrimitivePageStatsCollector.newInstance(meta);
switch (meta.getType()) {
case BYTE:
case SHORT:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
index ef8307e..f8a10ca 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/key/TablePageKey.java
@@ -68,12 +68,12 @@ public class TablePageKey {
currentNoDictionaryKey = WriteStepRowUtil.getNoDictAndComplexDimension(row);
}
if (rowId == 0) {
- startKey = WriteStepRowUtil.getMdk(row, model.getMDKeyGenerator());
+ startKey = WriteStepRowUtil.getMdk(row, mdkGenerator);
noDictStartKey = currentNoDictionaryKey;
}
noDictEndKey = currentNoDictionaryKey;
if (rowId == pageSize - 1) {
- endKey = WriteStepRowUtil.getMdk(row, model.getMDKeyGenerator());
+ endKey = WriteStepRowUtil.getMdk(row, mdkGenerator);
finalizeKeys();
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
index 73ada4b..af40f03 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
@@ -32,6 +32,7 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
private int minInt, maxInt;
private long minLong, maxLong;
private double minDouble, maxDouble;
+ private int scale, precision;
// scale of the double value
private int decimal;
@@ -40,17 +41,19 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
private BitSet nullBitSet;
// this is for encode flow
- public static PrimitivePageStatsCollector newInstance(DataType dataType, int pageSize) {
+ public static PrimitivePageStatsCollector newInstance(DataType dataType, int pageSize, int
+ scale, int precision) {
switch (dataType) {
default:
- return new PrimitivePageStatsCollector(dataType, pageSize);
+ return new PrimitivePageStatsCollector(dataType, pageSize, scale, precision);
}
}
// this is for decode flow, we do not need to create nullBits, so passing 0 as pageSize
public static PrimitivePageStatsCollector newInstance(ColumnPageCodecMeta meta) {
PrimitivePageStatsCollector instance =
- new PrimitivePageStatsCollector(meta.getSrcDataType(), 0);
+ new PrimitivePageStatsCollector(meta.getSrcDataType(), 0, meta.getScale(),
+ meta.getPrecision());
// set min max from meta
switch (meta.getSrcDataType()) {
case BYTE:
@@ -74,13 +77,20 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
instance.maxDouble = (double) meta.getMaxValue();
instance.decimal = meta.getDecimal();
break;
+ case DECIMAL:
+ instance.minLong = (long) meta.getMinValue();
+ instance.maxLong = (long) meta.getMaxValue();
+ instance.decimal = meta.getDecimal();
+ instance.scale = meta.getScale();
+ instance.precision = meta.getPrecision();
+ break;
}
return instance;
}
public static PrimitivePageStatsCollector newInstance(ValueEncoderMeta meta) {
PrimitivePageStatsCollector instance =
- new PrimitivePageStatsCollector(meta.getType(), 0);
+ new PrimitivePageStatsCollector(meta.getType(), 0, meta.getScale(), meta.getPrecision());
// set min max from meta
switch (meta.getType()) {
case BYTE:
@@ -104,11 +114,18 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
instance.maxDouble = (double) meta.getMaxValue();
instance.decimal = meta.getDecimal();
break;
+ case DECIMAL:
+ instance.minDouble = (double) meta.getMinValue();
+ instance.maxDouble = (double) meta.getMaxValue();
+ instance.decimal = meta.getDecimal();
+ instance.scale = meta.getScale();
+ instance.precision = meta.getPrecision();
+ break;
}
return instance;
}
- private PrimitivePageStatsCollector(DataType dataType, int pageSize) {
+ private PrimitivePageStatsCollector(DataType dataType, int pageSize, int scale, int precision) {
this.dataType = dataType;
this.nullBitSet = new BitSet(pageSize);
switch (dataType) {
@@ -134,6 +151,11 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
decimal = 0;
break;
case DECIMAL:
+ minLong = Long.MAX_VALUE;
+ maxLong = Long.MIN_VALUE;
+ decimal = scale;
+ this.scale = scale;
+ this.precision = precision;
}
}
@@ -255,6 +277,8 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
return minLong;
case DOUBLE:
return minDouble;
+ case DECIMAL:
+ return minLong;
}
return null;
}
@@ -272,6 +296,8 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
return maxLong;
case DOUBLE:
return maxDouble;
+ case DECIMAL:
+ return maxLong;
}
return null;
}
@@ -291,4 +317,12 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
return dataType;
}
+ @Override public int getScale() {
+ return scale;
+ }
+
+ @Override public int getPrecision() {
+ return precision;
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/SimpleStatsResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/SimpleStatsResult.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/SimpleStatsResult.java
index 1db86ff..b40d023 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/SimpleStatsResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/SimpleStatsResult.java
@@ -32,4 +32,8 @@ public interface SimpleStatsResult {
int getDecimalPoint();
DataType getDataType();
+
+ int getScale();
+
+ int getPrecision();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/VarLengthPageStatsCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/VarLengthPageStatsCollector.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/VarLengthPageStatsCollector.java
index e985f90..0fe5960 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/VarLengthPageStatsCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/VarLengthPageStatsCollector.java
@@ -102,6 +102,14 @@ public class VarLengthPageStatsCollector implements ColumnPageStatsCollector {
@Override public DataType getDataType() {
return null;
}
+
+ @Override public int getScale() {
+ return 0;
+ }
+
+ @Override public int getPrecision() {
+ return 0;
+ }
};
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/metadata/ColumnPageCodecMeta.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/ColumnPageCodecMeta.java b/core/src/main/java/org/apache/carbondata/core/metadata/ColumnPageCodecMeta.java
index 20a7568..6322670 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/ColumnPageCodecMeta.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/ColumnPageCodecMeta.java
@@ -57,6 +57,8 @@ public class ColumnPageCodecMeta extends ValueEncoderMeta implements Serializabl
meta.setMaxValue(stats.getMax());
meta.setMinValue(stats.getMin());
meta.setDecimal(stats.getDecimalPoint());
+ meta.setScale(stats.getScale());
+ meta.setPrecision(stats.getPrecision());
return meta;
}
@@ -169,8 +171,17 @@ public class ColumnPageCodecMeta extends ValueEncoderMeta implements Serializabl
buffer.putDouble((Double) 0d); // unique value is obsoleted, maintain for compatibility
break;
case DECIMAL:
- buffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE + 3);
+ buffer = ByteBuffer.allocate(
+ (CarbonCommonConstants.LONG_SIZE_IN_BYTE * 3) + (CarbonCommonConstants
+ .INT_SIZE_IN_BYTE * 3)
+ + 3);
buffer.putChar(getSrcDataTypeInChar());
+ buffer.putLong((Long) getMaxValue());
+ buffer.putLong((Long) getMinValue());
+ buffer.putLong((Long) 0L); // unique value is obsoleted, maintain for compatibility
+ buffer.putInt(getScale());
+ buffer.putInt(getPrecision());
+
break;
}
buffer.putInt(getDecimal());
@@ -190,8 +201,11 @@ public class ColumnPageCodecMeta extends ValueEncoderMeta implements Serializabl
buffer.getDouble(); // for non exist value which is obsoleted, it is backward compatibility;
break;
case BIG_DECIMAL_MEASURE:
- this.setMaxValue(0.0);
- this.setMinValue(0.0);
+ this.setMaxValue(buffer.getLong());
+ this.setMinValue(buffer.getLong());
+ buffer.getLong();
+ this.setScale(buffer.getInt());
+ this.setPrecision(buffer.getInt());
break;
case BYTE_VALUE_MEASURE:
this.setMaxValue(buffer.get());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java b/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java
index 971359d..f1ebbd3 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java
@@ -49,6 +49,10 @@ public class ValueEncoderMeta implements Serializable {
private byte dataTypeSelected;
+ private int scale;
+
+ private int precision;
+
public Object getMaxValue() {
return maxValue;
}
@@ -109,4 +113,20 @@ public class ValueEncoderMeta implements Serializable {
public void setDataTypeSelected(byte dataTypeSelected) {
this.dataTypeSelected = dataTypeSelected;
}
+
+ public int getScale() {
+ return scale;
+ }
+
+ public void setScale(int scale) {
+ this.scale = scale;
+ }
+
+ public int getPrecision() {
+ return precision;
+ }
+
+ public void setPrecision(int precision) {
+ this.precision = precision;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index eff8f0d..7628415 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -53,6 +53,7 @@ import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
index 1bcbe54..7857f4e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
@@ -17,10 +17,6 @@
package org.apache.carbondata.processing.newflow.sort;
-import java.math.BigDecimal;
-
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.core.util.NonDictionaryUtil;
import org.apache.carbondata.processing.sortandgroupby.sortdata.SortParameters;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
index b17c69a..9732e66 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
@@ -35,6 +35,7 @@ import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.core.util.NonDictionaryUtil;
import org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java b/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
index 9881e8e..03f3e5e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
@@ -47,7 +47,6 @@ import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.spark.sql.types.Decimal;
@@ -82,13 +81,13 @@ public class TablePage {
int numDictDimension = model.getMDKeyGenerator().getDimCount();
dictDimensionPages = new ColumnPage[numDictDimension];
for (int i = 0; i < dictDimensionPages.length; i++) {
- ColumnPage page = ColumnPage.newPage(DataType.BYTE_ARRAY, pageSize);
+ ColumnPage page = ColumnPage.newPage(DataType.BYTE_ARRAY, pageSize, -1, -1);
page.setStatsCollector(VarLengthPageStatsCollector.newInstance());
dictDimensionPages[i] = page;
}
noDictDimensionPages = new ColumnPage[model.getNoDictionaryCount()];
for (int i = 0; i < noDictDimensionPages.length; i++) {
- ColumnPage page = ColumnPage.newPage(DataType.BYTE_ARRAY, pageSize);
+ ColumnPage page = ColumnPage.newPage(DataType.BYTE_ARRAY, pageSize, -1, -1);
page.setStatsCollector(VarLengthPageStatsCollector.newInstance());
noDictDimensionPages[i] = page;
}
@@ -100,11 +99,12 @@ public class TablePage {
}
measurePage = new ColumnPage[model.getMeasureCount()];
DataType[] dataTypes = model.getMeasureDataType();
- TableSpec.MeasureSpec measureSpec = model.getTableSpec().getMeasureSpec();
for (int i = 0; i < measurePage.length; i++) {
+ TableSpec.MeasureSpec measureSpec = model.getTableSpec().getMeasureSpec(i);
ColumnPage page = ColumnPage
- .newPage(dataTypes[i], pageSize, measureSpec.getScale(i), measureSpec.getPrecision(i));
- page.setStatsCollector(PrimitivePageStatsCollector.newInstance(dataTypes[i], pageSize));
+ .newPage(dataTypes[i], pageSize, measureSpec.getScale(), measureSpec.getPrecision());
+ page.setStatsCollector(PrimitivePageStatsCollector.newInstance(dataTypes[i], pageSize,
+ measureSpec.getScale(), measureSpec.getPrecision()));
measurePage[i] = page;
}
boolean hasNoDictionary = noDictDimensionPages.length > 0;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/74226907/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index b83a82a..a34ed01 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -24,7 +24,7 @@ import java.io.FileFilter;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.nio.ByteBuffer;
+//import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Arrays;
@@ -61,7 +61,6 @@ import org.apache.carbondata.core.util.CarbonMergerUtil;
import org.apache.carbondata.core.util.CarbonMetadataUtil;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.NodeHolder;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.core.writer.CarbonIndexFileWriter;
import org.apache.carbondata.format.BlockIndex;
@@ -608,10 +607,11 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
* @return min max value without length
*/
protected byte[] updateMinMaxForNoDictionary(byte[] valueWithLength) {
- ByteBuffer buffer = ByteBuffer.wrap(valueWithLength);
- byte[] actualValue = new byte[buffer.getShort()];
- buffer.get(actualValue);
- return actualValue;
+ return valueWithLength;
+// ByteBuffer buffer = ByteBuffer.wrap(valueWithLength);
+// byte[] actualValue = new byte[buffer.getShort()];
+// buffer.get(actualValue);
+// return actualValue;
}
/**