You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/07/17 01:56:47 UTC
[01/15] carbondata git commit: Optimizing decimal datatype [Forced
Update!]
Repository: carbondata
Updated Branches:
refs/heads/datamap b385d14b4 -> f4ab1ff69 (forced update)
Optimizing decimal datatype
Optimized big decimal to use less space
Fixed comments
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/12911629
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/12911629
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/12911629
Branch: refs/heads/datamap
Commit: 12911629c84aabf2400fddb74c97a4ec54a533e8
Parents: 403c3d9
Author: ravipesala <ra...@gmail.com>
Authored: Wed Jun 28 16:03:03 2017 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Tue Jul 11 15:50:01 2017 +0800
----------------------------------------------------------------------
.../carbondata/core/datastore/TableSpec.java | 23 +-
.../AbstractMeasureChunkReaderV2V3Format.java | 15 +-
...CompressedMeasureChunkFileBasedReaderV1.java | 2 +-
.../core/datastore/page/ColumnPage.java | 107 ++++++---
.../core/datastore/page/LazyColumnPage.java | 13 +-
.../datastore/page/SafeFixLengthColumnPage.java | 14 +-
.../datastore/page/SafeVarLengthColumnPage.java | 11 +-
.../page/UnsafeFixLengthColumnPage.java | 13 +-
.../page/UnsafeVarLengthColumnPage.java | 19 +-
.../datastore/page/VarLengthColumnPageBase.java | 76 +++++-
.../page/encoding/AdaptiveIntegerCodec.java | 11 +-
.../page/encoding/DefaultEncodingStrategy.java | 8 +-
.../page/encoding/DeltaIntegerCodec.java | 11 +-
.../page/encoding/DirectCompressCodec.java | 16 +-
.../page/encoding/EncodingStrategy.java | 4 +-
.../page/statistics/ColumnPageStatsVO.java | 18 +-
.../datatype/DecimalConverterFactory.java | 237 +++++++++++++++++++
.../core/util/CarbonMetadataUtil.java | 28 +++
.../newflow/sort/SortStepRowUtil.java | 17 +-
.../sort/unsafe/UnsafeCarbonRowPage.java | 2 +-
.../holder/UnsafeSortTempFileChunkHolder.java | 3 +-
.../CarbonRowDataWriterProcessorStepImpl.java | 14 +-
.../sortdata/SortTempFileChunkHolder.java | 3 +-
.../carbondata/processing/store/TablePage.java | 14 +-
24 files changed, 557 insertions(+), 122 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java b/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
index 365f1ca..650c2a6 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/TableSpec.java
@@ -188,18 +188,27 @@ public class TableSpec {
// data type of each measure, in schema order
private DataType[] types;
+ private int[] scale;
+
+ private int[] precision;
+
MeasureSpec(List<CarbonMeasure> measures) {
fieldName = new String[measures.size()];
types = new DataType[measures.size()];
+ scale = new int[measures.size()];
+ precision = new int[measures.size()];
int i = 0;
for (CarbonMeasure measure: measures) {
- add(i++, measure.getColName(), measure.getDataType());
+ add(i++, measure.getColName(), measure.getDataType(), measure.getScale(),
+ measure.getPrecision());
}
}
- private void add(int index, String name, DataType type) {
+ private void add(int index, String name, DataType type, int scale, int precision) {
fieldName[index] = name;
types[index] = type;
+ this.scale[index] = scale;
+ this.precision[index] = precision;
}
/**
@@ -210,6 +219,16 @@ public class TableSpec {
return types[index];
}
+ public int getScale(int index) {
+ assert (index >= 0 && index < precision.length);
+ return scale[index];
+ }
+
+ public int getPrecision(int index) {
+ assert (index >= 0 && index < precision.length);
+ return precision[index];
+ }
+
/**
* return number of measures
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
index 2f5af87..c35cefb 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/AbstractMeasureChunkReaderV2V3Format.java
@@ -17,6 +17,7 @@
package org.apache.carbondata.core.datastore.chunk.reader.measure;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.BitSet;
import java.util.List;
@@ -132,10 +133,18 @@ public abstract class AbstractMeasureChunkReaderV2V3Format extends AbstractMeasu
protected ColumnPage decodeMeasure(MeasureRawColumnChunk measureRawColumnChunk,
DataChunk2 measureColumnChunk, int copyPoint) throws MemoryException {
// for measure, it should have only one ValueEncoderMeta
- assert (measureColumnChunk.getEncoder_meta().size() == 1);
- byte[] encodedMeta = measureColumnChunk.getEncoder_meta().get(0).array();
+ List<ByteBuffer> encoder_meta = measureColumnChunk.getEncoder_meta();
+ assert (encoder_meta.size() > 0);
+ byte[] encodedMeta = encoder_meta.get(0).array();
ValueEncoderMeta meta = CarbonUtil.deserializeEncoderMetaV3(encodedMeta);
- ColumnPageCodec codec = strategy.createCodec(meta);
+ int scale = -1;
+ int precision = -1;
+ if (encoder_meta.size() > 1) {
+ ByteBuffer decimalInfo = encoder_meta.get(1);
+ scale = decimalInfo.getInt();
+ precision = decimalInfo.getInt();
+ }
+ ColumnPageCodec codec = strategy.createCodec(meta, scale, precision);
byte[] rawData = measureRawColumnChunk.getRawData().array();
return codec.decode(rawData, copyPoint, measureColumnChunk.data_page_length);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
index 6e59b9f..8f69a7c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
@@ -98,7 +98,7 @@ public class CompressedMeasureChunkFileBasedReaderV1 extends AbstractMeasureChun
DataChunk dataChunk = measureColumnChunks.get(blockIndex);
ValueEncoderMeta meta = dataChunk.getValueEncoderMeta().get(0);
- ColumnPageCodec codec = strategy.createCodec(meta);
+ ColumnPageCodec codec = strategy.createCodec(meta, -1, -1);
ColumnPage page = codec.decode(measureRawColumnChunk.getRawData().array(),
measureRawColumnChunk.getOffSet(), dataChunk.getDataPageLength());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
index 730243c..2c43165 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
@@ -26,8 +26,8 @@ import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.page.statistics.ColumnPageStatsVO;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DecimalConverterFactory;
import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.DataTypeUtil;
import static org.apache.carbondata.core.metadata.datatype.DataType.BYTE;
import static org.apache.carbondata.core.metadata.datatype.DataType.BYTE_ARRAY;
@@ -43,6 +43,8 @@ public abstract class ColumnPage {
protected final int pageSize;
protected final DataType dataType;
+ protected int scale;
+ protected int precision;
// statistics of this column page
private ColumnPageStatsVO stats;
@@ -50,15 +52,22 @@ public abstract class ColumnPage {
// The index of the rowId whose value is null, will be set to 1
private BitSet nullBitSet;
+ protected DecimalConverterFactory.DecimalConverter decimalConverter;
+
protected static final boolean unsafe = Boolean.parseBoolean(CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE_LOADING,
CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE_LOADING_DEFAULT));
- protected ColumnPage(DataType dataType, int pageSize) {
+ protected ColumnPage(DataType dataType, int pageSize, int scale, int precision) {
this.dataType = dataType;
this.pageSize = pageSize;
+ this.scale = scale;
+ this.precision = precision;
this.stats = new ColumnPageStatsVO(dataType);
this.nullBitSet = new BitSet(pageSize);
+ if (dataType == DECIMAL) {
+ decimalConverter = DecimalConverterFactory.INSTANCE.getDecimalConverter(precision, scale);
+ }
}
public DataType getDataType() {
@@ -73,54 +82,62 @@ public abstract class ColumnPage {
return pageSize;
}
- private static ColumnPage createVarLengthPage(DataType dataType, int pageSize) {
+ private static ColumnPage createVarLengthPage(DataType dataType, int pageSize, int scale,
+ int precision) {
if (unsafe) {
try {
- return new UnsafeVarLengthColumnPage(dataType, pageSize);
+ return new UnsafeVarLengthColumnPage(dataType, pageSize, scale, precision);
} catch (MemoryException e) {
throw new RuntimeException(e);
}
} else {
- return new SafeVarLengthColumnPage(dataType, pageSize);
+ return new SafeVarLengthColumnPage(dataType, pageSize, scale, precision);
}
}
- private static ColumnPage createFixLengthPage(DataType dataType, int pageSize) {
+ private static ColumnPage createFixLengthPage(DataType dataType, int pageSize, int scale,
+ int precision) {
if (unsafe) {
try {
- return new UnsafeFixLengthColumnPage(dataType, pageSize);
+ return new UnsafeFixLengthColumnPage(dataType, pageSize, scale, precision);
} catch (MemoryException e) {
throw new RuntimeException(e);
}
} else {
- return new SafeFixLengthColumnPage(dataType, pageSize);
+ return new SafeFixLengthColumnPage(dataType, pageSize, scale, pageSize);
}
}
- private static ColumnPage createPage(DataType dataType, int pageSize) {
+ private static ColumnPage createPage(DataType dataType, int pageSize, int scale, int precision) {
if (dataType.equals(BYTE_ARRAY) | dataType.equals(DECIMAL)) {
- return createVarLengthPage(dataType, pageSize);
+ return createVarLengthPage(dataType, pageSize, scale, precision);
} else {
- return createFixLengthPage(dataType, pageSize);
+ return createFixLengthPage(dataType, pageSize, scale, precision);
}
}
- public static ColumnPage newVarLengthPath(DataType dataType, int pageSize) {
+ public static ColumnPage newVarLengthPage(DataType dataType, int pageSize) {
+ return newVarLengthPage(dataType, pageSize, -1, -1);
+ }
+
+ private static ColumnPage newVarLengthPage(DataType dataType, int pageSize, int scale,
+ int precision) {
if (unsafe) {
try {
- return new UnsafeVarLengthColumnPage(dataType, pageSize);
+ return new UnsafeVarLengthColumnPage(dataType, pageSize, scale, precision);
} catch (MemoryException e) {
throw new RuntimeException(e);
}
} else {
- return new SafeVarLengthColumnPage(dataType, pageSize);
+ return new SafeVarLengthColumnPage(dataType, pageSize, scale, precision);
}
}
/**
* Create a new page of dataType and number of row = pageSize
*/
- public static ColumnPage newPage(DataType dataType, int pageSize) throws MemoryException {
+ public static ColumnPage newPage(DataType dataType, int pageSize, int scale, int precision)
+ throws MemoryException {
ColumnPage instance;
if (unsafe) {
switch (dataType) {
@@ -131,11 +148,11 @@ public abstract class ColumnPage {
case LONG:
case FLOAT:
case DOUBLE:
- instance = new UnsafeFixLengthColumnPage(dataType, pageSize);
+ instance = new UnsafeFixLengthColumnPage(dataType, pageSize, scale, precision);
break;
case DECIMAL:
case BYTE_ARRAY:
- instance = new UnsafeVarLengthColumnPage(dataType, pageSize);
+ instance = new UnsafeVarLengthColumnPage(dataType, pageSize, scale, precision);
break;
default:
throw new RuntimeException("Unsupported data dataType: " + dataType);
@@ -164,7 +181,7 @@ public abstract class ColumnPage {
instance = newDoublePage(new double[pageSize]);
break;
case DECIMAL:
- instance = newDecimalPage(new byte[pageSize][]);
+ instance = newDecimalPage(new byte[pageSize][], scale, precision);
break;
default:
throw new RuntimeException("Unsupported data dataType: " + dataType);
@@ -174,55 +191,61 @@ public abstract class ColumnPage {
}
private static ColumnPage newBytePage(byte[] byteData) {
- ColumnPage columnPage = createPage(BYTE, byteData.length);
+ ColumnPage columnPage = createPage(BYTE, byteData.length, -1, -1);
columnPage.setBytePage(byteData);
return columnPage;
}
private static ColumnPage newShortPage(short[] shortData) {
- ColumnPage columnPage = createPage(SHORT, shortData.length);
+ ColumnPage columnPage = createPage(SHORT, shortData.length, -1, -1);
columnPage.setShortPage(shortData);
return columnPage;
}
private static ColumnPage newShortIntPage(byte[] shortIntData) {
- ColumnPage columnPage = createPage(SHORT_INT, shortIntData.length / 3);
+ ColumnPage columnPage = createPage(SHORT_INT, shortIntData.length / 3, -1, -1);
columnPage.setShortIntPage(shortIntData);
return columnPage;
}
private static ColumnPage newIntPage(int[] intData) {
- ColumnPage columnPage = createPage(INT, intData.length);
+ ColumnPage columnPage = createPage(INT, intData.length, -1, -1);
columnPage.setIntPage(intData);
return columnPage;
}
private static ColumnPage newLongPage(long[] longData) {
- ColumnPage columnPage = createPage(LONG, longData.length);
+ ColumnPage columnPage = createPage(LONG, longData.length, -1, -1);
columnPage.setLongPage(longData);
return columnPage;
}
private static ColumnPage newFloatPage(float[] floatData) {
- ColumnPage columnPage = createPage(FLOAT, floatData.length);
+ ColumnPage columnPage = createPage(FLOAT, floatData.length, -1, -1);
columnPage.setFloatPage(floatData);
return columnPage;
}
private static ColumnPage newDoublePage(double[] doubleData) {
- ColumnPage columnPage = createPage(DOUBLE, doubleData.length);
+ ColumnPage columnPage = createPage(DOUBLE, doubleData.length, -1, -1);
columnPage.setDoublePage(doubleData);
return columnPage;
}
- private static ColumnPage newDecimalPage(byte[][] byteArray) {
- ColumnPage columnPage = createPage(DECIMAL, byteArray.length);
+ private static ColumnPage newDecimalPage(byte[][] byteArray, int scale, int precision) {
+ ColumnPage columnPage = createPage(DECIMAL, byteArray.length, scale, precision);
columnPage.setByteArrayPage(byteArray);
return columnPage;
}
- private static ColumnPage newDecimalPage(byte[] lvEncodedByteArray) throws MemoryException {
- return VarLengthColumnPageBase.newDecimalColumnPage(lvEncodedByteArray);
+ private static ColumnPage newDecimalPage(byte[] lvEncodedByteArray, int scale, int precision)
+ throws MemoryException {
+ return VarLengthColumnPageBase.newDecimalColumnPage(lvEncodedByteArray, scale, precision);
+ }
+
+ private static ColumnPage newVarLengthPage(byte[] lvEncodedByteArray, int scale, int precision)
+ throws MemoryException {
+ return VarLengthColumnPageBase.newVarLengthColumnPage(lvEncodedByteArray, scale, precision);
}
/**
@@ -297,6 +320,8 @@ public abstract class ColumnPage {
putDouble(rowId, (double) value);
break;
case DECIMAL:
+ putDecimal(rowId, (BigDecimal) value);
+ break;
case BYTE_ARRAY:
putBytes(rowId, (byte[]) value);
break;
@@ -337,6 +362,11 @@ public abstract class ColumnPage {
public abstract void putBytes(int rowId, byte[] bytes);
/**
+ * Set byte array value at rowId
+ */
+ public abstract void putDecimal(int rowId, BigDecimal decimal);
+
+ /**
* Type cast int value to 3 bytes value and set at rowId
*/
public abstract void putShortInt(int rowId, int value);
@@ -346,7 +376,6 @@ public abstract class ColumnPage {
*/
public abstract void putBytes(int rowId, byte[] bytes, int offset, int length);
- private static final byte[] ZERO = DataTypeUtil.bigDecimalToByte(BigDecimal.ZERO);
/**
* Set null at rowId
@@ -370,7 +399,7 @@ public abstract class ColumnPage {
putDouble(rowId, 0.0);
break;
case DECIMAL:
- putBytes(rowId, ZERO);
+ putDecimal(rowId, BigDecimal.ZERO);
break;
}
}
@@ -468,6 +497,11 @@ public abstract class ColumnPage {
public abstract byte[] getFlattenedBytePage();
/**
+ * For decimals
+ */
+ public abstract byte[] getDecimalPage();
+
+ /**
* Encode the page data by codec (Visitor)
*/
public abstract void encode(PrimitiveCodec codec);
@@ -492,7 +526,7 @@ public abstract class ColumnPage {
case DOUBLE:
return compressor.compressDouble(getDoublePage());
case DECIMAL:
- return compressor.compressByte(getFlattenedBytePage());
+ return compressor.compressByte(getDecimalPage());
case BYTE_ARRAY:
return compressor.compressByte(getFlattenedBytePage());
default:
@@ -504,7 +538,8 @@ public abstract class ColumnPage {
* Decompress data and create a column page using the decompressed data
*/
public static ColumnPage decompress(Compressor compressor, DataType dataType,
- byte[] compressedData, int offset, int length) throws MemoryException {
+ byte[] compressedData, int offset, int length, int scale, int precision)
+ throws MemoryException {
switch (dataType) {
case BYTE:
byte[] byteData = compressor.unCompressByte(compressedData, offset, length);
@@ -528,9 +563,11 @@ public abstract class ColumnPage {
double[] doubleData = compressor.unCompressDouble(compressedData, offset, length);
return newDoublePage(doubleData);
case DECIMAL:
- case BYTE_ARRAY:
byte[] lvEncodedBytes = compressor.unCompressByte(compressedData, offset, length);
- return newDecimalPage(lvEncodedBytes);
+ return newDecimalPage(lvEncodedBytes, scale, precision);
+ case BYTE_ARRAY:
+ byte[] lvVarBytes = compressor.unCompressByte(compressedData, offset, length);
+ return newVarLengthPage(lvVarBytes, scale, precision);
default:
throw new UnsupportedOperationException("unsupport uncompress column page: " + dataType);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/core/src/main/java/org/apache/carbondata/core/datastore/page/LazyColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/LazyColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/LazyColumnPage.java
index 6ec2e07..b0978d3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/LazyColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/LazyColumnPage.java
@@ -32,7 +32,8 @@ public class LazyColumnPage extends ColumnPage {
private PrimitiveCodec codec;
private LazyColumnPage(ColumnPage columnPage, PrimitiveCodec codec) {
- super(columnPage.getDataType(), columnPage.getPageSize());
+ super(columnPage.getDataType(), columnPage.getPageSize(), columnPage.scale,
+ columnPage.precision);
this.columnPage = columnPage;
this.codec = codec;
}
@@ -137,6 +138,16 @@ public class LazyColumnPage extends ColumnPage {
}
@Override
+ public void putDecimal(int rowId, BigDecimal decimal) {
+ throw new UnsupportedOperationException("internal error");
+ }
+
+ @Override
+ public byte[] getDecimalPage() {
+ throw new UnsupportedOperationException("internal error");
+ }
+
+ @Override
public byte[] getFlattenedBytePage() {
throw new UnsupportedOperationException("internal error");
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java
index cfb1798..9bd85e6 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeFixLengthColumnPage.java
@@ -36,8 +36,8 @@ public class SafeFixLengthColumnPage extends ColumnPage {
private double[] doubleData;
private byte[] shortIntData;
- SafeFixLengthColumnPage(DataType dataType, int pageSize) {
- super(dataType, pageSize);
+ SafeFixLengthColumnPage(DataType dataType, int pageSize, int scale, int precision) {
+ super(dataType, pageSize, scale, precision);
}
/**
@@ -99,6 +99,16 @@ public class SafeFixLengthColumnPage extends ColumnPage {
throw new UnsupportedOperationException("invalid data type: " + dataType);
}
+ @Override
+ public void putDecimal(int rowId, BigDecimal decimal) {
+ throw new UnsupportedOperationException("invalid data type: " + dataType);
+ }
+
+ @Override
+ public byte[] getDecimalPage() {
+ throw new UnsupportedOperationException("invalid data type: " + dataType);
+ }
+
/**
* Get byte value at rowId
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java
index 3a76f55..63291f3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/SafeVarLengthColumnPage.java
@@ -20,15 +20,14 @@ package org.apache.carbondata.core.datastore.page;
import java.math.BigDecimal;
import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.util.DataTypeUtil;
public class SafeVarLengthColumnPage extends VarLengthColumnPageBase {
// for string and decimal data
private byte[][] byteArrayData;
- SafeVarLengthColumnPage(DataType dataType, int pageSize) {
- super(dataType, pageSize);
+ SafeVarLengthColumnPage(DataType dataType, int pageSize, int scale, int precision) {
+ super(dataType, pageSize, scale, precision);
byteArrayData = new byte[pageSize][];
}
@@ -47,10 +46,14 @@ public class SafeVarLengthColumnPage extends VarLengthColumnPageBase {
System.arraycopy(bytes, offset, byteArrayData[rowId], 0, length);
}
+ @Override public void putDecimal(int rowId, BigDecimal decimal) {
+ putBytes(rowId, decimalConverter.convert(decimal));
+ }
+
@Override
public BigDecimal getDecimal(int rowId) {
byte[] bytes = byteArrayData[rowId];
- return DataTypeUtil.byteToBigDecimal(bytes);
+ return decimalConverter.getDecimal(bytes);
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
index 9f71768..2382599 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
@@ -49,8 +49,9 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
private static final int floatBits = DataType.FLOAT.getSizeBits();
private static final int doubleBits = DataType.DOUBLE.getSizeBits();
- UnsafeFixLengthColumnPage(DataType dataType, int pageSize) throws MemoryException {
- super(dataType, pageSize);
+ UnsafeFixLengthColumnPage(DataType dataType, int pageSize, int scale, int precision)
+ throws MemoryException {
+ super(dataType, pageSize, scale, precision);
switch (dataType) {
case BYTE:
case SHORT:
@@ -124,6 +125,10 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
throw new UnsupportedOperationException("invalid data type: " + dataType);
}
+ @Override public void putDecimal(int rowId, BigDecimal decimal) {
+ throw new UnsupportedOperationException("invalid data type: " + dataType);
+ }
+
@Override
public byte getByte(int rowId) {
long offset = rowId << byteBits;
@@ -175,6 +180,10 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
throw new UnsupportedOperationException("invalid data type: " + dataType);
}
+ @Override public byte[] getDecimalPage() {
+ throw new UnsupportedOperationException("invalid data type: " + dataType);
+ }
+
@Override
public byte[] getBytePage() {
byte[] data = new byte[getPageSize()];
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
index dd6abc5..16cf94c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
@@ -24,7 +24,6 @@ import org.apache.carbondata.core.memory.MemoryBlock;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.memory.UnsafeMemoryManager;
import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.util.DataTypeUtil;
// This extension uses unsafe memory to store page data, for variable length data type (string,
// decimal)
@@ -52,10 +51,11 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase {
* @param dataType data type
* @param pageSize number of row
*/
- UnsafeVarLengthColumnPage(DataType dataType, int pageSize) throws MemoryException {
- super(dataType, pageSize);
+ UnsafeVarLengthColumnPage(DataType dataType, int pageSize, int scale, int precision)
+ throws MemoryException {
+ super(dataType, pageSize, scale, precision);
capacity = (int) (pageSize * DEFAULT_ROW_SIZE * FACTOR);
- memoryBlock = UnsafeMemoryManager.allocateMemoryWithRetry((long)(capacity));
+ memoryBlock = UnsafeMemoryManager.allocateMemoryWithRetry((long) (capacity));
baseAddress = memoryBlock.getBaseObject();
baseOffset = memoryBlock.getBaseOffset();
}
@@ -66,8 +66,9 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase {
* @param pageSize number of row
* @param capacity initial capacity of the page, in bytes
*/
- UnsafeVarLengthColumnPage(DataType dataType, int pageSize, int capacity) throws MemoryException {
- super(dataType, pageSize);
+ UnsafeVarLengthColumnPage(DataType dataType, int pageSize, int capacity,
+ int scale, int precision) throws MemoryException {
+ super(dataType, pageSize, scale, precision);
this.capacity = capacity;
memoryBlock = UnsafeMemoryManager.allocateMemoryWithRetry((long)(capacity));
baseAddress = memoryBlock.getBaseObject();
@@ -117,6 +118,10 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase {
baseAddress, baseOffset + rowOffset[rowId], length);
}
+ @Override public void putDecimal(int rowId, BigDecimal decimal) {
+ putBytes(rowId, decimalConverter.convert(decimal));
+ }
+
@Override
public BigDecimal getDecimal(int rowId) {
int length = rowOffset[rowId + 1] - rowOffset[rowId];
@@ -124,7 +129,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase {
CarbonUnsafe.unsafe.copyMemory(baseAddress, baseOffset + rowOffset[rowId],
bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
- return DataTypeUtil.byteToBigDecimal(bytes);
+ return decimalConverter.getDecimal(bytes);
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
index 801cfb3..46d6787 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPageBase.java
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DecimalConverterFactory;
import org.apache.carbondata.core.util.ByteUtil;
import static org.apache.carbondata.core.metadata.datatype.DataType.DECIMAL;
@@ -34,8 +35,8 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
// the length of bytes added in the page
int totalLength;
- VarLengthColumnPageBase(DataType dataType, int pageSize) {
- super(dataType, pageSize);
+ VarLengthColumnPageBase(DataType dataType, int pageSize, int scale, int precision) {
+ super(dataType, pageSize, scale, precision);
rowOffset = new int[pageSize + 1];
totalLength = 0;
}
@@ -83,7 +84,59 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
/**
* Create a new column page based on the LV (Length Value) encoded bytes
*/
- static ColumnPage newDecimalColumnPage(byte[] lvEncodedBytes) throws MemoryException {
+ static ColumnPage newDecimalColumnPage(byte[] lvEncodedBytes, int scale, int precision)
+ throws MemoryException {
+ DecimalConverterFactory.DecimalConverter decimalConverter =
+ DecimalConverterFactory.INSTANCE.getDecimalConverter(precision, scale);
+ int size = decimalConverter.getSize();
+ if (size < 0) {
+ return getLVBytesColumnPage(lvEncodedBytes, scale, precision, DataType.DECIMAL);
+ } else {
+ // Here the size is always fixed.
+ return getDecimalColumnPage(lvEncodedBytes, scale, precision, size);
+ }
+ }
+
+ /**
+ * Create a new column page based on the LV (Length Value) encoded bytes
+ */
+ static ColumnPage newVarLengthColumnPage(byte[] lvEncodedBytes, int scale, int precision)
+ throws MemoryException {
+ return getLVBytesColumnPage(lvEncodedBytes, scale, precision, DataType.BYTE_ARRAY);
+ }
+
+ private static ColumnPage getDecimalColumnPage(byte[] lvEncodedBytes, int scale, int precision,
+ int size) throws MemoryException {
+ List<Integer> rowOffset = new ArrayList<>();
+ int offset;
+ int rowId = 0;
+ for (offset = 0; offset < lvEncodedBytes.length; offset += size) {
+ rowOffset.add(offset);
+ rowId++;
+ }
+ rowOffset.add(offset);
+
+ VarLengthColumnPageBase page;
+ if (unsafe) {
+ page = new UnsafeVarLengthColumnPage(DECIMAL, rowId, scale, precision);
+ } else {
+ page = new SafeVarLengthColumnPage(DECIMAL, rowId, scale, precision);
+ }
+
+ // set total length and rowOffset in page
+ page.totalLength = offset;
+ page.rowOffset = new int[rowId + 1];
+ for (int i = 0; i < rowId + 1; i++) {
+ page.rowOffset[i] = rowOffset.get(i);
+ }
+ for (int i = 0; i < rowId; i++) {
+ page.putBytes(i, lvEncodedBytes, i * size, size);
+ }
+ return page;
+ }
+
+ private static ColumnPage getLVBytesColumnPage(byte[] lvEncodedBytes, int scale,
+ int precision, DataType dataType) throws MemoryException {
// extract length and data, set them to rowOffset and unsafe memory correspondingly
int rowId = 0;
List<Integer> rowOffset = new ArrayList<>();
@@ -107,9 +160,9 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
VarLengthColumnPageBase page;
int inputDataLength = offset;
if (unsafe) {
- page = new UnsafeVarLengthColumnPage(DECIMAL, numRows, inputDataLength);
+ page = new UnsafeVarLengthColumnPage(DECIMAL, numRows, inputDataLength, scale, precision);
} else {
- page = new SafeVarLengthColumnPage(DECIMAL, numRows);
+ page = new SafeVarLengthColumnPage(dataType, numRows, scale, precision);
}
// set total length and rowOffset in page
@@ -242,6 +295,19 @@ public abstract class VarLengthColumnPageBase extends ColumnPage {
throw new UnsupportedOperationException("invalid data type: " + dataType);
}
+ @Override
+ public byte[] getDecimalPage() {
+ // output LV encoded byte array
+ int offset = 0;
+ byte[] data = new byte[totalLength];
+ for (int rowId = 0; rowId < pageSize; rowId++) {
+ int length = rowOffset[rowId + 1] - rowOffset[rowId];
+ copyBytes(rowId, data, offset, length);
+ offset += length;
+ }
+ return data;
+ }
+
/**
* Copy `length` bytes from data at rowId to dest start from destOffset
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/AdaptiveIntegerCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/AdaptiveIntegerCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/AdaptiveIntegerCodec.java
index 3d56f0c..fe15ba7 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/AdaptiveIntegerCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/AdaptiveIntegerCodec.java
@@ -52,7 +52,8 @@ class AdaptiveIntegerCodec extends AdaptiveCompressionCodec {
@Override
public byte[] encode(ColumnPage input) throws MemoryException, IOException {
- encodedPage = ColumnPage.newPage(targetDataType, input.getPageSize());
+ encodedPage = ColumnPage
+ .newPage(targetDataType, input.getPageSize(), stats.getScale(), stats.getPrecision());
input.encode(codec);
byte[] result = encodedPage.compress(compressor);
encodedPage.freeMemory();
@@ -62,9 +63,13 @@ class AdaptiveIntegerCodec extends AdaptiveCompressionCodec {
@Override
public ColumnPage decode(byte[] input, int offset, int length) throws MemoryException {
if (srcDataType.equals(targetDataType)) {
- return ColumnPage.decompress(compressor, targetDataType, input, offset, length);
+ return ColumnPage
+ .decompress(compressor, targetDataType, input, offset, length, stats.getScale(),
+ stats.getPrecision());
} else {
- ColumnPage page = ColumnPage.decompress(compressor, targetDataType, input, offset, length);
+ ColumnPage page = ColumnPage
+ .decompress(compressor, targetDataType, input, offset, length, stats.getScale(),
+ stats.getPrecision());
return LazyColumnPage.newPage(page, codec);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingStrategy.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingStrategy.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingStrategy.java
index 3818263..659dc2a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingStrategy.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingStrategy.java
@@ -79,7 +79,7 @@ public class DefaultEncodingStrategy extends EncodingStrategy {
if (Math.min(adaptiveDataType.getSizeInBytes(), deltaDataType.getSizeInBytes()) ==
srcDataType.getSizeInBytes()) {
// no effect to use adaptive or delta, use compression only
- return DirectCompressCodec.newInstance(srcDataType, compressor);
+ return DirectCompressCodec.newInstance(stats, compressor);
}
if (adaptiveDataType.getSizeInBytes() <= deltaDataType.getSizeInBytes()) {
// choose adaptive encoding
@@ -93,17 +93,17 @@ public class DefaultEncodingStrategy extends EncodingStrategy {
@Override
ColumnPageCodec newCodecForFloatingType(ColumnPageStatsVO stats) {
- return DirectCompressCodec.newInstance(stats.getDataType(), compressor);
+ return DirectCompressCodec.newInstance(stats, compressor);
}
// for decimal, currently it is a very basic implementation
@Override
ColumnPageCodec newCodecForDecimalType(ColumnPageStatsVO stats) {
- return DirectCompressCodec.newInstance(stats.getDataType(), compressor);
+ return DirectCompressCodec.newInstance(stats, compressor);
}
@Override
ColumnPageCodec newCodecForByteArrayType(ColumnPageStatsVO stats) {
- return DirectCompressCodec.newInstance(stats.getDataType(), compressor);
+ return DirectCompressCodec.newInstance(stats, compressor);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegerCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegerCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegerCodec.java
index 6cf59a6..a45552a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegerCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegerCodec.java
@@ -67,7 +67,8 @@ public class DeltaIntegerCodec extends AdaptiveCompressionCodec {
@Override
public byte[] encode(ColumnPage input) throws MemoryException, IOException {
- encodedPage = ColumnPage.newPage(targetDataType, input.getPageSize());
+ encodedPage = ColumnPage
+ .newPage(targetDataType, input.getPageSize(), stats.getScale(), stats.getPrecision());
input.encode(codec);
byte[] result = encodedPage.compress(compressor);
encodedPage.freeMemory();
@@ -77,9 +78,13 @@ public class DeltaIntegerCodec extends AdaptiveCompressionCodec {
@Override
public ColumnPage decode(byte[] input, int offset, int length) throws MemoryException {
if (srcDataType.equals(targetDataType)) {
- return ColumnPage.decompress(compressor, targetDataType, input, offset, length);
+ return ColumnPage
+ .decompress(compressor, targetDataType, input, offset, length, stats.getScale(),
+ stats.getPrecision());
} else {
- ColumnPage page = ColumnPage.decompress(compressor, targetDataType, input, offset, length);
+ ColumnPage page = ColumnPage
+ .decompress(compressor, targetDataType, input, offset, length, stats.getScale(),
+ stats.getPrecision());
return LazyColumnPage.newPage(page, codec);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DirectCompressCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DirectCompressCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DirectCompressCodec.java
index dcb9b7c..d608fea 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DirectCompressCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DirectCompressCodec.java
@@ -21,8 +21,8 @@ import java.io.IOException;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.datastore.page.statistics.ColumnPageStatsVO;
import org.apache.carbondata.core.memory.MemoryException;
-import org.apache.carbondata.core.metadata.datatype.DataType;
/**
* This codec directly apply compression on the input data
@@ -30,15 +30,15 @@ import org.apache.carbondata.core.metadata.datatype.DataType;
public class DirectCompressCodec implements ColumnPageCodec {
private Compressor compressor;
- private DataType dataType;
+ private ColumnPageStatsVO stats;
- private DirectCompressCodec(DataType dataType, Compressor compressor) {
+ private DirectCompressCodec(ColumnPageStatsVO stats, Compressor compressor) {
this.compressor = compressor;
- this.dataType = dataType;
+ this.stats = stats;
}
- public static DirectCompressCodec newInstance(DataType dataType, Compressor compressor) {
- return new DirectCompressCodec(dataType, compressor);
+ public static DirectCompressCodec newInstance(ColumnPageStatsVO stats, Compressor compressor) {
+ return new DirectCompressCodec(stats, compressor);
}
@Override
@@ -53,6 +53,8 @@ public class DirectCompressCodec implements ColumnPageCodec {
@Override
public ColumnPage decode(byte[] input, int offset, int length) throws MemoryException {
- return ColumnPage.decompress(compressor, dataType, input, offset, length);
+ return ColumnPage
+ .decompress(compressor, stats.getDataType(), input, offset, length, stats.getScale(),
+ stats.getPrecision());
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingStrategy.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingStrategy.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingStrategy.java
index 49fb625..77d3b74 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingStrategy.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingStrategy.java
@@ -51,8 +51,8 @@ public abstract class EncodingStrategy {
/**
* create codec based on the page data type and statistics contained by ValueEncoderMeta
*/
- public ColumnPageCodec createCodec(ValueEncoderMeta meta) {
- ColumnPageStatsVO stats = ColumnPageStatsVO.copyFrom(meta);
+ public ColumnPageCodec createCodec(ValueEncoderMeta meta, int scale, int precision) {
+ ColumnPageStatsVO stats = ColumnPageStatsVO.copyFrom(meta, scale, precision);
return createCodec(stats);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/ColumnPageStatsVO.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/ColumnPageStatsVO.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/ColumnPageStatsVO.java
index 058699a..3629101 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/ColumnPageStatsVO.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/ColumnPageStatsVO.java
@@ -40,6 +40,10 @@ public class ColumnPageStatsVO {
/** decimal count of the measures */
private int decimal;
+ private int scale;
+
+ private int precision;
+
public ColumnPageStatsVO(DataType dataType) {
this.dataType = dataType;
switch (dataType) {
@@ -64,12 +68,14 @@ public class ColumnPageStatsVO {
decimal = 0;
}
- public static ColumnPageStatsVO copyFrom(ValueEncoderMeta meta) {
+ public static ColumnPageStatsVO copyFrom(ValueEncoderMeta meta, int scale, int precision) {
ColumnPageStatsVO instance = new ColumnPageStatsVO(meta.getType());
instance.min = meta.getMinValue();
instance.max = meta.getMaxValue();
instance.decimal = meta.getDecimal();
instance.nonExistValue = meta.getUniqueValue();
+ instance.scale = scale;
+ instance.precision = precision;
return instance;
}
@@ -101,7 +107,7 @@ public class ColumnPageStatsVO {
nonExistValue = (double) min - 1;
break;
case DECIMAL:
- BigDecimal decimalValue = DataTypeUtil.byteToBigDecimal((byte[]) value);
+ BigDecimal decimalValue = (BigDecimal) value;
decimal = decimalValue.scale();
BigDecimal val = (BigDecimal) min;
nonExistValue = (val.subtract(new BigDecimal(1.0)));
@@ -215,6 +221,14 @@ public class ColumnPageStatsVO {
return dataType;
}
+ public int getScale() {
+ return scale;
+ }
+
+ public int getPrecision() {
+ return precision;
+ }
+
@Override
public String toString() {
return String.format("min: %s, max: %s, decimal: %s ", min, max, decimal);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
new file mode 100644
index 0000000..555df1c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.metadata.datatype;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
+import org.apache.carbondata.core.util.DataTypeUtil;
+
+/**
+ * Decimal converter to keep the data compact.
+ */
+public final class DecimalConverterFactory {
+
+ public static DecimalConverterFactory INSTANCE = new DecimalConverterFactory();
+
+ private int[] minBytesForPrecision = minBytesForPrecision();
+
+ private DecimalConverterFactory() {
+
+ }
+
+ private int computeMinBytesForPrecision(int precision) {
+ int numBytes = 1;
+ while (Math.pow(2.0, 8 * numBytes - 1) < Math.pow(10.0, precision)) {
+ numBytes += 1;
+ }
+ return numBytes;
+ }
+
+ private int[] minBytesForPrecision() {
+ int[] data = new int[39];
+ for (int i = 0; i < data.length; i++) {
+ data[i] = computeMinBytesForPrecision(i);
+ }
+ return data;
+ }
+
+ public interface DecimalConverter {
+
+ byte[] convert(BigDecimal decimal);
+
+ BigDecimal getDecimal(byte[] bytes);
+
+ void writeToColumnVector(byte[] bytes, CarbonColumnVector vector, int rowId);
+
+ int getSize();
+
+ }
+
+ public class DecimalIntConverter implements DecimalConverter {
+
+ private ByteBuffer buffer = ByteBuffer.allocate(4);
+
+ private int precision;
+ private int scale;
+
+ public DecimalIntConverter(int precision, int scale) {
+ this.precision = precision;
+ this.scale = scale;
+ }
+
+ @Override public byte[] convert(BigDecimal decimal) {
+ long longValue = decimal.unscaledValue().longValue();
+ buffer.putInt(0, (int) longValue);
+ return buffer.array().clone();
+ }
+
+ @Override public BigDecimal getDecimal(byte[] bytes) {
+ long unscaled = getUnscaledLong(bytes);
+ return BigDecimal.valueOf(unscaled, scale);
+ }
+
+ @Override public void writeToColumnVector(byte[] bytes, CarbonColumnVector vector, int rowId) {
+ long unscaled = getUnscaledLong(bytes);
+ vector.putInt(rowId, (int) unscaled);
+ }
+
+ @Override public int getSize() {
+ return 4;
+ }
+ }
+
+ private long getUnscaledLong(byte[] bytes) {
+ long unscaled = 0L;
+ int i = 0;
+
+ while (i < bytes.length) {
+ unscaled = (unscaled << 8) | (bytes[i] & 0xff);
+ i += 1;
+ }
+
+ int bits = 8 * bytes.length;
+ unscaled = (unscaled << (64 - bits)) >> (64 - bits);
+ return unscaled;
+ }
+
+ public class DecimalLongConverter implements DecimalConverter {
+
+ private ByteBuffer buffer = ByteBuffer.allocate(8);
+
+ private int precision;
+ private int scale;
+
+ public DecimalLongConverter(int precision, int scale) {
+ this.precision = precision;
+ this.scale = scale;
+ }
+
+ @Override public byte[] convert(BigDecimal decimal) {
+ long longValue = decimal.unscaledValue().longValue();
+ buffer.putLong(0, longValue);
+ return buffer.array().clone();
+ }
+
+ @Override public BigDecimal getDecimal(byte[] bytes) {
+ long unscaled = getUnscaledLong(bytes);
+ return BigDecimal.valueOf(unscaled, scale);
+ }
+
+ @Override public void writeToColumnVector(byte[] bytes, CarbonColumnVector vector, int rowId) {
+ long unscaled = getUnscaledLong(bytes);
+ vector.putLong(rowId, unscaled);
+ }
+
+ @Override public int getSize() {
+ return 8;
+ }
+ }
+
+ public class DecimalUnscaledConverter implements DecimalConverter {
+
+ private int precision;
+
+ private int scale;
+
+ private int numBytes;
+
+ private byte[] decimalBuffer = new byte[minBytesForPrecision[38]];
+
+ public DecimalUnscaledConverter(int precision, int scale) {
+ this.precision = precision;
+ this.scale = scale;
+ this.numBytes = minBytesForPrecision[precision];
+ }
+
+ @Override public byte[] convert(BigDecimal decimal) {
+ byte[] bytes = decimal.unscaledValue().toByteArray();
+ byte[] fixedLengthBytes = null;
+ if (bytes.length == numBytes) {
+ // If the length of the underlying byte array of the unscaled `BigInteger` happens to be
+ // `numBytes`, just reuse it, so that we don't bother copying it to `decimalBuffer`.
+ fixedLengthBytes = bytes;
+ } else {
+ // Otherwise, the length must be less than `numBytes`. In this case we copy contents of
+ // the underlying bytes with padding sign bytes to `decimalBuffer` to form the result
+ // fixed-length byte array.
+ byte signByte = 0;
+ if (bytes[0] < 0) {
+ signByte = (byte) -1;
+ } else {
+ signByte = (byte) 0;
+ }
+ Arrays.fill(decimalBuffer, 0, numBytes - bytes.length, signByte);
+ System.arraycopy(bytes, 0, decimalBuffer, numBytes - bytes.length, bytes.length);
+ fixedLengthBytes = decimalBuffer;
+ }
+ byte[] value = new byte[numBytes];
+ System.arraycopy(fixedLengthBytes, 0, value, 0, numBytes);
+ return value;
+ }
+
+ @Override public BigDecimal getDecimal(byte[] bytes) {
+ BigInteger bigInteger = new BigInteger(bytes);
+ return new BigDecimal(bigInteger, scale);
+ }
+
+ @Override public void writeToColumnVector(byte[] bytes, CarbonColumnVector vector, int rowId) {
+ vector.putBytes(rowId, bytes);
+ }
+
+ @Override public int getSize() {
+ return numBytes;
+ }
+ }
+
+ public static class LVBytesDecimalConverter implements DecimalConverter {
+
+ public static LVBytesDecimalConverter INSTANCE = new LVBytesDecimalConverter();
+
+ @Override public byte[] convert(BigDecimal decimal) {
+ return DataTypeUtil.bigDecimalToByte(decimal);
+ }
+
+ @Override public BigDecimal getDecimal(byte[] bytes) {
+ return DataTypeUtil.byteToBigDecimal(bytes);
+ }
+
+ @Override public void writeToColumnVector(byte[] bytes, CarbonColumnVector vector, int rowId) {
+ throw new UnsupportedOperationException("Unsupported in vector reading for legacy format");
+ }
+
+ @Override public int getSize() {
+ return -1;
+ }
+ }
+
+ public DecimalConverter getDecimalConverter(int precision, int scale) {
+ if (precision < 0) {
+ return new LVBytesDecimalConverter();
+ } else if (precision <= 9) {
+ return new DecimalIntConverter(precision, scale);
+ } else if (precision <= 18) {
+ return new DecimalLongConverter(precision, scale);
+ } else {
+ return new DecimalUnscaledConverter(precision, scale);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index e89ce12..0a27a74 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -24,6 +24,7 @@ import java.io.ObjectOutputStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
@@ -431,6 +432,29 @@ public class CarbonMetadataUtil {
return false;
}
+ private static ByteBuffer writeInfoIfDecimal(int blockIndex,
+ SegmentProperties segmentProperties) {
+ Map<Integer, Integer> blockMapping = segmentProperties.getMeasuresOrdinalToBlockMapping();
+ List<CarbonMeasure> measures = segmentProperties.getMeasures();
+ CarbonMeasure selectedMeasure = null;
+ for (CarbonMeasure measure : measures) {
+ Integer blockId = blockMapping.get(measure.getOrdinal());
+ selectedMeasure = measure;
+ if (blockId == blockIndex) {
+ break;
+ }
+ }
+ assert (selectedMeasure != null);
+ if (selectedMeasure.getDataType() == DataType.DECIMAL) {
+ ByteBuffer buffer = ByteBuffer.allocate(8);
+ buffer.putInt(selectedMeasure.getScale());
+ buffer.putInt(selectedMeasure.getPrecision());
+ buffer.flip();
+ return buffer;
+ }
+ return null;
+ }
+
private static byte[] serializeEncoderMeta(ValueEncoderMeta encoderMeta) throws IOException {
// TODO : should remove the unnecessary fields.
ByteArrayOutputStream aos = new ByteArrayOutputStream();
@@ -788,6 +812,10 @@ public class CarbonMetadataUtil {
List<ByteBuffer> encoderMetaList = new ArrayList<ByteBuffer>();
encoderMetaList.add(ByteBuffer.wrap(serializeEncodeMetaUsingByteBuffer(
createValueEncoderMeta(nodeHolder.getStats(), index))));
+ ByteBuffer decimalMeta = writeInfoIfDecimal(index, segmentProperties);
+ if (decimalMeta != null) {
+ encoderMetaList.add(decimalMeta);
+ }
dataChunk.setEncoder_meta(encoderMetaList);
dataChunk.min_max
.addToMax_values(ByteBuffer.wrap(nodeHolder.getMeasureColumnMaxData()[index]));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
index 5b0685b..50fb4c5 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/SortStepRowUtil.java
@@ -17,10 +17,6 @@
package org.apache.carbondata.processing.newflow.sort;
-import java.math.BigDecimal;
-
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.sortandgroupby.sortdata.SortParameters;
import org.apache.carbondata.processing.util.NonDictionaryUtil;
@@ -60,21 +56,10 @@ public class SortStepRowUtil {
index = 0;
- DataType[] measureDataType = parameters.getMeasureDataType();
// read measure values
for (int i = 0; i < measureCount; i++) {
if (needConvertDecimalToByte) {
- Object value = data[allCount];
- if (null != value) {
- if (measureDataType[i] == DataType.DECIMAL) {
- BigDecimal decimal = (BigDecimal) value;
- measures[index++] = DataTypeUtil.bigDecimalToByte(decimal);
- } else {
- measures[index++] = value;
- }
- } else {
- measures[index++] = null;
- }
+ measures[index++] = data[allCount];
} else {
measures[index++] = data[allCount];
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
index 2ac138b..55a8693 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
@@ -230,7 +230,7 @@ public class UnsafeCarbonRowPage {
CarbonUnsafe.unsafe.copyMemory(baseObject, address + size, bigDecimalInBytes,
CarbonUnsafe.BYTE_ARRAY_OFFSET, bigDecimalInBytes.length);
size += bigDecimalInBytes.length;
- rowToFill[dimensionSize + mesCount] = bigDecimalInBytes;
+ rowToFill[dimensionSize + mesCount] = DataTypeUtil.byteToBigDecimal(bigDecimalInBytes);
break;
}
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
index d69d137..7fb9b6e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
@@ -35,6 +35,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.newflow.sort.unsafe.UnsafeCarbonRowPage;
import org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.sortandgroupby.sortdata.NewRowComparator;
@@ -341,7 +342,7 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder {
short aShort = stream.readShort();
byte[] bigDecimalInBytes = new byte[aShort];
stream.readFully(bigDecimalInBytes);
- row[dimensionCount + mesCount] = bigDecimalInBytes;
+ row[dimensionCount + mesCount] = DataTypeUtil.byteToBigDecimal(bigDecimalInBytes);
break;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
index a83e09e..71e5727 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
@@ -18,7 +18,6 @@ package org.apache.carbondata.processing.newflow.steps;
import java.io.File;
import java.io.IOException;
-import java.math.BigDecimal;
import java.util.Iterator;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -33,7 +32,6 @@ import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
-import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.newflow.AbstractDataLoadProcessorStep;
import org.apache.carbondata.processing.newflow.CarbonDataLoadConfiguration;
import org.apache.carbondata.processing.newflow.DataField;
@@ -257,17 +255,7 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
Object[] measures = new Object[outputLength];
for (int i = 0; i < this.measureCount; i++) {
- Object value = row.getObject(i + this.dimensionWithComplexCount);
- if (null != value) {
- if (measureDataType[i] == DataType.DECIMAL) {
- BigDecimal val = (BigDecimal) value;
- measures[i] = DataTypeUtil.bigDecimalToByte(val);
- } else {
- measures[i] = value;
- }
- } else {
- measures[i] = null;
- }
+ measures[i] = row.getObject(i + this.dimensionWithComplexCount);
}
return WriteStepRowUtil.fromColumnCategory(dim, nonDicArray, measures);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
index 0f7ae1a..ce7b321 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
@@ -35,6 +35,7 @@ import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.util.NonDictionaryUtil;
@@ -356,7 +357,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
int len = stream.readInt();
byte[] buff = new byte[len];
stream.readFully(buff);
- measures[index++] = buff;
+ measures[index++] = DataTypeUtil.byteToBigDecimal(buff);
break;
}
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/12911629/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java b/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
index f068400..7930763 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/TablePage.java
@@ -20,12 +20,12 @@ package org.apache.carbondata.processing.store;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import org.apache.carbondata.core.datastore.GenericDataType;
+import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.datastore.exception.CarbonDataWriterException;
import org.apache.carbondata.core.datastore.page.ColumnPage;
import org.apache.carbondata.core.datastore.page.ComplexColumnPage;
@@ -35,7 +35,6 @@ import org.apache.carbondata.core.datastore.row.WriteStepRowUtil;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.spark.sql.types.Decimal;
@@ -69,11 +68,11 @@ public class TablePage {
int numDictDimension = model.getMDKeyGenerator().getDimCount();
dictDimensionPage = new ColumnPage[numDictDimension];
for (int i = 0; i < dictDimensionPage.length; i++) {
- dictDimensionPage[i] = ColumnPage.newVarLengthPath(DataType.BYTE_ARRAY, pageSize);
+ dictDimensionPage[i] = ColumnPage.newVarLengthPage(DataType.BYTE_ARRAY, pageSize);
}
noDictDimensionPage = new ColumnPage[model.getNoDictionaryCount()];
for (int i = 0; i < noDictDimensionPage.length; i++) {
- noDictDimensionPage[i] = ColumnPage.newVarLengthPath(DataType.BYTE_ARRAY, pageSize);
+ noDictDimensionPage[i] = ColumnPage.newVarLengthPage(DataType.BYTE_ARRAY, pageSize);
}
complexDimensionPage = new ComplexColumnPage[model.getComplexColumnCount()];
for (int i = 0; i < complexDimensionPage.length; i++) {
@@ -83,8 +82,10 @@ public class TablePage {
}
measurePage = new ColumnPage[model.getMeasureCount()];
DataType[] dataTypes = model.getMeasureDataType();
+ TableSpec.MeasureSpec measureSpec = model.getTableSpec().getMeasureSpec();
for (int i = 0; i < measurePage.length; i++) {
- measurePage[i] = ColumnPage.newPage(dataTypes[i], pageSize);
+ measurePage[i] = ColumnPage
+ .newPage(dataTypes[i], pageSize, measureSpec.getScale(i), measureSpec.getPrecision(i));
}
}
@@ -132,8 +133,7 @@ public class TablePage {
if (measurePage[i].getDataType() == DataType.DECIMAL &&
model.isCompactionFlow() &&
value != null) {
- BigDecimal bigDecimal = ((Decimal) value).toJavaBigDecimal();
- value = DataTypeUtil.bigDecimalToByte(bigDecimal);
+ value = ((Decimal) value).toJavaBigDecimal();
}
measurePage[i].putData(rowId, value);
}
[13/15] carbondata git commit: [CARBONDATA-1232] Datamap
implementation for Blocklet
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRow.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRow.java b/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRow.java
new file mode 100644
index 0000000..defe766
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRow.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore.row;
+
+import org.apache.carbondata.core.indexstore.schema.DataMapSchema;
+
+/**
+ * It is just a normal row to store data. Implementation classes could be safe and unsafe.
+ * TODO move this class a global row and use across loading after DataType is changed class
+ */
+public abstract class DataMapRow {
+
+ protected DataMapSchema[] schemas;
+
+ public DataMapRow(DataMapSchema[] schemas) {
+ this.schemas = schemas;
+ }
+
+ public abstract byte[] getByteArray(int ordinal);
+
+ public abstract DataMapRow getRow(int ordinal);
+
+ public abstract void setRow(DataMapRow row, int ordinal);
+
+ public abstract void setByteArray(byte[] byteArray, int ordinal);
+
+ public abstract int getInt(int ordinal);
+
+ public abstract void setInt(int value, int ordinal);
+
+ public abstract void setByte(byte value, int ordinal);
+
+ public abstract byte getByte(int ordinal);
+
+ public abstract void setShort(short value, int ordinal);
+
+ public abstract short getShort(int ordinal);
+
+ public abstract void setLong(long value, int ordinal);
+
+ public abstract long getLong(int ordinal);
+
+ public abstract void setFloat(float value, int ordinal);
+
+ public abstract float getFloat(int ordinal);
+
+ public abstract void setDouble(double value, int ordinal);
+
+ public abstract double getDouble(int ordinal);
+
+ public int getTotalSizeInBytes() {
+ int len = 0;
+ for (int i = 0; i < schemas.length; i++) {
+ len += getSizeInBytes(i);
+ }
+ return len;
+ }
+
+ public int getSizeInBytes(int ordinal) {
+ switch (schemas[ordinal].getSchemaType()) {
+ case FIXED:
+ return schemas[ordinal].getLength();
+ case VARIABLE:
+ return getByteArray(ordinal).length + 2;
+ case STRUCT:
+ return getRow(ordinal).getTotalSizeInBytes();
+ default:
+ throw new UnsupportedOperationException("wrong type");
+ }
+ }
+
+ public int getColumnCount() {
+ return schemas.length;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRowImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRowImpl.java b/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRowImpl.java
new file mode 100644
index 0000000..adec346
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRowImpl.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore.row;
+
+import org.apache.carbondata.core.indexstore.schema.DataMapSchema;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+
+/**
+ * Data map row.
+ */
+public class DataMapRowImpl extends DataMapRow {
+
+ private Object[] data;
+
+ public DataMapRowImpl(DataMapSchema[] schemas) {
+ super(schemas);
+ this.data = new Object[schemas.length];
+ }
+
+ @Override public byte[] getByteArray(int ordinal) {
+ return (byte[]) data[ordinal];
+ }
+
+ @Override public DataMapRow getRow(int ordinal) {
+ return (DataMapRow) data[ordinal];
+ }
+
+ @Override public void setByteArray(byte[] byteArray, int ordinal) {
+ assert (schemas[ordinal].getDataType() == DataType.BYTE_ARRAY);
+ data[ordinal] = byteArray;
+ }
+
+ @Override public int getInt(int ordinal) {
+ return (Integer) data[ordinal];
+ }
+
+ @Override public void setInt(int value, int ordinal) {
+ assert (schemas[ordinal].getDataType() == DataType.INT);
+ data[ordinal] = value;
+ }
+
+ @Override public void setByte(byte value, int ordinal) {
+ assert (schemas[ordinal].getDataType() == DataType.BYTE);
+ data[ordinal] = value;
+ }
+
+ @Override public byte getByte(int ordinal) {
+ return (Byte) data[ordinal];
+ }
+
+ @Override public void setShort(short value, int ordinal) {
+ assert (schemas[ordinal].getDataType() == DataType.SHORT);
+ data[ordinal] = value;
+ }
+
+ @Override public short getShort(int ordinal) {
+ return (Short) data[ordinal];
+ }
+
+ @Override public void setLong(long value, int ordinal) {
+ assert (schemas[ordinal].getDataType() == DataType.LONG);
+ data[ordinal] = value;
+ }
+
+ @Override public long getLong(int ordinal) {
+ return (Long) data[ordinal];
+ }
+
+ @Override public void setFloat(float value, int ordinal) {
+ assert (schemas[ordinal].getDataType() == DataType.FLOAT);
+ data[ordinal] = value;
+ }
+
+ @Override public float getFloat(int ordinal) {
+ return (Float) data[ordinal];
+ }
+
+ @Override public void setDouble(double value, int ordinal) {
+ assert (schemas[ordinal].getDataType() == DataType.DOUBLE);
+ data[ordinal] = value;
+ }
+
+ @Override public void setRow(DataMapRow row, int ordinal) {
+ assert (schemas[ordinal].getDataType() == DataType.STRUCT);
+ data[ordinal] = row;
+ }
+
+ @Override public double getDouble(int ordinal) {
+ return (Double) data[ordinal];
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java b/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java
new file mode 100644
index 0000000..ef78514
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.indexstore.row;
+
+import org.apache.carbondata.core.indexstore.schema.DataMapSchema;
+import org.apache.carbondata.core.memory.MemoryBlock;
+
+import static org.apache.carbondata.core.memory.CarbonUnsafe.BYTE_ARRAY_OFFSET;
+import static org.apache.carbondata.core.memory.CarbonUnsafe.unsafe;
+
+/**
+ * Unsafe implementation of data map row.
+ */
+public class UnsafeDataMapRow extends DataMapRow {
+
+ private MemoryBlock block;
+
+ private int pointer;
+
+ public UnsafeDataMapRow(DataMapSchema[] schemas, MemoryBlock block, int pointer) {
+ super(schemas);
+ this.block = block;
+ this.pointer = pointer;
+ }
+
+ @Override public byte[] getByteArray(int ordinal) {
+ int length;
+ int position = getPosition(ordinal);
+ switch (schemas[ordinal].getSchemaType()) {
+ case VARIABLE:
+ length = unsafe.getShort(block.getBaseObject(), block.getBaseOffset() + pointer + position);
+ position += 2;
+ break;
+ default:
+ length = schemas[ordinal].getLength();
+ }
+ byte[] data = new byte[length];
+ unsafe.copyMemory(block.getBaseObject(), block.getBaseOffset() + pointer + position, data,
+ BYTE_ARRAY_OFFSET, data.length);
+ return data;
+ }
+
+ @Override public DataMapRow getRow(int ordinal) {
+ DataMapSchema[] childSchemas =
+ ((DataMapSchema.StructDataMapSchema) schemas[ordinal]).getChildSchemas();
+ return new UnsafeDataMapRow(childSchemas, block, pointer + getPosition(ordinal));
+ }
+
+ @Override public void setByteArray(byte[] byteArray, int ordinal) {
+ throw new UnsupportedOperationException("Not supported to set on unsafe row");
+ }
+
+ @Override public int getInt(int ordinal) {
+ return unsafe
+ .getInt(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal));
+ }
+
+ @Override public void setInt(int value, int ordinal) {
+ throw new UnsupportedOperationException("Not supported to set on unsafe row");
+ }
+
+ @Override public void setByte(byte value, int ordinal) {
+ throw new UnsupportedOperationException("Not supported to set on unsafe row");
+ }
+
+ @Override public byte getByte(int ordinal) {
+ return unsafe
+ .getByte(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal));
+ }
+
+ @Override public void setShort(short value, int ordinal) {
+ throw new UnsupportedOperationException("Not supported to set on unsafe row");
+ }
+
+ @Override public short getShort(int ordinal) {
+ return unsafe
+ .getShort(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal));
+ }
+
+ @Override public void setLong(long value, int ordinal) {
+ throw new UnsupportedOperationException("Not supported to set on unsafe row");
+ }
+
+ @Override public long getLong(int ordinal) {
+ return unsafe
+ .getLong(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal));
+ }
+
+ @Override public void setFloat(float value, int ordinal) {
+ throw new UnsupportedOperationException("Not supported to set on unsafe row");
+ }
+
+ @Override public float getFloat(int ordinal) {
+ return unsafe
+ .getFloat(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal));
+ }
+
+ @Override public void setDouble(double value, int ordinal) {
+ throw new UnsupportedOperationException("Not supported to set on unsafe row");
+ }
+
+ @Override public double getDouble(int ordinal) {
+ return unsafe
+ .getDouble(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal));
+ }
+
+ @Override public void setRow(DataMapRow row, int ordinal) {
+ throw new UnsupportedOperationException("Not supported to set on unsafe row");
+ }
+
+ private int getPosition(int ordinal) {
+ int position = 0;
+ for (int i = 0; i < ordinal; i++) {
+ position += getSizeInBytes(i);
+ }
+ return position;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/indexstore/schema/DataMapSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/schema/DataMapSchema.java b/core/src/main/java/org/apache/carbondata/core/indexstore/schema/DataMapSchema.java
new file mode 100644
index 0000000..80c68ac
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/schema/DataMapSchema.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore.schema;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+
+/**
+ * It just have 2 types right now, either fixed or variable.
+ */
+public abstract class DataMapSchema {
+
+ protected DataType dataType;
+
+ public DataMapSchema(DataType dataType) {
+ this.dataType = dataType;
+ }
+
+ /**
+ * Either fixed or variable length.
+ *
+ * @return
+ */
+ public DataType getDataType() {
+ return dataType;
+ }
+
+ /**
+ * Gives length in case of fixed schema other wise returns length
+ *
+ * @return
+ */
+ public abstract int getLength();
+
+ /**
+ * schema type
+ * @return
+ */
+ public abstract DataMapSchemaType getSchemaType();
+
+ /*
+ * It has always fixed length, length cannot be updated later.
+ * Usage examples : all primitive types like short, int etc
+ */
+ public static class FixedDataMapSchema extends DataMapSchema {
+
+ private int length;
+
+ public FixedDataMapSchema(DataType dataType) {
+ super(dataType);
+ }
+
+ public FixedDataMapSchema(DataType dataType, int length) {
+ super(dataType);
+ this.length = length;
+ }
+
+ @Override public int getLength() {
+ if (length == 0) {
+ return dataType.getSizeInBytes();
+ } else {
+ return length;
+ }
+ }
+
+ @Override public DataMapSchemaType getSchemaType() {
+ return DataMapSchemaType.FIXED;
+ }
+ }
+
+ public static class VariableDataMapSchema extends DataMapSchema {
+
+ public VariableDataMapSchema(DataType dataType) {
+ super(dataType);
+ }
+
+ @Override public int getLength() {
+ return dataType.getSizeInBytes();
+ }
+
+ @Override public DataMapSchemaType getSchemaType() {
+ return DataMapSchemaType.VARIABLE;
+ }
+ }
+
+ public static class StructDataMapSchema extends DataMapSchema {
+
+ private DataMapSchema[] childSchemas;
+
+ public StructDataMapSchema(DataType dataType, DataMapSchema[] childSchemas) {
+ super(dataType);
+ this.childSchemas = childSchemas;
+ }
+
+ @Override public int getLength() {
+ return dataType.getSizeInBytes();
+ }
+
+ public DataMapSchema[] getChildSchemas() {
+ return childSchemas;
+ }
+
+ @Override public DataMapSchemaType getSchemaType() {
+ return DataMapSchemaType.STRUCT;
+ }
+ }
+
+ public enum DataMapSchemaType {
+ FIXED, VARIABLE, STRUCT
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/indexstore/schema/FilterType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/schema/FilterType.java b/core/src/main/java/org/apache/carbondata/core/indexstore/schema/FilterType.java
new file mode 100644
index 0000000..9d77010
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/schema/FilterType.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore.schema;
+
+/**
+ * Types of filters of select query
+ */
+public enum FilterType {
+ EQUALTO, GREATER_THAN, LESS_THAN, GREATER_THAN_EQUAL, LESS_THAN_EQUAL, LIKE
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
index bfa9d7e..f81f805 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
@@ -17,16 +17,22 @@
package org.apache.carbondata.core.metadata.blocklet;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
import java.io.Serializable;
+import java.util.ArrayList;
import java.util.List;
import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
+import org.apache.hadoop.io.Writable;
+
/**
* class to store the information about the blocklet
*/
-public class BlockletInfo implements Serializable {
+public class BlockletInfo implements Serializable, Writable {
/**
* serialization id
@@ -189,4 +195,49 @@ public class BlockletInfo implements Serializable {
this.numberOfPages = numberOfPages;
}
+ @Override public void write(DataOutput output) throws IOException {
+ output.writeLong(dimensionOffset);
+ output.writeLong(measureOffsets);
+ int dsize = dimensionChunkOffsets != null ? dimensionChunkOffsets.size() : 0;
+ output.writeShort(dsize);
+ for (int i = 0; i < dsize; i++) {
+ output.writeLong(dimensionChunkOffsets.get(i));
+ }
+ for (int i = 0; i < dsize; i++) {
+ output.writeInt(dimensionChunksLength.get(i));
+ }
+ int mSize = measureChunkOffsets != null ? measureChunkOffsets.size() : 0;
+ output.writeShort(mSize);
+ for (int i = 0; i < mSize; i++) {
+ output.writeLong(measureChunkOffsets.get(i));
+ }
+ for (int i = 0; i < mSize; i++) {
+ output.writeInt(measureChunksLength.get(i));
+ }
+ }
+
+ @Override public void readFields(DataInput input) throws IOException {
+ dimensionOffset = input.readLong();
+ measureOffsets = input.readLong();
+ short dimensionChunkOffsetsSize = input.readShort();
+ dimensionChunkOffsets = new ArrayList<>(dimensionChunkOffsetsSize);
+ for (int i = 0; i < dimensionChunkOffsetsSize; i++) {
+ dimensionChunkOffsets.add(input.readLong());
+ }
+ dimensionChunksLength = new ArrayList<>(dimensionChunkOffsetsSize);
+ for (int i = 0; i < dimensionChunkOffsetsSize; i++) {
+ dimensionChunksLength.add(input.readInt());
+ }
+
+ short measureChunkOffsetsSize = input.readShort();
+ measureChunkOffsets = new ArrayList<>(measureChunkOffsetsSize);
+ for (int i = 0; i < measureChunkOffsetsSize; i++) {
+ measureChunkOffsets.add(input.readLong());
+ }
+ measureChunksLength = new ArrayList<>(measureChunkOffsetsSize);
+ for (int i = 0; i < measureChunkOffsetsSize; i++) {
+ measureChunksLength.add(input.readInt());
+ }
+
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/metadata/index/BlockIndexInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/index/BlockIndexInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/index/BlockIndexInfo.java
index cd86a07..ae99ed8 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/index/BlockIndexInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/index/BlockIndexInfo.java
@@ -16,6 +16,7 @@
*/
package org.apache.carbondata.core.metadata.index;
+import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
/**
@@ -45,6 +46,11 @@ public class BlockIndexInfo {
private BlockletIndex blockletIndex;
/**
+ * to store blocklet info like offsets and lengths of each column.
+ */
+ private BlockletInfo blockletInfo;
+
+ /**
* Constructor
*
* @param numberOfRows number of rows
@@ -61,6 +67,20 @@ public class BlockIndexInfo {
}
/**
+ *
+ * @param numberOfRows
+ * @param fileName
+ * @param offset
+ * @param blockletIndex
+ * @param blockletInfo
+ */
+ public BlockIndexInfo(long numberOfRows, String fileName, long offset,
+ BlockletIndex blockletIndex, BlockletInfo blockletInfo) {
+ this(numberOfRows, fileName, offset, blockletIndex);
+ this.blockletInfo = blockletInfo;
+ }
+
+ /**
* @return the numberOfRows
*/
public long getNumberOfRows() {
@@ -87,4 +107,11 @@ public class BlockIndexInfo {
public BlockletIndex getBlockletIndex() {
return blockletIndex;
}
+
+ /**
+ * @return BlockletInfo
+ */
+ public BlockletInfo getBlockletInfo() {
+ return blockletInfo;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index ff54673..e0ee5bb 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -21,8 +21,10 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
@@ -41,6 +43,7 @@ import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier;
+import org.apache.carbondata.core.indexstore.blockletindex.IndexWrapper;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
@@ -116,23 +119,40 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
// so block will be loaded in sorted order this will be required for
// query execution
Collections.sort(queryModel.getTableBlockInfos());
- // get the table blocks
- CacheProvider cacheProvider = CacheProvider.getInstance();
- BlockIndexStore<TableBlockUniqueIdentifier, AbstractIndex> cache =
- (BlockIndexStore) cacheProvider
- .createCache(CacheType.EXECUTOR_BTREE, queryModel.getTable().getStorePath());
- // remove the invalid table blocks, block which is deleted or compacted
- cache.removeTableBlocks(queryModel.getInvalidSegmentIds(),
- queryModel.getAbsoluteTableIdentifier());
- List<TableBlockUniqueIdentifier> tableBlockUniqueIdentifiers =
- prepareTableBlockUniqueIdentifier(queryModel.getTableBlockInfos(),
- queryModel.getAbsoluteTableIdentifier());
- cache.removeTableBlocksIfHorizontalCompactionDone(queryModel);
- queryProperties.dataBlocks = cache.getAll(tableBlockUniqueIdentifiers);
- queryStatistic
- .addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR, System.currentTimeMillis());
- queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic);
+ if (queryModel.getTableBlockInfos().get(0).getDetailInfo() != null) {
+ List<AbstractIndex> indexList = new ArrayList<>();
+ Map<String, List<TableBlockInfo>> listMap = new LinkedHashMap<>();
+ for (TableBlockInfo blockInfo: queryModel.getTableBlockInfos()) {
+ List<TableBlockInfo> tableBlockInfos = listMap.get(blockInfo.getFilePath());
+ if (tableBlockInfos == null) {
+ tableBlockInfos = new ArrayList<>();
+ listMap.put(blockInfo.getFilePath(), tableBlockInfos);
+ }
+ tableBlockInfos.add(blockInfo);
+ }
+ for (List<TableBlockInfo> tableBlockInfos: listMap.values()) {
+ indexList.add(new IndexWrapper(tableBlockInfos));
+ }
+ queryProperties.dataBlocks = indexList;
+ } else {
+ // get the table blocks
+ CacheProvider cacheProvider = CacheProvider.getInstance();
+ BlockIndexStore<TableBlockUniqueIdentifier, AbstractIndex> cache =
+ (BlockIndexStore) cacheProvider
+ .createCache(CacheType.EXECUTOR_BTREE, queryModel.getTable().getStorePath());
+ // remove the invalid table blocks, block which is deleted or compacted
+ cache.removeTableBlocks(queryModel.getInvalidSegmentIds(),
+ queryModel.getAbsoluteTableIdentifier());
+ List<TableBlockUniqueIdentifier> tableBlockUniqueIdentifiers =
+ prepareTableBlockUniqueIdentifier(queryModel.getTableBlockInfos(),
+ queryModel.getAbsoluteTableIdentifier());
+ cache.removeTableBlocksIfHorizontalCompactionDone(queryModel);
+ queryProperties.dataBlocks = cache.getAll(tableBlockUniqueIdentifiers);
+ queryStatistic
+ .addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR, System.currentTimeMillis());
+ queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic);
+ }
// calculating the total number of aggeragted columns
int aggTypeCount = queryModel.getQueryMeasures().size();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
index 8704496..a874835 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -156,7 +156,7 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
int columnIndex = dimColumnEvaluatorInfo.getColumnIndex();
int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex);
- boolean isScanRequired =
+ boolean isScanRequired = blockIndex >= blkMaxVal.length ||
isScanRequired(blkMaxVal[blockIndex], blkMinVal[blockIndex], filterValues);
if (isScanRequired) {
bitSet.set(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
index 6823531..c2e077e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
@@ -287,7 +287,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
BitSet bitSet = new BitSet(1);
byte[][] filterValues = this.filterRangesValues;
int columnIndex = this.dimColEvaluatorInfo.getColumnIndex();
- boolean isScanRequired =
+ boolean isScanRequired = columnIndex >= blockMinValue.length ||
isScanRequired(blockMinValue[columnIndex], blockMaxValue[columnIndex], filterValues);
if (isScanRequired) {
bitSet.set(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
index be82be7..73352cb 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
@@ -79,7 +79,7 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
@Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
- boolean isScanRequired =
+ boolean isScanRequired = dimensionBlocksIndex[0] >= blockMaxValue.length ||
isScanRequired(blockMaxValue[dimensionBlocksIndex[0]], filterRangeValues);
if (isScanRequired) {
bitSet.set(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
index 53da6c5..6e8e188 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
@@ -81,7 +81,7 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
@Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
- boolean isScanRequired =
+ boolean isScanRequired = dimensionBlocksIndex[0] >= blockMaxValue.length ||
isScanRequired(blockMaxValue[dimensionBlocksIndex[0]], filterRangeValues);
if (isScanRequired) {
bitSet.set(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index d694960..d6f7c86 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -81,7 +81,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
@Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
- boolean isScanRequired =
+ boolean isScanRequired = dimensionBlocksIndex[0] >= blockMaxValue.length ||
isScanRequired(blockMinValue[dimensionBlocksIndex[0]], filterRangeValues);
if (isScanRequired) {
bitSet.set(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index b3dd921..597ba52 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -82,7 +82,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
@Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
BitSet bitSet = new BitSet(1);
- boolean isScanRequired =
+ boolean isScanRequired = dimensionBlocksIndex[0] >= blockMaxValue.length ||
isScanRequired(blockMinValue[dimensionBlocksIndex[0]], filterRangeValues);
if (isScanRequired) {
bitSet.set(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
index fdb5483..ff4f5dd 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
@@ -165,6 +165,9 @@ public abstract class AbstractDataBlockIterator extends CarbonIterator<List<Obje
new BlocksChunkHolder(blockExecutionInfo.getTotalNumberDimensionBlock(),
blockExecutionInfo.getTotalNumberOfMeasureBlock(), fileReader);
blocksChunkHolder.setDataBlock(dataBlockIterator.next());
+ if (blocksChunkHolder.getDataBlock().getColumnsMaxValue() == null) {
+ return blocksChunkHolder;
+ }
if (blockletScanner.isScanRequired(blocksChunkHolder)) {
return blocksChunkHolder;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
index 92e9594..95030d3 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@ -32,6 +32,7 @@ import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.datastore.impl.btree.BTreeDataRefNodeFinder;
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataRefNodeWrapper;
import org.apache.carbondata.core.mutate.DeleteDeltaVo;
import org.apache.carbondata.core.reader.CarbonDeleteFilesDataReader;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
@@ -127,20 +128,27 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
// set the deleted row to block execution info
blockInfo.setDeletedRecordsMap(deletedRowsMap);
}
- DataRefNode startDataBlock = finder
- .findFirstDataBlock(blockInfo.getDataBlock().getDataRefNode(), blockInfo.getStartKey());
- while (startDataBlock.nodeNumber() < blockInfo.getStartBlockletIndex()) {
- startDataBlock = startDataBlock.getNextDataRefNode();
- }
- long numberOfBlockToScan = blockInfo.getNumberOfBlockletToScan();
- //if number of block is less than 0 then take end block.
- if (numberOfBlockToScan <= 0) {
- DataRefNode endDataBlock = finder
- .findLastDataBlock(blockInfo.getDataBlock().getDataRefNode(), blockInfo.getEndKey());
- numberOfBlockToScan = endDataBlock.nodeNumber() - startDataBlock.nodeNumber() + 1;
+ DataRefNode dataRefNode = blockInfo.getDataBlock().getDataRefNode();
+ if (dataRefNode instanceof BlockletDataRefNodeWrapper) {
+ BlockletDataRefNodeWrapper wrapper = (BlockletDataRefNodeWrapper) dataRefNode;
+ blockInfo.setFirstDataBlock(wrapper);
+ blockInfo.setNumberOfBlockToScan(wrapper.numberOfNodes());
+
+ } else {
+ DataRefNode startDataBlock =
+ finder.findFirstDataBlock(dataRefNode, blockInfo.getStartKey());
+ while (startDataBlock.nodeNumber() < blockInfo.getStartBlockletIndex()) {
+ startDataBlock = startDataBlock.getNextDataRefNode();
+ }
+ long numberOfBlockToScan = blockInfo.getNumberOfBlockletToScan();
+ //if number of block is less than 0 then take end block.
+ if (numberOfBlockToScan <= 0) {
+ DataRefNode endDataBlock = finder.findLastDataBlock(dataRefNode, blockInfo.getEndKey());
+ numberOfBlockToScan = endDataBlock.nodeNumber() - startDataBlock.nodeNumber() + 1;
+ }
+ blockInfo.setFirstDataBlock(startDataBlock);
+ blockInfo.setNumberOfBlockToScan(numberOfBlockToScan);
}
- blockInfo.setFirstDataBlock(startDataBlock);
- blockInfo.setNumberOfBlockToScan(numberOfBlockToScan);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index 97b1a1f..34c7709 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -122,6 +122,57 @@ public abstract class AbstractDataFileFooterConverter {
}
/**
+ * Below method will be used to get the index info from index file
+ *
+ * @param filePath file path of the index file
+ * @return list of index info
+ * @throws IOException problem while reading the index file
+ */
+ public List<DataFileFooter> getIndexInfo(String filePath) throws IOException {
+ CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
+ List<DataFileFooter> dataFileFooters = new ArrayList<DataFileFooter>();
+ String parentPath = filePath.substring(0, filePath.lastIndexOf("/"));
+ try {
+ // open the reader
+ indexReader.openThriftReader(filePath);
+ // get the index header
+ org.apache.carbondata.format.IndexHeader readIndexHeader = indexReader.readIndexHeader();
+ List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
+ List<org.apache.carbondata.format.ColumnSchema> table_columns =
+ readIndexHeader.getTable_columns();
+ for (int i = 0; i < table_columns.size(); i++) {
+ columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i)));
+ }
+ // get the segment info
+ SegmentInfo segmentInfo = getSegmentInfo(readIndexHeader.getSegment_info());
+ BlockletIndex blockletIndex = null;
+ DataFileFooter dataFileFooter = null;
+ // read the block info from file
+ while (indexReader.hasNext()) {
+ BlockIndex readBlockIndexInfo = indexReader.readBlockIndexInfo();
+ blockletIndex = getBlockletIndex(readBlockIndexInfo.getBlock_index());
+ dataFileFooter = new DataFileFooter();
+ TableBlockInfo tableBlockInfo = new TableBlockInfo();
+ tableBlockInfo.setBlockOffset(readBlockIndexInfo.getOffset());
+ tableBlockInfo.setVersion(
+ ColumnarFormatVersion.valueOf((short) readIndexHeader.getVersion()));
+ int blockletSize = getBlockletSize(readBlockIndexInfo);
+ tableBlockInfo.getBlockletInfos().setNoOfBlockLets(blockletSize);
+ tableBlockInfo.setFilePath(parentPath + "/" + readBlockIndexInfo.file_name);
+ dataFileFooter.setBlockletIndex(blockletIndex);
+ dataFileFooter.setColumnInTable(columnSchemaList);
+ dataFileFooter.setNumberOfRows(readBlockIndexInfo.getNum_rows());
+ dataFileFooter.setBlockInfo(new BlockInfo(tableBlockInfo));
+ dataFileFooter.setSegmentInfo(segmentInfo);
+ dataFileFooters.add(dataFileFooter);
+ }
+ } finally {
+ indexReader.closeThriftReader();
+ }
+ return dataFileFooters;
+ }
+
+ /**
* the methods returns the number of blocklets in a block
*
* @param readBlockIndexInfo
@@ -148,6 +199,8 @@ public abstract class AbstractDataFileFooterConverter {
public abstract DataFileFooter readDataFileFooter(TableBlockInfo tableBlockInfo)
throws IOException;
+ public abstract List<ColumnSchema> getSchema(TableBlockInfo tableBlockInfo) throws IOException;
+
/**
* Below method will be used to get blocklet index for data file meta
*
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index b9c164a..f62f3d8 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -54,10 +54,13 @@ import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.datastore.page.statistics.MeasurePageStatsVO;
+import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
+import org.apache.carbondata.core.metadata.blocklet.SegmentInfo;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
@@ -926,10 +929,26 @@ public final class CarbonUtil {
* Below method will be used to read the data file matadata
*/
public static DataFileFooter readMetadatFile(TableBlockInfo tableBlockInfo) throws IOException {
- AbstractDataFileFooterConverter fileFooterConverter =
- DataFileFooterConverterFactory.getInstance()
- .getDataFileFooterConverter(tableBlockInfo.getVersion());
- return fileFooterConverter.readDataFileFooter(tableBlockInfo);
+ BlockletDetailInfo detailInfo = tableBlockInfo.getDetailInfo();
+ if (detailInfo == null) {
+ AbstractDataFileFooterConverter fileFooterConverter =
+ DataFileFooterConverterFactory.getInstance()
+ .getDataFileFooterConverter(tableBlockInfo.getVersion());
+ return fileFooterConverter.readDataFileFooter(tableBlockInfo);
+ } else {
+ DataFileFooter fileFooter = new DataFileFooter();
+ fileFooter.setSchemaUpdatedTimeStamp(detailInfo.getSchemaUpdatedTimeStamp());
+ ColumnarFormatVersion version =
+ ColumnarFormatVersion.valueOf(detailInfo.getVersionNumber());
+ AbstractDataFileFooterConverter dataFileFooterConverter =
+ DataFileFooterConverterFactory.getInstance().getDataFileFooterConverter(version);
+ fileFooter.setColumnInTable(dataFileFooterConverter.getSchema(tableBlockInfo));
+ SegmentInfo segmentInfo = new SegmentInfo();
+ segmentInfo.setColumnCardinality(detailInfo.getDimLens());
+ segmentInfo.setNumberOfColumns(detailInfo.getRowCount());
+ fileFooter.setSegmentInfo(segmentInfo);
+ return fileFooter;
+ }
}
/**
@@ -1567,24 +1586,23 @@ public final class CarbonUtil {
}
/**
- * @param tableInfo
* @param invalidBlockVOForSegmentId
* @param updateStatusMngr
* @return
*/
- public static boolean isInvalidTableBlock(TableBlockInfo tableInfo,
+ public static boolean isInvalidTableBlock(String segmentId, String filePath,
UpdateVO invalidBlockVOForSegmentId, SegmentUpdateStatusManager updateStatusMngr) {
- if (!updateStatusMngr.isBlockValid(tableInfo.getSegmentId(),
- CarbonTablePath.getCarbonDataFileName(tableInfo.getFilePath()) + CarbonTablePath
+ if (!updateStatusMngr.isBlockValid(segmentId,
+ CarbonTablePath.getCarbonDataFileName(filePath) + CarbonTablePath
.getCarbonDataExtension())) {
return true;
}
if (null != invalidBlockVOForSegmentId) {
- Long blockTimeStamp = Long.parseLong(tableInfo.getFilePath()
- .substring(tableInfo.getFilePath().lastIndexOf('-') + 1,
- tableInfo.getFilePath().lastIndexOf('.')));
+ Long blockTimeStamp = Long.parseLong(filePath
+ .substring(filePath.lastIndexOf('-') + 1,
+ filePath.lastIndexOf('.')));
if ((blockTimeStamp > invalidBlockVOForSegmentId.getFactTimestamp() && (
invalidBlockVOForSegmentId.getUpdateDeltaStartTimestamp() != null
&& blockTimeStamp < invalidBlockVOForSegmentId.getUpdateDeltaStartTimestamp()))) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
index 0f82b95..3ac6987 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
@@ -121,4 +121,8 @@ public class DataFileFooterConverter extends AbstractDataFileFooterConverter {
blockletInfo.setNumberOfRows(blockletInfoThrift.getNum_rows());
return blockletInfo;
}
+
+ @Override public List<ColumnSchema> getSchema(TableBlockInfo tableBlockInfo) throws IOException {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
index 4882b0f..8cd437f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
@@ -140,4 +140,7 @@ public class DataFileFooterConverter2 extends AbstractDataFileFooterConverter {
return numberOfDimensionColumns;
}
+ @Override public List<ColumnSchema> getSchema(TableBlockInfo tableBlockInfo) throws IOException {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterV3.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterV3.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterV3.java
index 143c1b1..ccb8b29 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterV3.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterV3.java
@@ -85,6 +85,17 @@ public class DataFileFooterConverterV3 extends AbstractDataFileFooterConverter {
return dataFileFooter;
}
+ @Override public List<ColumnSchema> getSchema(TableBlockInfo tableBlockInfo) throws IOException {
+ CarbonHeaderReader carbonHeaderReader = new CarbonHeaderReader(tableBlockInfo.getFilePath());
+ FileHeader fileHeader = carbonHeaderReader.readHeader();
+ List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
+ List<org.apache.carbondata.format.ColumnSchema> table_columns = fileHeader.getColumn_schema();
+ for (int i = 0; i < table_columns.size(); i++) {
+ columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i)));
+ }
+ return columnSchemaList;
+ }
+
/**
* Below method is to convert the blocklet info of the thrift to wrapper
* blocklet info
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/format/src/main/thrift/carbondata_index.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/carbondata_index.thrift b/format/src/main/thrift/carbondata_index.thrift
index c055031..4df085a 100644
--- a/format/src/main/thrift/carbondata_index.thrift
+++ b/format/src/main/thrift/carbondata_index.thrift
@@ -41,4 +41,5 @@ struct BlockIndex{
2: required string file_name; // Block file name
3: required i64 offset; // Offset of the footer
4: required carbondata.BlockletIndex block_index; // Blocklet index
+ 5: optional carbondata.BlockletInfo3 blocklet_info;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index 1673193..81226a2 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -21,7 +21,14 @@ import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.DataRefNode;
@@ -375,8 +382,9 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
if (isIUDTable) {
// In case IUD is not performed in this table avoid searching for
// invalidated blocks.
- if (CarbonUtil.isInvalidTableBlock(tableBlockInfo, invalidBlockVOForSegmentId,
- updateStatusManager)) {
+ if (CarbonUtil
+ .isInvalidTableBlock(tableBlockInfo.getSegmentId(), tableBlockInfo.getFilePath(),
+ invalidBlockVOForSegmentId, updateStatusManager)) {
continue;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
index 631bc2c..56bade7 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
@@ -29,6 +29,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.BlockletInfos;
import org.apache.carbondata.core.datastore.block.Distributable;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.mutate.UpdateVO;
import org.apache.carbondata.core.util.ByteUtil;
@@ -77,6 +78,8 @@ public class CarbonInputSplit extends FileSplit
*/
private String[] deleteDeltaFiles;
+ private BlockletDetailInfo detailInfo;
+
public CarbonInputSplit() {
segmentId = null;
taskId = "0";
@@ -138,10 +141,12 @@ public class CarbonInputSplit extends FileSplit
BlockletInfos blockletInfos =
new BlockletInfos(split.getNumberOfBlocklets(), 0, split.getNumberOfBlocklets());
try {
- tableBlockInfoList.add(
+ TableBlockInfo blockInfo =
new TableBlockInfo(split.getPath().toString(), split.getStart(), split.getSegmentId(),
split.getLocations(), split.getLength(), blockletInfos, split.getVersion(),
- split.getDeleteDeltaFiles()));
+ split.getDeleteDeltaFiles());
+ blockInfo.setDetailInfo(split.getDetailInfo());
+ tableBlockInfoList.add(blockInfo);
} catch (IOException e) {
throw new RuntimeException("fail to get location of split: " + split, e);
}
@@ -153,9 +158,12 @@ public class CarbonInputSplit extends FileSplit
BlockletInfos blockletInfos =
new BlockletInfos(inputSplit.getNumberOfBlocklets(), 0, inputSplit.getNumberOfBlocklets());
try {
- return new TableBlockInfo(inputSplit.getPath().toString(), inputSplit.getStart(),
- inputSplit.getSegmentId(), inputSplit.getLocations(), inputSplit.getLength(),
- blockletInfos, inputSplit.getVersion(), inputSplit.getDeleteDeltaFiles());
+ TableBlockInfo blockInfo =
+ new TableBlockInfo(inputSplit.getPath().toString(), inputSplit.getStart(),
+ inputSplit.getSegmentId(), inputSplit.getLocations(), inputSplit.getLength(),
+ blockletInfos, inputSplit.getVersion(), inputSplit.getDeleteDeltaFiles());
+ blockInfo.setDetailInfo(inputSplit.getDetailInfo());
+ return blockInfo;
} catch (IOException e) {
throw new RuntimeException("fail to get location of split: " + inputSplit, e);
}
@@ -180,6 +188,11 @@ public class CarbonInputSplit extends FileSplit
for (int i = 0; i < numberOfDeleteDeltaFiles; i++) {
deleteDeltaFiles[i] = in.readUTF();
}
+ boolean detailInfoExists = in.readBoolean();
+ if (detailInfoExists) {
+ detailInfo = new BlockletDetailInfo();
+ detailInfo.readFields(in);
+ }
}
@Override public void write(DataOutput out) throws IOException {
@@ -197,6 +210,10 @@ public class CarbonInputSplit extends FileSplit
out.writeUTF(deleteDeltaFiles[i]);
}
}
+ out.writeBoolean(detailInfo != null);
+ if (detailInfo != null) {
+ detailInfo.write(out);
+ }
}
public List<String> getInvalidSegments() {
@@ -310,4 +327,16 @@ public class CarbonInputSplit extends FileSplit
public String[] getDeleteDeltaFiles() {
return deleteDeltaFiles;
}
+
+ public void setDeleteDeltaFiles(String[] deleteDeltaFiles) {
+ this.deleteDeltaFiles = deleteDeltaFiles;
+ }
+
+ public BlockletDetailInfo getDetailInfo() {
+ return detailInfo;
+ }
+
+ public void setDetailInfo(BlockletDetailInfo detailInfo) {
+ this.detailInfo = detailInfo;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index ae9c676..e73c04a 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -18,152 +18,556 @@
package org.apache.carbondata.hadoop.api;
import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.TableSegmentUniqueIdentifier;
+import org.apache.carbondata.core.indexstore.Blocklet;
+import org.apache.carbondata.core.indexstore.DataMapStoreManager;
+import org.apache.carbondata.core.indexstore.DataMapType;
+import org.apache.carbondata.core.indexstore.TableDataMap;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.schema.PartitionInfo;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
+import org.apache.carbondata.core.mutate.SegmentUpdateDetails;
+import org.apache.carbondata.core.mutate.UpdateVO;
+import org.apache.carbondata.core.mutate.data.BlockMappingVO;
import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.filter.FilterExpressionProcessor;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.scan.model.CarbonQueryPlan;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.partition.PartitionUtil;
+import org.apache.carbondata.core.scan.partition.Partitioner;
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
+import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
+import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.path.CarbonStorePath;
+import org.apache.carbondata.core.util.path.CarbonTablePath;
+import org.apache.carbondata.hadoop.CarbonMultiBlockSplit;
import org.apache.carbondata.hadoop.CarbonProjection;
-import org.apache.carbondata.hadoop.internal.CarbonInputSplit;
-import org.apache.carbondata.hadoop.internal.segment.Segment;
-import org.apache.carbondata.hadoop.internal.segment.SegmentManager;
-import org.apache.carbondata.hadoop.internal.segment.SegmentManagerFactory;
+import org.apache.carbondata.hadoop.CarbonRecordReader;
+import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport;
+import org.apache.carbondata.hadoop.readsupport.impl.DictionaryDecodeReadSupport;
import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil;
import org.apache.carbondata.hadoop.util.ObjectSerializationUtil;
+import org.apache.carbondata.hadoop.util.SchemaReader;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.InvalidPathException;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.mapreduce.security.TokenCache;
+import org.apache.hadoop.util.StringUtils;
/**
* Input format of CarbonData file.
+ *
* @param <T>
*/
public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
+ // comma separated list of input segment numbers
+ public static final String INPUT_SEGMENT_NUMBERS =
+ "mapreduce.input.carboninputformat.segmentnumbers";
+ // comma separated list of input files
+ public static final String INPUT_FILES = "mapreduce.input.carboninputformat.files";
+ private static final Log LOG = LogFactory.getLog(CarbonTableInputFormat.class);
private static final String FILTER_PREDICATE =
"mapreduce.input.carboninputformat.filter.predicate";
+ private static final String COLUMN_PROJECTION = "mapreduce.input.carboninputformat.projection";
+ private static final String CARBON_TABLE = "mapreduce.input.carboninputformat.table";
+ private static final String CARBON_READ_SUPPORT = "mapreduce.input.carboninputformat.readsupport";
- private SegmentManager segmentManager;
+ /**
+ * It is optional, if user does not set then it reads from store
+ *
+ * @param configuration
+ * @param carbonTable
+ * @throws IOException
+ */
+ public static void setCarbonTable(Configuration configuration, CarbonTable carbonTable)
+ throws IOException {
+ if (null != carbonTable) {
+ configuration.set(CARBON_TABLE, ObjectSerializationUtil.convertObjectToString(carbonTable));
+ }
+ }
- public CarbonTableInputFormat() {
- this.segmentManager = SegmentManagerFactory.getGlobalSegmentManager();
+ public static CarbonTable getCarbonTable(Configuration configuration) throws IOException {
+ String carbonTableStr = configuration.get(CARBON_TABLE);
+ if (carbonTableStr == null) {
+ populateCarbonTable(configuration);
+ // read it from schema file in the store
+ carbonTableStr = configuration.get(CARBON_TABLE);
+ return (CarbonTable) ObjectSerializationUtil.convertStringToObject(carbonTableStr);
+ }
+ return (CarbonTable) ObjectSerializationUtil.convertStringToObject(carbonTableStr);
}
- @Override
- public RecordReader<Void, T> createRecordReader(InputSplit split,
- TaskAttemptContext context) throws IOException, InterruptedException {
- switch (((CarbonInputSplit)split).formatType()) {
- case COLUMNAR:
- // TODO: create record reader for columnar format
- break;
- default:
- throw new RuntimeException("Unsupported format type");
+ /**
+ * this method will read the schema from the physical file and populate into CARBON_TABLE
+ *
+ * @param configuration
+ * @throws IOException
+ */
+ private static void populateCarbonTable(Configuration configuration) throws IOException {
+ String dirs = configuration.get(INPUT_DIR, "");
+ String[] inputPaths = StringUtils.split(dirs);
+ if (inputPaths.length == 0) {
+ throw new InvalidPathException("No input paths specified in job");
}
- return null;
+ AbsoluteTableIdentifier absoluteTableIdentifier =
+ AbsoluteTableIdentifier.fromTablePath(inputPaths[0]);
+ // read the schema file to get the absoluteTableIdentifier having the correct table id
+ // persisted in the schema
+ CarbonTable carbonTable = SchemaReader.readCarbonTableFromStore(absoluteTableIdentifier);
+ setCarbonTable(configuration, carbonTable);
}
- @Override
- public List<InputSplit> getSplits(JobContext job) throws IOException {
+ public static void setTablePath(Configuration configuration, String tablePath)
+ throws IOException {
+ configuration.set(FileInputFormat.INPUT_DIR, tablePath);
+ }
- // work as following steps:
- // get all current valid segment
- // for each segment, get all input split
+ /**
+ * It sets unresolved filter expression.
+ *
+ * @param configuration
+ * @param filterExpression
+ */
+ public static void setFilterPredicates(Configuration configuration, Expression filterExpression) {
+ if (filterExpression == null) {
+ return;
+ }
+ try {
+ String filterString = ObjectSerializationUtil.convertObjectToString(filterExpression);
+ configuration.set(FILTER_PREDICATE, filterString);
+ } catch (Exception e) {
+ throw new RuntimeException("Error while setting filter expression to Job", e);
+ }
+ }
- List<InputSplit> output = new LinkedList<>();
- Expression filter = getFilter(job.getConfiguration());
- Segment[] segments = segmentManager.getAllValidSegments();
- FilterResolverIntf filterResolver = CarbonInputFormatUtil.resolveFilter(filter, null);
- for (Segment segment: segments) {
- List<InputSplit> splits = segment.getSplits(job, filterResolver);
- output.addAll(splits);
+ public static void setColumnProjection(Configuration configuration, CarbonProjection projection) {
+ if (projection == null || projection.isEmpty()) {
+ return;
+ }
+ String[] allColumns = projection.getAllColumns();
+ StringBuilder builder = new StringBuilder();
+ for (String column : allColumns) {
+ builder.append(column).append(",");
}
- return output;
+ String columnString = builder.toString();
+ columnString = columnString.substring(0, columnString.length() - 1);
+ configuration.set(COLUMN_PROJECTION, columnString);
}
- /**
- * set the table path into configuration
- * @param conf configuration of the job
- * @param tablePath table path string
- */
- public void setTablePath(Configuration conf, String tablePath) {
+ public static String getColumnProjection(Configuration configuration) {
+ return configuration.get(COLUMN_PROJECTION);
+ }
+
+ public static void setCarbonReadSupport(Configuration configuration,
+ Class<? extends CarbonReadSupport> readSupportClass) {
+ if (readSupportClass != null) {
+ configuration.set(CARBON_READ_SUPPORT, readSupportClass.getName());
+ }
+ }
+ private static CarbonTablePath getTablePath(AbsoluteTableIdentifier absIdentifier) {
+ return CarbonStorePath.getCarbonTablePath(absIdentifier);
}
/**
- * return the table path in the configuration
- * @param conf configuration of the job
- * @return table path string
+ * Set list of segments to access
*/
- public String getTablePath(Configuration conf) {
- return null;
+ public static void setSegmentsToAccess(Configuration configuration, List<String> validSegments) {
+ configuration.set(INPUT_SEGMENT_NUMBERS, CarbonUtil.getSegmentString(validSegments));
}
/**
- * set projection columns into configuration
- * @param conf configuration of the job
- * @param projection projection
+ * Set list of files to access
*/
- public void setProjection(Configuration conf, CarbonProjection projection) {
+ public static void setFilesToAccess(Configuration configuration, List<String> validFiles) {
+ configuration.set(INPUT_FILES, CarbonUtil.getSegmentString(validFiles));
+ }
+ private static AbsoluteTableIdentifier getAbsoluteTableIdentifier(Configuration configuration)
+ throws IOException {
+ return getCarbonTable(configuration).getAbsoluteTableIdentifier();
}
/**
- * return the projection in the configuration
- * @param conf configuration of the job
- * @return projection
+ * {@inheritDoc}
+ * Configurations FileInputFormat.INPUT_DIR
+ * are used to get table path to read.
+ *
+ * @param job
+ * @return List<InputSplit> list of CarbonInputSplit
+ * @throws IOException
*/
- public CarbonProjection getProjection(Configuration conf) {
- return null;
+ @Override public List<InputSplit> getSplits(JobContext job) throws IOException {
+ AbsoluteTableIdentifier identifier = getAbsoluteTableIdentifier(job.getConfiguration());
+ TableDataMap blockletMap =
+ DataMapStoreManager.getInstance().getDataMap(identifier, "blocklet", DataMapType.BLOCKLET);
+ List<String> invalidSegments = new ArrayList<>();
+ List<UpdateVO> invalidTimestampsList = new ArrayList<>();
+ List<String> validSegments = Arrays.asList(getSegmentsToAccess(job));
+ // get all valid segments and set them into the configuration
+ if (validSegments.size() == 0) {
+ SegmentStatusManager segmentStatusManager = new SegmentStatusManager(identifier);
+ SegmentStatusManager.ValidAndInvalidSegmentsInfo segments =
+ segmentStatusManager.getValidAndInvalidSegments();
+ SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(identifier);
+ validSegments = segments.getValidSegments();
+ if (validSegments.size() == 0) {
+ return new ArrayList<>(0);
+ }
+
+ // remove entry in the segment index if there are invalid segments
+ invalidSegments.addAll(segments.getInvalidSegments());
+ for (String invalidSegmentId : invalidSegments) {
+ invalidTimestampsList.add(updateStatusManager.getInvalidTimestampRange(invalidSegmentId));
+ }
+ if (invalidSegments.size() > 0) {
+ List<TableSegmentUniqueIdentifier> invalidSegmentsIds =
+ new ArrayList<>(invalidSegments.size());
+ for (String segId : invalidSegments) {
+ invalidSegmentsIds.add(new TableSegmentUniqueIdentifier(identifier, segId));
+ }
+ blockletMap.clear(invalidSegments);
+ }
+ }
+
+ // process and resolve the expression
+ Expression filter = getFilterPredicates(job.getConfiguration());
+ CarbonTable carbonTable = getCarbonTable(job.getConfiguration());
+ // this will be null in case of corrupt schema file.
+ if (null == carbonTable) {
+ throw new IOException("Missing/Corrupt schema file for table.");
+ }
+
+ CarbonInputFormatUtil.processFilterExpression(filter, carbonTable);
+
+ // prune partitions for filter query on partition table
+ BitSet matchedPartitions = null;
+ if (null != filter) {
+ PartitionInfo partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName());
+ if (null != partitionInfo) {
+ Partitioner partitioner = PartitionUtil.getPartitioner(partitionInfo);
+ matchedPartitions = new FilterExpressionProcessor()
+ .getFilteredPartitions(filter, partitionInfo, partitioner);
+ if (matchedPartitions.cardinality() == 0) {
+ // no partition is required
+ return new ArrayList<InputSplit>();
+ }
+ if (matchedPartitions.cardinality() == partitioner.numPartitions()) {
+ // all partitions are required, no need to prune partitions
+ matchedPartitions = null;
+ }
+ }
+ }
+
+ FilterResolverIntf filterInterface = CarbonInputFormatUtil.resolveFilter(filter, identifier);
+
+ // do block filtering and get split
+ List<InputSplit> splits = getSplits(job, filterInterface, validSegments, matchedPartitions);
+ // pass the invalid segment to task side in order to remove index entry in task side
+ if (invalidSegments.size() > 0) {
+ for (InputSplit split : splits) {
+ ((org.apache.carbondata.hadoop.CarbonInputSplit) split).setInvalidSegments(invalidSegments);
+ ((org.apache.carbondata.hadoop.CarbonInputSplit) split)
+ .setInvalidTimestampRange(invalidTimestampsList);
+ }
+ }
+ return splits;
}
/**
- * set filter expression into the configuration
- * @param conf configuration of the job
- * @param filter filter expression
+ * {@inheritDoc}
+ * Configurations FileInputFormat.INPUT_DIR, CarbonInputFormat.INPUT_SEGMENT_NUMBERS
+ * are used to get table path to read.
+ *
+ * @return
+ * @throws IOException
*/
- public void setFilter(Configuration conf, Expression filter) {
+ private List<InputSplit> getSplits(JobContext job, FilterResolverIntf filterResolver,
+ List<String> validSegments, BitSet matchedPartitions) throws IOException {
+
+ List<InputSplit> result = new LinkedList<InputSplit>();
+ UpdateVO invalidBlockVOForSegmentId = null;
+ Boolean isIUDTable = false;
+
+ AbsoluteTableIdentifier absoluteTableIdentifier =
+ getCarbonTable(job.getConfiguration()).getAbsoluteTableIdentifier();
+ SegmentUpdateStatusManager updateStatusManager =
+ new SegmentUpdateStatusManager(absoluteTableIdentifier);
+
+ isIUDTable = (updateStatusManager.getUpdateStatusDetails().length != 0);
+
+ //for each segment fetch blocks matching filter in Driver BTree
+ List<org.apache.carbondata.hadoop.CarbonInputSplit> dataBlocksOfSegment =
+ getDataBlocksOfSegment(job, absoluteTableIdentifier, filterResolver, matchedPartitions,
+ validSegments);
+ for (org.apache.carbondata.hadoop.CarbonInputSplit inputSplit : dataBlocksOfSegment) {
+
+ // Get the UpdateVO for those tables on which IUD operations being performed.
+ if (isIUDTable) {
+ invalidBlockVOForSegmentId =
+ updateStatusManager.getInvalidTimestampRange(inputSplit.getSegmentId());
+ }
+ if (isIUDTable) {
+ // In case IUD is not performed in this table avoid searching for
+ // invalidated blocks.
+ if (CarbonUtil
+ .isInvalidTableBlock(inputSplit.getSegmentId(), inputSplit.getPath().toString(),
+ invalidBlockVOForSegmentId, updateStatusManager)) {
+ continue;
+ }
+ }
+ String[] deleteDeltaFilePath = null;
+ try {
+ deleteDeltaFilePath =
+ updateStatusManager.getDeleteDeltaFilePath(inputSplit.getPath().toString());
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ inputSplit.setDeleteDeltaFiles(deleteDeltaFilePath);
+ result.add(inputSplit);
+ }
+ return result;
+ }
+
+ protected Expression getFilterPredicates(Configuration configuration) {
try {
- String filterString = ObjectSerializationUtil.convertObjectToString(filter);
- conf.set(FILTER_PREDICATE, filterString);
- } catch (Exception e) {
- throw new RuntimeException("Error while setting filter expression to Job", e);
+ String filterExprString = configuration.get(FILTER_PREDICATE);
+ if (filterExprString == null) {
+ return null;
+ }
+ Object filter = ObjectSerializationUtil.convertStringToObject(filterExprString);
+ return (Expression) filter;
+ } catch (IOException e) {
+ throw new RuntimeException("Error while reading filter expression", e);
}
}
/**
- * return filter expression in the configuration
- * @param conf configuration of the job
- * @return filter expression
+ * get data blocks of given segment
*/
- public Expression getFilter(Configuration conf) {
- Object filter;
- String filterExprString = conf.get(FILTER_PREDICATE);
- if (filterExprString == null) {
- return null;
+ private List<org.apache.carbondata.hadoop.CarbonInputSplit> getDataBlocksOfSegment(JobContext job,
+ AbsoluteTableIdentifier absoluteTableIdentifier, FilterResolverIntf resolver,
+ BitSet matchedPartitions, List<String> segmentIds) throws IOException {
+
+ QueryStatisticsRecorder recorder = CarbonTimeStatisticsFactory.createDriverRecorder();
+ QueryStatistic statistic = new QueryStatistic();
+
+ // get tokens for all the required FileSystem for table path
+ TokenCache.obtainTokensForNamenodes(job.getCredentials(),
+ new Path[] { new Path(absoluteTableIdentifier.getTablePath()) }, job.getConfiguration());
+
+ TableDataMap blockletMap = DataMapStoreManager.getInstance()
+ .getDataMap(absoluteTableIdentifier, "blocklet", DataMapType.BLOCKLET);
+ List<Blocklet> prunedBlocklets = blockletMap.prune(segmentIds, resolver);
+
+ List<org.apache.carbondata.hadoop.CarbonInputSplit> resultFilterredBlocks = new ArrayList<>();
+ for (Blocklet blocklet : prunedBlocklets) {
+ int taskId = CarbonTablePath.DataFileUtil.getTaskIdFromTaskNo(
+ CarbonTablePath.DataFileUtil.getTaskNo(blocklet.getPath().toString()));
+
+ // matchedPartitions variable will be null in two cases as follows
+ // 1. the table is not a partition table
+ // 2. the table is a partition table, and all partitions are matched by query
+ // for partition table, the task id of carbaondata file name is the partition id.
+ // if this partition is not required, here will skip it.
+ if (matchedPartitions == null || matchedPartitions.get(taskId)) {
+ resultFilterredBlocks.add(convertToCarbonInputSplit(blocklet));
+ }
}
+ statistic
+ .addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_DRIVER, System.currentTimeMillis());
+ recorder.recordStatisticsForDriver(statistic, job.getConfiguration().get("query.id"));
+ return resultFilterredBlocks;
+ }
+
+ private org.apache.carbondata.hadoop.CarbonInputSplit convertToCarbonInputSplit(Blocklet blocklet)
+ throws IOException {
+ blocklet.updateLocations();
+ org.apache.carbondata.hadoop.CarbonInputSplit split =
+ org.apache.carbondata.hadoop.CarbonInputSplit.from(blocklet.getSegmentId(),
+ new FileSplit(blocklet.getPath(), 0, blocklet.getLength(), blocklet.getLocations()),
+ ColumnarFormatVersion.valueOf((short) blocklet.getDetailInfo().getVersionNumber()));
+ split.setDetailInfo(blocklet.getDetailInfo());
+ return split;
+ }
+
+ @Override public RecordReader<Void, T> createRecordReader(InputSplit inputSplit,
+ TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
+ Configuration configuration = taskAttemptContext.getConfiguration();
+ QueryModel queryModel = getQueryModel(inputSplit, taskAttemptContext);
+ CarbonReadSupport<T> readSupport = getReadSupportClass(configuration);
+ return new CarbonRecordReader<T>(queryModel, readSupport);
+ }
+
+ public QueryModel getQueryModel(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
+ throws IOException {
+ Configuration configuration = taskAttemptContext.getConfiguration();
+ CarbonTable carbonTable = getCarbonTable(configuration);
+ // getting the table absoluteTableIdentifier from the carbonTable
+ // to avoid unnecessary deserialization
+ AbsoluteTableIdentifier identifier = carbonTable.getAbsoluteTableIdentifier();
+
+ // query plan includes projection column
+ String projection = getColumnProjection(configuration);
+ CarbonQueryPlan queryPlan = CarbonInputFormatUtil.createQueryPlan(carbonTable, projection);
+ QueryModel queryModel = QueryModel.createModel(identifier, queryPlan, carbonTable);
+
+ // set the filter to the query model in order to filter blocklet before scan
+ Expression filter = getFilterPredicates(configuration);
+ CarbonInputFormatUtil.processFilterExpression(filter, carbonTable);
+ FilterResolverIntf filterIntf = CarbonInputFormatUtil.resolveFilter(filter, identifier);
+ queryModel.setFilterExpressionResolverTree(filterIntf);
+
+ // update the file level index store if there are invalid segment
+ if (inputSplit instanceof CarbonMultiBlockSplit) {
+ CarbonMultiBlockSplit split = (CarbonMultiBlockSplit) inputSplit;
+ List<String> invalidSegments = split.getAllSplits().get(0).getInvalidSegments();
+ if (invalidSegments.size() > 0) {
+ queryModel.setInvalidSegmentIds(invalidSegments);
+ }
+ List<UpdateVO> invalidTimestampRangeList =
+ split.getAllSplits().get(0).getInvalidTimestampRange();
+ if ((null != invalidTimestampRangeList) && (invalidTimestampRangeList.size() > 0)) {
+ queryModel.setInvalidBlockForSegmentId(invalidTimestampRangeList);
+ }
+ }
+ return queryModel;
+ }
+
+ public CarbonReadSupport<T> getReadSupportClass(Configuration configuration) {
+ String readSupportClass = configuration.get(CARBON_READ_SUPPORT);
+ //By default it uses dictionary decoder read class
+ CarbonReadSupport<T> readSupport = null;
+ if (readSupportClass != null) {
+ try {
+ Class<?> myClass = Class.forName(readSupportClass);
+ Constructor<?> constructor = myClass.getConstructors()[0];
+ Object object = constructor.newInstance();
+ if (object instanceof CarbonReadSupport) {
+ readSupport = (CarbonReadSupport) object;
+ }
+ } catch (ClassNotFoundException ex) {
+ LOG.error("Class " + readSupportClass + "not found", ex);
+ } catch (Exception ex) {
+ LOG.error("Error while creating " + readSupportClass, ex);
+ }
+ } else {
+ readSupport = new DictionaryDecodeReadSupport<>();
+ }
+ return readSupport;
+ }
+
+ @Override protected boolean isSplitable(JobContext context, Path filename) {
try {
- filter = ObjectSerializationUtil.convertStringToObject(filterExprString);
- } catch (IOException e) {
- throw new RuntimeException("Error while reading filter expression", e);
+ // Don't split the file if it is local file system
+ FileSystem fileSystem = filename.getFileSystem(context.getConfiguration());
+ if (fileSystem instanceof LocalFileSystem) {
+ return false;
+ }
+ } catch (Exception e) {
+ return true;
+ }
+ return true;
+ }
+
+ /**
+ * required to be moved to core
+ *
+ * @return updateExtension
+ */
+ private String getUpdateExtension() {
+ // TODO: required to modify when supporting update, mostly will be update timestamp
+ return "update";
+ }
+
+ /**
+ * return valid segment to access
+ */
+ private String[] getSegmentsToAccess(JobContext job) {
+ String segmentString = job.getConfiguration().get(INPUT_SEGMENT_NUMBERS, "");
+ if (segmentString.trim().isEmpty()) {
+ return new String[0];
}
- assert (filter instanceof Expression);
- return (Expression) filter;
+ return segmentString.split(",");
}
/**
- * Optional API. It can be used by query optimizer to select index based on filter
- * in the configuration of the job. After selecting index internally, index' name will be set
- * in the configuration.
+ * Get the row count of the Block and mapping of segment and Block count.
*
- * The process of selection is simple, just use the default index. Subclass can provide a more
- * advanced selection logic like cost based.
- * @param conf job configuration
+ * @param job
+ * @param identifier
+ * @return
+ * @throws IOException
+ * @throws KeyGenException
*/
- public void selectIndex(Configuration conf) {
- // set the default index in configuration
+ public BlockMappingVO getBlockRowCount(JobContext job, AbsoluteTableIdentifier identifier)
+ throws IOException, KeyGenException {
+ TableDataMap blockletMap =
+ DataMapStoreManager.getInstance().getDataMap(identifier, "blocklet", DataMapType.BLOCKLET);
+ SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(identifier);
+ SegmentStatusManager.ValidAndInvalidSegmentsInfo validAndInvalidSegments =
+ new SegmentStatusManager(identifier).getValidAndInvalidSegments();
+ Map<String, Long> blockRowCountMapping =
+ new HashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ Map<String, Long> segmentAndBlockCountMapping =
+ new HashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ List<Blocklet> blocklets = blockletMap.prune(validAndInvalidSegments.getValidSegments(), null);
+ for (Blocklet blocklet : blocklets) {
+ String blockName = blocklet.getPath().toString();
+ blockName = CarbonTablePath.getCarbonDataFileName(blockName);
+ blockName = blockName + CarbonTablePath.getCarbonDataExtension();
+
+ long rowCount = blocklet.getDetailInfo().getRowCount();
+
+ String key = CarbonUpdateUtil.getSegmentBlockNameKey(blocklet.getSegmentId(), blockName);
+
+ // if block is invalid then dont add the count
+ SegmentUpdateDetails details = updateStatusManager.getDetailsForABlock(key);
+
+ if (null == details || !CarbonUpdateUtil.isBlockInvalid(details.getStatus())) {
+ Long blockCount = blockRowCountMapping.get(key);
+ if (blockCount == null) {
+ blockCount = 0L;
+ Long count = segmentAndBlockCountMapping.get(blocklet.getSegmentId());
+ if (count == null) {
+ count = 0L;
+ }
+ segmentAndBlockCountMapping.put(blocklet.getSegmentId(), count + 1);
+ }
+ blockCount += rowCount;
+ blockRowCountMapping.put(key, blockCount);
+ }
+ }
+ return new BlockMappingVO(blockRowCountMapping, segmentAndBlockCountMapping);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
index 8270304..8269757 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
@@ -34,7 +34,7 @@ import org.apache.carbondata.core.scan.model.CarbonQueryPlan;
import org.apache.carbondata.core.scan.model.QueryDimension;
import org.apache.carbondata.core.scan.model.QueryMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
-import org.apache.carbondata.hadoop.CarbonInputFormat;
+import org.apache.carbondata.hadoop.api.CarbonTableInputFormat;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
@@ -77,9 +77,10 @@ public class CarbonInputFormatUtil {
return plan;
}
- public static <V> CarbonInputFormat<V> createCarbonInputFormat(AbsoluteTableIdentifier identifier,
+ public static <V> CarbonTableInputFormat<V> createCarbonInputFormat(
+ AbsoluteTableIdentifier identifier,
Job job) throws IOException {
- CarbonInputFormat<V> carbonInputFormat = new CarbonInputFormat<>();
+ CarbonTableInputFormat<V> carbonInputFormat = new CarbonTableInputFormat<>();
FileInputFormat.addInputPath(job, new Path(identifier.getTablePath()));
return carbonInputFormat;
}
[09/15] carbondata git commit: [CARBONDATA-1301] change command to
update schema and data separately
Posted by ra...@apache.org.
[CARBONDATA-1301] change command to update schema and data separately
This closes #1160
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c7f0b108
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c7f0b108
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c7f0b108
Branch: refs/heads/datamap
Commit: c7f0b1081c6055970458712bf29d874fe95c7004
Parents: 2a9debf
Author: jackylk <ja...@huawei.com>
Authored: Wed Jul 12 00:40:44 2017 +0800
Committer: Ravindra Pesala <ra...@gmail.com>
Committed: Wed Jul 12 21:07:14 2017 +0530
----------------------------------------------------------------------
.../sql/execution/command/IUDCommands.scala | 146 +++----
.../execution/command/carbonTableSchema.scala | 414 ++++++++++---------
.../spark/sql/hive/CarbonFileMetastore.scala | 3 +-
.../spark/sql/hive/CarbonHiveMetaStore.scala | 11 -
4 files changed, 296 insertions(+), 278 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c7f0b108/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
index 2c1de52..d3a80d4 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
@@ -58,31 +58,35 @@ import org.apache.carbondata.spark.util.QueryPlanUtil
private[sql] case class ProjectForDeleteCommand(
plan: LogicalPlan,
identifier: Seq[String],
- timestamp: String) extends RunnableCommand {
+ timestamp: String) extends RunnableCommand with DataProcessCommand {
- val LOG = LogServiceFactory.getLogService(this.getClass.getName)
var horizontalCompactionFailed = false
override def run(sparkSession: SparkSession): Seq[Row] = {
+ processData(sparkSession)
+ }
+
+ override def processData(sparkSession: SparkSession): Seq[Row] = {
+ val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
val dataFrame = Dataset.ofRows(sparkSession, plan)
-// dataFrame.show(truncate = false)
-// dataFrame.collect().foreach(println)
+ // dataFrame.show(truncate = false)
+ // dataFrame.collect().foreach(println)
val dataRdd = dataFrame.rdd
val relation = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .lookupRelation(deleteExecution.getTableIdentifier(identifier))(sparkSession).
- asInstanceOf[CarbonRelation]
+ .lookupRelation(deleteExecution.getTableIdentifier(identifier))(sparkSession).
+ asInstanceOf[CarbonRelation]
val carbonTable = relation.tableMeta.carbonTable
val metadataLock = CarbonLockFactory
- .getCarbonLockObj(carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
- LockUsage.METADATA_LOCK)
+ .getCarbonLockObj(carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
+ LockUsage.METADATA_LOCK)
var lockStatus = false
try {
lockStatus = metadataLock.lockWithRetries()
- LOG.audit(s" Delete data request has been received " +
- s"for ${ relation.databaseName }.${ relation.tableName }.")
+ LOGGER.audit(s" Delete data request has been received " +
+ s"for ${ relation.databaseName }.${ relation.tableName }.")
if (lockStatus) {
- LOG.info("Successfully able to get the table metadata file lock")
+ LOGGER.info("Successfully able to get the table metadata file lock")
}
else {
throw new Exception("Table is locked for deletion. Please try after some time")
@@ -92,23 +96,23 @@ private[sql] case class ProjectForDeleteCommand(
carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier)
var executorErrors = new ExecutionErrors(FailureCauses.NONE, "")
- // handle the clean up of IUD.
- CarbonUpdateUtil.cleanUpDeltaFiles(carbonTable, false)
+ // handle the clean up of IUD.
+ CarbonUpdateUtil.cleanUpDeltaFiles(carbonTable, false)
- if (deleteExecution
- .deleteDeltaExecution(identifier, sparkSession, dataRdd, timestamp, relation,
- false, executorErrors)) {
- // call IUD Compaction.
- IUDCommon.tryHorizontalCompaction(sparkSession, relation, isUpdateOperation = false)
- }
+ if (deleteExecution
+ .deleteDeltaExecution(identifier, sparkSession, dataRdd, timestamp, relation,
+ false, executorErrors)) {
+ // call IUD Compaction.
+ IUDCommon.tryHorizontalCompaction(sparkSession, relation, isUpdateOperation = false)
+ }
} catch {
case e: HorizontalCompactionException =>
- LOG.error("Delete operation passed. Exception in Horizontal Compaction." +
- " Please check logs. " + e.getMessage)
- CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, e.compactionTimeStamp.toString)
+ LOGGER.error("Delete operation passed. Exception in Horizontal Compaction." +
+ " Please check logs. " + e.getMessage)
+ CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, e.compactionTimeStamp.toString)
case e: Exception =>
- LOG.error(e, "Exception in Delete data operation " + e.getMessage)
+ LOGGER.error(e, "Exception in Delete data operation " + e.getMessage)
// ****** start clean up.
// In case of failure , clean all related delete delta files
CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, timestamp)
@@ -130,47 +134,51 @@ private[sql] case class ProjectForDeleteCommand(
}
private[sql] case class ProjectForUpdateCommand(
- plan: LogicalPlan, tableIdentifier: Seq[String]) extends RunnableCommand {
- val LOGGER = LogServiceFactory.getLogService(ProjectForUpdateCommand.getClass.getName)
+ plan: LogicalPlan, tableIdentifier: Seq[String]) extends RunnableCommand
+ with DataProcessCommand {
override def run(sparkSession: SparkSession): Seq[Row] = {
+ processData(sparkSession)
+ }
+ override def processData(sparkSession: SparkSession): Seq[Row] = {
+ val LOGGER = LogServiceFactory.getLogService(ProjectForUpdateCommand.getClass.getName)
- // sqlContext.sparkContext.setLocalProperty(org.apache.spark.sql.execution.SQLExecution
+ // sqlContext.sparkContext.setLocalProperty(org.apache.spark.sql.execution.SQLExecution
// .EXECUTION_ID_KEY, null)
// DataFrame(sqlContext, plan).show(truncate = false)
// return Seq.empty
val res = plan find {
- case relation: LogicalRelation if (relation.relation
- .isInstanceOf[CarbonDatasourceHadoopRelation]) =>
+ case relation: LogicalRelation if relation.relation
+ .isInstanceOf[CarbonDatasourceHadoopRelation] =>
true
case _ => false
}
- if (!res.isDefined) {
+ if (res.isEmpty) {
return Seq.empty
}
val relation = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .lookupRelation(deleteExecution.getTableIdentifier(tableIdentifier))(sparkSession).
- asInstanceOf[CarbonRelation]
-// val relation = CarbonEnv.get.carbonMetastore
-// .lookupRelation1(deleteExecution.getTableIdentifier(tableIdentifier))(sqlContext).
-// asInstanceOf[CarbonRelation]
+ .lookupRelation(deleteExecution.getTableIdentifier(tableIdentifier))(sparkSession).
+ asInstanceOf[CarbonRelation]
+ // val relation = CarbonEnv.get.carbonMetastore
+ // .lookupRelation1(deleteExecution.getTableIdentifier(tableIdentifier))(sqlContext).
+ // asInstanceOf[CarbonRelation]
val carbonTable = relation.tableMeta.carbonTable
val metadataLock = CarbonLockFactory
- .getCarbonLockObj(carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
- LockUsage.METADATA_LOCK)
+ .getCarbonLockObj(carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
+ LockUsage.METADATA_LOCK)
var lockStatus = false
// get the current time stamp which should be same for delete and update.
val currentTime = CarbonUpdateUtil.readCurrentTime
-// var dataFrame: DataFrame = null
+ // var dataFrame: DataFrame = null
var dataSet: DataFrame = null
val isPersistEnabledUserValue = CarbonProperties.getInstance
- .getProperty(CarbonCommonConstants.isPersistEnabled,
- CarbonCommonConstants.defaultValueIsPersistEnabled)
- var isPersistEnabled = CarbonCommonConstants.defaultValueIsPersistEnabled.toBoolean
+ .getProperty(CarbonCommonConstants.isPersistEnabled,
+ CarbonCommonConstants.defaultValueIsPersistEnabled)
+ var isPersistEnabled = CarbonCommonConstants.defaultValueIsPersistEnabled.toBoolean
if (isPersistEnabledUserValue.equalsIgnoreCase("false")) {
isPersistEnabled = false
}
@@ -188,48 +196,46 @@ private[sql] case class ProjectForUpdateCommand(
val tablePath = CarbonStorePath.getCarbonTablePath(
carbonTable.getStorePath,
carbonTable.getAbsoluteTableIdentifier.getCarbonTableIdentifier)
- // Get RDD.
+ // Get RDD.
dataSet = if (isPersistEnabled) {
- Dataset.ofRows(sparkSession, plan).persist(StorageLevel.MEMORY_AND_DISK)
-// DataFrame(sqlContext, plan)
-// .persist(StorageLevel.MEMORY_AND_DISK)
- }
- else {
- Dataset.ofRows(sparkSession, plan)
-// DataFrame(sqlContext, plan)
- }
- var executionErrors = new ExecutionErrors(FailureCauses.NONE, "")
+ Dataset.ofRows(sparkSession, plan).persist(StorageLevel.MEMORY_AND_DISK)
+ // DataFrame(sqlContext, plan)
+ // .persist(StorageLevel.MEMORY_AND_DISK)
+ }
+ else {
+ Dataset.ofRows(sparkSession, plan)
+ // DataFrame(sqlContext, plan)
+ }
+ var executionErrors = new ExecutionErrors(FailureCauses.NONE, "")
- // handle the clean up of IUD.
- CarbonUpdateUtil.cleanUpDeltaFiles(carbonTable, false)
+ // handle the clean up of IUD.
+ CarbonUpdateUtil.cleanUpDeltaFiles(carbonTable, false)
- // do delete operation.
- deleteExecution.deleteDeltaExecution(tableIdentifier, sparkSession, dataSet.rdd,
- currentTime + "",
+ // do delete operation.
+ deleteExecution.deleteDeltaExecution(tableIdentifier, sparkSession, dataSet.rdd,
+ currentTime + "",
relation, isUpdateOperation = true, executionErrors)
- if(executionErrors.failureCauses != FailureCauses.NONE) {
- throw new Exception(executionErrors.errorMsg)
- }
-
- // do update operation.
- UpdateExecution.performUpdate(dataSet, tableIdentifier, plan,
- sparkSession, currentTime, executionErrors)
+ if(executionErrors.failureCauses != FailureCauses.NONE) {
+ throw new Exception(executionErrors.errorMsg)
+ }
- if(executionErrors.failureCauses != FailureCauses.NONE) {
- throw new Exception(executionErrors.errorMsg)
- }
+ // do update operation.
+ UpdateExecution.performUpdate(dataSet, tableIdentifier, plan,
+ sparkSession, currentTime, executionErrors)
- // Do IUD Compaction.
- IUDCommon.tryHorizontalCompaction(sparkSession, relation, isUpdateOperation = true)
- }
+ if(executionErrors.failureCauses != FailureCauses.NONE) {
+ throw new Exception(executionErrors.errorMsg)
+ }
- catch {
+ // Do IUD Compaction.
+ IUDCommon.tryHorizontalCompaction(sparkSession, relation, isUpdateOperation = true)
+ } catch {
case e: HorizontalCompactionException =>
LOGGER.error(
- "Update operation passed. Exception in Horizontal Compaction. Please check logs." + e)
+ "Update operation passed. Exception in Horizontal Compaction. Please check logs." + e)
// In case of failure , clean all related delta files
CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, e.compactionTimeStamp.toString)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c7f0b108/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 88e89ad..cc18fa3 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -72,18 +72,36 @@ object Checker {
}
/**
+ * Interface for command that modifies schema
+ */
+trait SchemaProcessCommand {
+ def processSchema(sparkSession: SparkSession): Seq[Row]
+}
+
+/**
+ * Interface for command that need to process data in file system
+ */
+trait DataProcessCommand {
+ def processData(sparkSession: SparkSession): Seq[Row]
+}
+
+/**
* Command for show table partitions Command
*
* @param tableIdentifier
*/
private[sql] case class ShowCarbonPartitionsCommand(
- tableIdentifier: TableIdentifier) extends RunnableCommand {
- val LOGGER = LogServiceFactory.getLogService(ShowCarbonPartitionsCommand.getClass.getName)
+ tableIdentifier: TableIdentifier) extends RunnableCommand with SchemaProcessCommand {
+
override val output = CommonUtil.partitionInfoOutput
override def run(sparkSession: SparkSession): Seq[Row] = {
+ processSchema(sparkSession)
+ }
+
+ override def processSchema(sparkSession: SparkSession): Seq[Row] = {
val relation = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .lookupRelation(tableIdentifier)(sparkSession).
- asInstanceOf[CarbonRelation]
+ .lookupRelation(tableIdentifier)(sparkSession).
+ asInstanceOf[CarbonRelation]
val carbonTable = relation.tableMeta.carbonTable
var tableName = carbonTable.getFactTableName
var partitionInfo = carbonTable.getPartitionInfo(
@@ -94,6 +112,7 @@ private[sql] case class ShowCarbonPartitionsCommand(
}
var partitionType = partitionInfo.getPartitionType
var columnName = partitionInfo.getColumnSchemaList.get(0).getColumnName
+ val LOGGER = LogServiceFactory.getLogService(ShowCarbonPartitionsCommand.getClass.getName)
LOGGER.info("partition column name:" + columnName)
CommonUtil.getPartitionInfo(columnName, partitionType, partitionInfo)
}
@@ -104,17 +123,22 @@ private[sql] case class ShowCarbonPartitionsCommand(
*
* @param alterTableModel
*/
-case class AlterTableCompaction(alterTableModel: AlterTableModel) extends RunnableCommand {
+case class AlterTableCompaction(alterTableModel: AlterTableModel) extends RunnableCommand
+ with DataProcessCommand {
val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
def run(sparkSession: SparkSession): Seq[Row] = {
+ processData(sparkSession)
+ }
+
+ override def processData(sparkSession: SparkSession): Seq[Row] = {
val tableName = alterTableModel.tableName.toLowerCase
val databaseName = alterTableModel.dbName.getOrElse(sparkSession.catalog.currentDatabase)
val relation =
CarbonEnv.getInstance(sparkSession).carbonMetastore
- .lookupRelation(Option(databaseName), tableName)(sparkSession)
- .asInstanceOf[CarbonRelation]
+ .lookupRelation(Option(databaseName), tableName)(sparkSession)
+ .asInstanceOf[CarbonRelation]
if (relation == null) {
sys.error(s"Table $databaseName.$tableName does not exist")
}
@@ -135,18 +159,18 @@ case class AlterTableCompaction(alterTableModel: AlterTableModel) extends Runnab
carbonLoadModel.setStorePath(relation.tableMeta.storePath)
var storeLocation = CarbonProperties.getInstance
- .getProperty(CarbonCommonConstants.STORE_LOCATION_TEMP_PATH,
- System.getProperty("java.io.tmpdir")
- )
+ .getProperty(CarbonCommonConstants.STORE_LOCATION_TEMP_PATH,
+ System.getProperty("java.io.tmpdir")
+ )
storeLocation = storeLocation + "/carbonstore/" + System.nanoTime()
try {
CarbonDataRDDFactory
- .alterTableForCompaction(sparkSession.sqlContext,
- alterTableModel,
- carbonLoadModel,
- relation.tableMeta.storePath,
- storeLocation
- )
+ .alterTableForCompaction(sparkSession.sqlContext,
+ alterTableModel,
+ carbonLoadModel,
+ relation.tableMeta.storePath,
+ storeLocation
+ )
} catch {
case e: Exception =>
if (null != e.getMessage) {
@@ -159,9 +183,19 @@ case class AlterTableCompaction(alterTableModel: AlterTableModel) extends Runnab
}
}
-case class CreateTable(cm: TableModel, createDSTable: Boolean = true) extends RunnableCommand {
+case class CreateTable(cm: TableModel, createDSTable: Boolean = true) extends RunnableCommand
+ with SchemaProcessCommand {
def run(sparkSession: SparkSession): Seq[Row] = {
+ processSchema(sparkSession)
+ }
+
+ def setV(ref: Any, name: String, value: Any): Unit = {
+ ref.getClass.getFields.find(_.getName == name).get
+ .set(ref, value.asInstanceOf[AnyRef])
+ }
+
+ override def processSchema(sparkSession: SparkSession): Seq[Row] = {
CarbonEnv.getInstance(sparkSession).carbonMetastore.checkSchemasModifiedTimeAndReloadTables()
val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
cm.databaseName = getDB.getDatabaseName(cm.databaseNameOp, sparkSession)
@@ -176,11 +210,11 @@ case class CreateTable(cm: TableModel, createDSTable: Boolean = true) extends Ru
}
if (sparkSession.sessionState.catalog.listTables(dbName)
- .exists(_.table.equalsIgnoreCase(tbName))) {
+ .exists(_.table.equalsIgnoreCase(tbName))) {
if (!cm.ifNotExistsSet) {
LOGGER.audit(
s"Table creation with Database name [$dbName] and Table name [$tbName] failed. " +
- s"Table [$tbName] already exists under database [$dbName]")
+ s"Table [$tbName] already exists under database [$dbName]")
sys.error(s"Table [$tbName] already exists under database [$dbName]")
}
} else {
@@ -196,48 +230,43 @@ case class CreateTable(cm: TableModel, createDSTable: Boolean = true) extends Ru
sparkSession.sql(
s"""CREATE TABLE $dbName.$tbName
- |(${ fields.map(f => f.rawSchema).mkString(",") })
- |USING org.apache.spark.sql.CarbonSource""".stripMargin +
- s""" OPTIONS (tableName "$tbName", dbName "$dbName", tablePath """.stripMargin +
- s""""$tablePath" $carbonSchemaString) """)
+ |(${ fields.map(f => f.rawSchema).mkString(",") })
+ |USING org.apache.spark.sql.CarbonSource""".stripMargin +
+ s""" OPTIONS (tableName "$tbName", dbName "$dbName", tablePath """.stripMargin +
+ s""""$tablePath" $carbonSchemaString) """)
} catch {
case e: Exception =>
val identifier: TableIdentifier = TableIdentifier(tbName, Some(dbName))
// call the drop table to delete the created table.
CarbonEnv.getInstance(sparkSession).carbonMetastore
- .dropTable(catalog.storePath, identifier)(sparkSession)
+ .dropTable(catalog.storePath, identifier)(sparkSession)
LOGGER.audit(s"Table creation with Database name [$dbName] " +
- s"and Table name [$tbName] failed")
+ s"and Table name [$tbName] failed")
throw e
}
}
LOGGER.audit(s"Table created with Database name [$dbName] and Table name [$tbName]")
}
-
Seq.empty
}
-
- def setV(ref: Any, name: String, value: Any): Unit = {
- ref.getClass.getFields.find(_.getName == name).get
- .set(ref, value.asInstanceOf[AnyRef])
- }
}
case class DeleteLoadsById(
loadids: Seq[String],
databaseNameOp: Option[String],
- tableName: String) extends RunnableCommand {
-
- val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
+ tableName: String) extends RunnableCommand with DataProcessCommand {
def run(sparkSession: SparkSession): Seq[Row] = {
+ processData(sparkSession)
+ }
+ override def processData(sparkSession: SparkSession): Seq[Row] = {
Checker.validateTableExists(databaseNameOp, tableName, sparkSession)
val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore.
- lookupRelation(databaseNameOp, tableName)(sparkSession).asInstanceOf[CarbonRelation].
- tableMeta.carbonTable
+ lookupRelation(databaseNameOp, tableName)(sparkSession).asInstanceOf[CarbonRelation].
+ tableMeta.carbonTable
CarbonStore.deleteLoadById(
loadids,
getDB.getDatabaseName(databaseNameOp, sparkSession),
@@ -245,24 +274,24 @@ case class DeleteLoadsById(
carbonTable
)
Seq.empty
-
}
-
}
case class DeleteLoadsByLoadDate(
databaseNameOp: Option[String],
tableName: String,
dateField: String,
- loadDate: String) extends RunnableCommand {
-
- val LOGGER = LogServiceFactory.getLogService("org.apache.spark.sql.TableModel.tableSchema")
+ loadDate: String) extends RunnableCommand with DataProcessCommand {
def run(sparkSession: SparkSession): Seq[Row] = {
+ processData(sparkSession)
+ }
+
+ override def processData(sparkSession: SparkSession): Seq[Row] = {
Checker.validateTableExists(databaseNameOp, tableName, sparkSession)
val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore.
- lookupRelation(databaseNameOp, tableName)(sparkSession).asInstanceOf[CarbonRelation].
- tableMeta.carbonTable
+ lookupRelation(databaseNameOp, tableName)(sparkSession).asInstanceOf[CarbonRelation].
+ tableMeta.carbonTable
CarbonStore.deleteLoadByDate(
loadDate,
getDB.getDatabaseName(databaseNameOp, sparkSession),
@@ -271,7 +300,6 @@ case class DeleteLoadsByLoadDate(
)
Seq.empty
}
-
}
object LoadTable {
@@ -318,10 +346,13 @@ object LoadTable {
}
case class LoadTableByInsert(relation: CarbonDatasourceHadoopRelation, child: LogicalPlan)
- extends RunnableCommand {
- val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
+ extends RunnableCommand with DataProcessCommand {
def run(sparkSession: SparkSession): Seq[Row] = {
+ processData(sparkSession)
+ }
+
+ override def processData(sparkSession: SparkSession): Seq[Row] = {
val df = Dataset.ofRows(sparkSession, child)
val header = relation.tableSchema.get.fields.map(_.name).mkString(",")
val load = LoadTable(
@@ -335,7 +366,7 @@ case class LoadTableByInsert(relation: CarbonDatasourceHadoopRelation, child: Lo
Some(df)).run(sparkSession)
// updating relation metadata. This is in case of auto detect high cardinality
relation.carbonRelation.metaData =
- CarbonSparkUtil.createSparkMeta(relation.carbonRelation.tableMeta.carbonTable)
+ CarbonSparkUtil.createSparkMeta(relation.carbonRelation.tableMeta.carbonTable)
load
}
}
@@ -349,12 +380,10 @@ case class LoadTable(
isOverwriteExist: Boolean = false,
var inputSqlString: String = null,
dataFrame: Option[DataFrame] = None,
- updateModel: Option[UpdateTableModel] = None) extends RunnableCommand {
-
- val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
+ updateModel: Option[UpdateTableModel] = None) extends RunnableCommand with DataProcessCommand {
- private def getFinalOptions(carbonProperty: CarbonProperties): scala.collection
- .mutable.Map[String, String] = {
+ private def getFinalOptions(carbonProperty: CarbonProperties):
+ scala.collection.mutable.Map[String, String] = {
var optionsFinal = scala.collection.mutable.Map[String, String]()
optionsFinal.put("delimiter", options.getOrElse("delimiter", ","))
optionsFinal.put("quotechar", options.getOrElse("quotechar", "\""))
@@ -415,14 +444,15 @@ case class LoadTable(
case "false" =>
// when single_pass = false and if either alldictionarypath
// or columnDict is configured the do not allow load
- if (StringUtils.isNotEmpty(optionsFinal.get("all_dictionary_path").get) ||
- StringUtils.isNotEmpty(optionsFinal.get("columndict").get)) {
+ if (StringUtils.isNotEmpty(optionsFinal("all_dictionary_path")) ||
+ StringUtils.isNotEmpty(optionsFinal("columndict"))) {
throw new MalformedCarbonCommandException(
"Can not use all_dictionary_path or columndict without single_pass.")
} else {
false
}
case illegal =>
+ val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
LOGGER.error(s"Can't use single_pass, because illegal syntax found: [" + illegal + "] " +
"Please set it as 'true' or 'false'")
false
@@ -440,7 +470,12 @@ case class LoadTable(
}
def run(sparkSession: SparkSession): Seq[Row] = {
- if (dataFrame.isDefined && !updateModel.isDefined) {
+ processData(sparkSession)
+ }
+
+ override def processData(sparkSession: SparkSession): Seq[Row] = {
+ val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
+ if (dataFrame.isDefined && updateModel.isEmpty) {
val rdd = dataFrame.get.rdd
if (rdd.partitions == null || rdd.partitions.length == 0) {
LOGGER.warn("DataLoading finished. No data was loaded.")
@@ -452,8 +487,9 @@ case class LoadTable(
if (isOverwriteExist) {
sys.error(s"Overwrite is not supported for carbon table with $dbName.$tableName")
}
+
val relation = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .lookupRelation(Option(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation]
+ .lookupRelation(Option(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation]
if (relation == null) {
sys.error(s"Table $dbName.$tableName does not exist")
}
@@ -467,10 +503,10 @@ case class LoadTable(
carbonProperty.addProperty("zookeeper.enable.lock", "false")
val optionsFinal = getFinalOptions(carbonProperty)
val carbonLock = CarbonLockFactory
- .getCarbonLockObj(relation.tableMeta.carbonTable.getAbsoluteTableIdentifier
- .getCarbonTableIdentifier,
- LockUsage.METADATA_LOCK
- )
+ .getCarbonLockObj(relation.tableMeta.carbonTable.getAbsoluteTableIdentifier
+ .getCarbonTableIdentifier,
+ LockUsage.METADATA_LOCK
+ )
try {
// take lock only in case of normal data load.
if (updateModel.isEmpty) {
@@ -499,21 +535,21 @@ case class LoadTable(
carbonLoadModel.setCarbonDataLoadSchema(dataLoadSchema)
val partitionLocation = relation.tableMeta.storePath + "/partition/" +
- relation.tableMeta.carbonTableIdentifier.getDatabaseName + "/" +
- relation.tableMeta.carbonTableIdentifier.getTableName + "/"
+ relation.tableMeta.carbonTableIdentifier.getDatabaseName + "/" +
+ relation.tableMeta.carbonTableIdentifier.getTableName + "/"
val columnar = sparkSession.conf.get("carbon.is.columnar.storage", "true").toBoolean
- val sort_scope = optionsFinal.get("sort_scope").get
- val single_pass = optionsFinal.get("single_pass").get
- val bad_records_logger_enable = optionsFinal.get("bad_records_logger_enable").get
- val bad_records_action = optionsFinal.get("bad_records_action").get
- val bad_record_path = optionsFinal.get("bad_record_path").get
- val global_sort_partitions = optionsFinal.get("global_sort_partitions").get
- val dateFormat = optionsFinal.get("dateformat").get
- val delimeter = optionsFinal.get("delimiter").get
- val complex_delimeter_level1 = optionsFinal.get("complex_delimiter_level_1").get
- val complex_delimeter_level2 = optionsFinal.get("complex_delimiter_level_2").get
- val all_dictionary_path = optionsFinal.get("all_dictionary_path").get
- val column_dict = optionsFinal.get("columndict").get
+ val sort_scope = optionsFinal("sort_scope")
+ val single_pass = optionsFinal("single_pass")
+ val bad_records_logger_enable = optionsFinal("bad_records_logger_enable")
+ val bad_records_action = optionsFinal("bad_records_action")
+ val bad_record_path = optionsFinal("bad_record_path")
+ val global_sort_partitions = optionsFinal("global_sort_partitions")
+ val dateFormat = optionsFinal("dateformat")
+ val delimeter = optionsFinal("delimiter")
+ val complex_delimeter_level1 = optionsFinal("complex_delimiter_level_1")
+ val complex_delimeter_level2 = optionsFinal("complex_delimiter_level_2")
+ val all_dictionary_path = optionsFinal("all_dictionary_path")
+ val column_dict = optionsFinal("columndict")
if (sort_scope.equals("GLOBAL_SORT") &&
single_pass.equals("TRUE")) {
sys.error("Global_Sort can't be used with single_pass flow")
@@ -531,13 +567,13 @@ case class LoadTable(
carbonLoadModel.setBadRecordsLocation(bad_record_path)
ValidateUtil.validateGlobalSortPartitions(global_sort_partitions)
- carbonLoadModel.setEscapeChar(checkDefaultValue(optionsFinal.get("escapechar").get, "\\"))
- carbonLoadModel.setQuoteChar(checkDefaultValue(optionsFinal.get("quotechar").get, "\""))
- carbonLoadModel.setCommentChar(checkDefaultValue(optionsFinal.get("commentchar").get, "#"))
+ carbonLoadModel.setEscapeChar(checkDefaultValue(optionsFinal("escapechar"), "\\"))
+ carbonLoadModel.setQuoteChar(checkDefaultValue(optionsFinal("quotechar"), "\""))
+ carbonLoadModel.setCommentChar(checkDefaultValue(optionsFinal("commentchar"), "#"))
// if there isn't file header in csv file and load sql doesn't provide FILEHEADER option,
// we should use table schema to generate file header.
- var fileHeader = optionsFinal.get("fileheader").get
+ var fileHeader = optionsFinal("fileheader")
val headerOption = options.get("header")
if (headerOption.isDefined) {
// whether the csv file has file header
@@ -559,7 +595,7 @@ case class LoadTable(
// generate file header
if (fileHeader.isEmpty) {
fileHeader = table.getCreateOrderColumn(table.getFactTableName)
- .asScala.map(_.getColName).mkString(",")
+ .asScala.map(_.getColName).mkString(",")
}
}
}
@@ -572,21 +608,21 @@ case class LoadTable(
CarbonCommonConstants.CARBON_DATE_FORMAT,
CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT))
carbonLoadModel
- .setSerializationNullFormat(
- TableOptionConstant.SERIALIZATION_NULL_FORMAT.getName + "," +
- optionsFinal.get("serialization_null_format").get)
+ .setSerializationNullFormat(
+ TableOptionConstant.SERIALIZATION_NULL_FORMAT.getName + "," +
+ optionsFinal("serialization_null_format"))
carbonLoadModel
- .setBadRecordsLoggerEnable(
- TableOptionConstant.BAD_RECORDS_LOGGER_ENABLE.getName + "," + bad_records_logger_enable)
+ .setBadRecordsLoggerEnable(
+ TableOptionConstant.BAD_RECORDS_LOGGER_ENABLE.getName + "," + bad_records_logger_enable)
carbonLoadModel
- .setBadRecordsAction(
- TableOptionConstant.BAD_RECORDS_ACTION.getName + "," + bad_records_action)
+ .setBadRecordsAction(
+ TableOptionConstant.BAD_RECORDS_ACTION.getName + "," + bad_records_action)
carbonLoadModel
- .setIsEmptyDataBadRecord(
- DataLoadProcessorConstants.IS_EMPTY_DATA_BAD_RECORD + "," +
- optionsFinal.get("is_empty_data_bad_record").get)
+ .setIsEmptyDataBadRecord(
+ DataLoadProcessorConstants.IS_EMPTY_DATA_BAD_RECORD + "," +
+ optionsFinal("is_empty_data_bad_record"))
carbonLoadModel.setSortScope(sort_scope)
- carbonLoadModel.setBatchSortSizeInMb(optionsFinal.get("batch_sort_size_inmb").get)
+ carbonLoadModel.setBatchSortSizeInMb(optionsFinal("batch_sort_size_inmb"))
carbonLoadModel.setGlobalSortPartitions(global_sort_partitions)
carbonLoadModel.setUseOnePass(single_pass.toBoolean)
if (delimeter.equalsIgnoreCase(complex_delimeter_level1) ||
@@ -614,7 +650,7 @@ case class LoadTable(
carbonLoadModel.setDirectLoad(true)
carbonLoadModel.setCsvHeaderColumns(CommonUtil.getCsvHeaderColumns(carbonLoadModel))
val validatedMaxColumns = CommonUtil.validateMaxColumns(carbonLoadModel.getCsvHeaderColumns,
- optionsFinal.get("maxcolumns").get)
+ optionsFinal("maxcolumns"))
carbonLoadModel.setMaxColumns(validatedMaxColumns.toString)
GlobalDictionaryUtil.updateTableMetadataFunc = LoadTable.updateTableMetadata
val storePath = relation.tableMeta.storePath
@@ -629,7 +665,7 @@ case class LoadTable(
}
// Create table and metadata folders if not exist
val carbonTablePath = CarbonStorePath
- .getCarbonTablePath(storePath, table.getCarbonTableIdentifier)
+ .getCarbonTablePath(storePath, table.getCarbonTableIdentifier)
val metadataDirectoryPath = carbonTablePath.getMetadataDirectoryPath
val fileType = FileFactory.getFileType(metadataDirectoryPath)
if (!FileFactory.isFileExist(metadataDirectoryPath, fileType)) {
@@ -638,9 +674,9 @@ case class LoadTable(
if (carbonLoadModel.getUseOnePass) {
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
val carbonTableIdentifier = carbonTable.getAbsoluteTableIdentifier
- .getCarbonTableIdentifier
+ .getCarbonTableIdentifier
val carbonTablePath = CarbonStorePath
- .getCarbonTablePath(storePath, carbonTableIdentifier)
+ .getCarbonTablePath(storePath, carbonTableIdentifier)
val dictFolderPath = carbonTablePath.getMetadataDirectoryPath
val dimensions = carbonTable.getDimensionByTableName(
carbonTable.getFactTableName).asScala.toArray
@@ -649,37 +685,37 @@ case class LoadTable(
carbonLoadModel.initPredefDictMap()
// generate predefined dictionary
GlobalDictionaryUtil
- .generatePredefinedColDictionary(colDictFilePath, carbonTableIdentifier,
- dimensions, carbonLoadModel, sparkSession.sqlContext, storePath, dictFolderPath)
+ .generatePredefinedColDictionary(colDictFilePath, carbonTableIdentifier,
+ dimensions, carbonLoadModel, sparkSession.sqlContext, storePath, dictFolderPath)
}
if (!StringUtils.isEmpty(all_dictionary_path)) {
carbonLoadModel.initPredefDictMap()
GlobalDictionaryUtil
- .generateDictionaryFromDictionaryFiles(sparkSession.sqlContext,
- carbonLoadModel,
- storePath,
- carbonTableIdentifier,
- dictFolderPath,
- dimensions,
- all_dictionary_path)
+ .generateDictionaryFromDictionaryFiles(sparkSession.sqlContext,
+ carbonLoadModel,
+ storePath,
+ carbonTableIdentifier,
+ dictFolderPath,
+ dimensions,
+ all_dictionary_path)
}
// dictionaryServerClient dictionary generator
val dictionaryServerPort = carbonProperty
- .getProperty(CarbonCommonConstants.DICTIONARY_SERVER_PORT,
- CarbonCommonConstants.DICTIONARY_SERVER_PORT_DEFAULT)
+ .getProperty(CarbonCommonConstants.DICTIONARY_SERVER_PORT,
+ CarbonCommonConstants.DICTIONARY_SERVER_PORT_DEFAULT)
val sparkDriverHost = sparkSession.sqlContext.sparkContext.
- getConf.get("spark.driver.host")
+ getConf.get("spark.driver.host")
carbonLoadModel.setDictionaryServerHost(sparkDriverHost)
// start dictionary server when use one pass load and dimension with DICTIONARY
// encoding is present.
val allDimensions = table.getAllDimensions.asScala.toList
val createDictionary = allDimensions.exists {
carbonDimension => carbonDimension.hasEncoding(Encoding.DICTIONARY) &&
- !carbonDimension.hasEncoding(Encoding.DIRECT_DICTIONARY)
+ !carbonDimension.hasEncoding(Encoding.DIRECT_DICTIONARY)
}
val server: Option[DictionaryServer] = if (createDictionary) {
val dictionaryServer = DictionaryServer
- .getInstance(dictionaryServerPort.toInt)
+ .getInstance(dictionaryServerPort.toInt)
carbonLoadModel.setDictionaryServerPort(dictionaryServer.getPort)
sparkSession.sparkContext.addSparkListener(new SparkListener() {
override def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd) {
@@ -707,22 +743,13 @@ case class LoadTable(
CarbonUpdateUtil.getRequiredFieldFromTID(tupleId, TupleIdEnum.SEGMENT_ID))
// getting all fields except tupleId field as it is not required in the value
var otherFields = fields.toSeq
- .filter(field => !field.name
- .equalsIgnoreCase(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID))
- .map(field => {
- if (field.name.endsWith(CarbonCommonConstants.UPDATED_COL_EXTENSION) && false) {
- new Column(field.name
- .substring(0,
- field.name.lastIndexOf(CarbonCommonConstants.UPDATED_COL_EXTENSION)))
- } else {
-
- new Column(field.name)
- }
- })
+ .filter(field => !field.name
+ .equalsIgnoreCase(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID))
+ .map(field => new Column(field.name))
// extract tupleId field which will be used as a key
val segIdColumn = getSegIdUDF(new Column(UnresolvedAttribute
- .quotedString(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID))).as("segId")
+ .quotedString(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID))).as("segId")
// use dataFrameWithoutTupleId as dictionaryDataFrame
val dataFrameWithoutTupleId = dataFrame.get.select(otherFields: _*)
otherFields = otherFields :+ segIdColumn
@@ -733,12 +760,11 @@ case class LoadTable(
(dataFrame, dataFrame)
}
- GlobalDictionaryUtil
- .generateGlobalDictionary(
- sparkSession.sqlContext,
- carbonLoadModel,
- relation.tableMeta.storePath,
- dictionaryDataFrame)
+ GlobalDictionaryUtil.generateGlobalDictionary(
+ sparkSession.sqlContext,
+ carbonLoadModel,
+ relation.tableMeta.storePath,
+ dictionaryDataFrame)
CarbonDataRDDFactory.loadCarbonData(sparkSession.sqlContext,
carbonLoadModel,
relation.tableMeta.storePath,
@@ -759,14 +785,14 @@ case class LoadTable(
val fileType = FileFactory.getFileType(partitionLocation)
if (FileFactory.isFileExist(partitionLocation, fileType)) {
val file = FileFactory
- .getCarbonFile(partitionLocation, fileType)
+ .getCarbonFile(partitionLocation, fileType)
CarbonUtil.deleteFoldersAndFiles(file)
}
} catch {
case ex: Exception =>
LOGGER.error(ex)
LOGGER.audit(s"Dataload failure for $dbName.$tableName. " +
- "Problem deleting the partition folder")
+ "Problem deleting the partition folder")
throw ex
}
@@ -791,40 +817,19 @@ case class LoadTable(
}
}
-private[sql] case class DeleteLoadByDate(
+case class CleanFiles(
databaseNameOp: Option[String],
- tableName: String,
- dateField: String,
- loadDate: String) {
-
- val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
+ tableName: String) extends RunnableCommand with DataProcessCommand {
def run(sparkSession: SparkSession): Seq[Row] = {
- Checker.validateTableExists(databaseNameOp, tableName, sparkSession)
- val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore.
- lookupRelation(databaseNameOp, tableName)(sparkSession).asInstanceOf[CarbonRelation].
- tableMeta.carbonTable
- CarbonStore.deleteLoadByDate(
- loadDate,
- getDB.getDatabaseName(databaseNameOp, sparkSession),
- tableName,
- carbonTable
- )
- Seq.empty
+ processData(sparkSession)
}
-}
-
-case class CleanFiles(
- databaseNameOp: Option[String],
- tableName: String) extends RunnableCommand {
-
- val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
- def run(sparkSession: SparkSession): Seq[Row] = {
+ override def processData(sparkSession: SparkSession): Seq[Row] = {
Checker.validateTableExists(databaseNameOp, tableName, sparkSession)
val catalog = CarbonEnv.getInstance(sparkSession).carbonMetastore
val relation = catalog
- .lookupRelation(databaseNameOp, tableName)(sparkSession).asInstanceOf[CarbonRelation]
+ .lookupRelation(databaseNameOp, tableName)(sparkSession).asInstanceOf[CarbonRelation]
val carbonTable = relation.tableMeta.carbonTable
CarbonStore.cleanFiles(
getDB.getDatabaseName(databaseNameOp, sparkSession),
@@ -840,13 +845,17 @@ case class ShowLoads(
databaseNameOp: Option[String],
tableName: String,
limit: Option[String],
- override val output: Seq[Attribute]) extends RunnableCommand {
+ override val output: Seq[Attribute]) extends RunnableCommand with DataProcessCommand {
def run(sparkSession: SparkSession): Seq[Row] = {
+ processData(sparkSession)
+ }
+
+ override def processData(sparkSession: SparkSession): Seq[Row] = {
Checker.validateTableExists(databaseNameOp, tableName, sparkSession)
val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore.
- lookupRelation(databaseNameOp, tableName)(sparkSession).asInstanceOf[CarbonRelation].
- tableMeta.carbonTable
+ lookupRelation(databaseNameOp, tableName)(sparkSession).asInstanceOf[CarbonRelation].
+ tableMeta.carbonTable
CarbonStore.showSegments(
getDB.getDatabaseName(databaseNameOp, sparkSession),
tableName,
@@ -859,16 +868,20 @@ case class ShowLoads(
case class CarbonDropTableCommand(ifExistsSet: Boolean,
databaseNameOp: Option[String],
tableName: String)
- extends RunnableCommand {
+ extends RunnableCommand with SchemaProcessCommand with DataProcessCommand {
def run(sparkSession: SparkSession): Seq[Row] = {
+ processSchema(sparkSession)
+ processData(sparkSession)
+ }
+
+ override def processSchema(sparkSession: SparkSession): Seq[Row] = {
val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
val dbName = getDB.getDatabaseName(databaseNameOp, sparkSession)
val identifier = TableIdentifier(tableName, Option(dbName))
val carbonTableIdentifier = new CarbonTableIdentifier(dbName, tableName, "")
val locksToBeAcquired = List(LockUsage.METADATA_LOCK, LockUsage.DROP_TABLE_LOCK)
val catalog = CarbonEnv.getInstance(sparkSession).carbonMetastore
- val storePath = catalog.storePath
catalog.checkSchemasModifiedTimeAndReloadTables()
val carbonLocks: scala.collection.mutable.ListBuffer[ICarbonLock] = ListBuffer()
try {
@@ -878,7 +891,7 @@ case class CarbonDropTableCommand(ifExistsSet: Boolean,
LOGGER.audit(s"Deleting table [$tableName] under database [$dbName]")
CarbonEnv.getInstance(sparkSession).carbonMetastore
- .dropTable(storePath, identifier)(sparkSession)
+ .dropTable(catalog.storePath, identifier)(sparkSession)
LOGGER.audit(s"Deleted table [$tableName] under database [$dbName]")
} catch {
case ex: Exception =>
@@ -889,29 +902,57 @@ case class CarbonDropTableCommand(ifExistsSet: Boolean,
if (unlocked) {
logInfo("Table MetaData Unlocked Successfully")
}
- // deleting any remaining files.
- val metadataFilePath = CarbonStorePath
- .getCarbonTablePath(storePath, carbonTableIdentifier).getMetadataDirectoryPath
- val fileType = FileFactory.getFileType(metadataFilePath)
- if (FileFactory.isFileExist(metadataFilePath, fileType)) {
- val file = FileFactory.getCarbonFile(metadataFilePath, fileType)
- CarbonUtil.deleteFoldersAndFiles(file.getParentFile)
- }
}
}
Seq.empty
}
+
+ override def processData(sparkSession: SparkSession): Seq[Row] = {
+ // delete the table folder
+ val dbName = getDB.getDatabaseName(databaseNameOp, sparkSession)
+ val carbonTableIdentifier = new CarbonTableIdentifier(dbName, tableName, "")
+ val catalog = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val metadataFilePath = CarbonStorePath
+ .getCarbonTablePath(catalog.storePath, carbonTableIdentifier).getMetadataDirectoryPath
+ val fileType = FileFactory.getFileType(metadataFilePath)
+ if (FileFactory.isFileExist(metadataFilePath, fileType)) {
+ val file = FileFactory.getCarbonFile(metadataFilePath, fileType)
+ CarbonUtil.deleteFoldersAndFilesSilent(file.getParentFile)
+ }
+ Seq.empty
+ }
}
private[sql] case class DescribeCommandFormatted(
child: SparkPlan,
override val output: Seq[Attribute],
tblIdentifier: TableIdentifier)
- extends RunnableCommand {
+ extends RunnableCommand with SchemaProcessCommand {
override def run(sparkSession: SparkSession): Seq[Row] = {
+ processSchema(sparkSession)
+ }
+
+ private def getColumnGroups(dimensions: List[CarbonDimension]): Seq[(String, String, String)] = {
+ var results: Seq[(String, String, String)] =
+ Seq(("", "", ""), ("##Column Group Information", "", ""))
+ val groupedDimensions = dimensions.groupBy(x => x.columnGroupId()).filter {
+ case (groupId, _) => groupId != -1
+ }.toSeq.sortBy(_._1)
+ val groups = groupedDimensions.map(colGroups => {
+ colGroups._2.map(dim => dim.getColName).mkString(", ")
+ })
+ var index = 1
+ groups.foreach { x =>
+ results = results :+ (s"Column Group $index", x, "")
+ index = index + 1
+ }
+ results
+ }
+
+ override def processSchema(sparkSession: SparkSession): Seq[Row] = {
val relation = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .lookupRelation(tblIdentifier)(sparkSession).asInstanceOf[CarbonRelation]
+ .lookupRelation(tblIdentifier)(sparkSession).asInstanceOf[CarbonRelation]
val mapper = new ObjectMapper()
val colProps = StringBuilder.newBuilder
val dims = relation.metaData.dims.map(x => x.toLowerCase)
@@ -922,8 +963,8 @@ private[sql] case class DescribeCommandFormatted(
relation.tableMeta.carbonTableIdentifier.getTableName, fieldName)
if (null != dimension.getColumnProperties && !dimension.getColumnProperties.isEmpty) {
colProps.append(fieldName).append(".")
- .append(mapper.writeValueAsString(dimension.getColumnProperties))
- .append(",")
+ .append(mapper.writeValueAsString(dimension.getColumnProperties))
+ .append(",")
}
if (dimension.hasEncoding(Encoding.DICTIONARY) &&
!dimension.hasEncoding(Encoding.DIRECT_DICTIONARY)) {
@@ -950,7 +991,7 @@ private[sql] case class DescribeCommandFormatted(
}
results ++= Seq(("", "", ""), ("##Detailed Table Information", "", ""))
results ++= Seq(("Database Name: ", relation.tableMeta.carbonTableIdentifier
- .getDatabaseName, "")
+ .getDatabaseName, "")
)
results ++= Seq(("Table Name: ", relation.tableMeta.carbonTableIdentifier.getTableName, ""))
results ++= Seq(("CARBON Store Path: ", relation.tableMeta.storePath, ""))
@@ -964,34 +1005,17 @@ private[sql] case class DescribeCommandFormatted(
}
results ++= Seq(("SORT_COLUMNS", relation.metaData.carbonTable.getSortColumns(
relation.tableMeta.carbonTableIdentifier.getTableName).asScala
- .map(column => column).mkString(","), ""))
+ .map(column => column).mkString(","), ""))
val dimension = carbonTable
- .getDimensionByTableName(relation.tableMeta.carbonTableIdentifier.getTableName)
+ .getDimensionByTableName(relation.tableMeta.carbonTableIdentifier.getTableName)
results ++= getColumnGroups(dimension.asScala.toList)
if (carbonTable.getPartitionInfo(carbonTable.getFactTableName) != null) {
results ++=
- Seq(("Partition Columns: ", carbonTable.getPartitionInfo(carbonTable.getFactTableName)
- .getColumnSchemaList.asScala.map(_.getColumnName).mkString(","), ""))
+ Seq(("Partition Columns: ", carbonTable.getPartitionInfo(carbonTable.getFactTableName)
+ .getColumnSchemaList.asScala.map(_.getColumnName).mkString(","), ""))
}
results.map { case (name, dataType, comment) =>
Row(f"$name%-36s", f"$dataType%-80s", f"$comment%-72s")
}
}
-
- private def getColumnGroups(dimensions: List[CarbonDimension]): Seq[(String, String, String)] = {
- var results: Seq[(String, String, String)] =
- Seq(("", "", ""), ("##Column Group Information", "", ""))
- val groupedDimensions = dimensions.groupBy(x => x.columnGroupId()).filter {
- case (groupId, _) => groupId != -1
- }.toSeq.sortBy(_._1)
- val groups = groupedDimensions.map(colGroups => {
- colGroups._2.map(dim => dim.getColName).mkString(", ")
- })
- var index = 1
- groups.foreach { x =>
- results = results :+ (s"Column Group $index", x, "")
- index = index + 1
- }
- results
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c7f0b108/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
index 048681c..549841b 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
@@ -430,8 +430,7 @@ class CarbonFileMetastore(conf: RuntimeConfig, val storePath: String) extends Ca
// while drop we should refresh the schema modified time so that if any thing has changed
// in the other beeline need to update.
checkSchemasModifiedTimeAndReloadTables
- val file = FileFactory.getCarbonFile(metadataFilePath, fileType)
- CarbonUtil.deleteFoldersAndFilesSilent(file.getParentFile)
+
val metadataToBeRemoved: Option[TableMeta] = getTableFromMetadata(dbName,
tableIdentifier.table)
metadataToBeRemoved match {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c7f0b108/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
index 03d0bde..a8f92ce 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
@@ -163,22 +163,11 @@ class CarbonHiveMetaStore(conf: RuntimeConfig, storePath: String)
(sparkSession: SparkSession): Unit = {
val dbName = tableIdentifier.database.get
val tableName = tableIdentifier.table
-
- val metadataFilePath = CarbonStorePath.getCarbonTablePath(tableStorePath,
- new CarbonTableIdentifier(dbName, tableName, "")).getMetadataDirectoryPath
val carbonTable = CarbonMetadata.getInstance.getCarbonTable(dbName + "_" + tableName)
if (null != carbonTable) {
// clear driver B-tree and dictionary cache
ManageDictionaryAndBTree.clearBTreeAndDictionaryLRUCache(carbonTable)
}
- val fileType = FileFactory.getFileType(metadataFilePath)
-
- if (FileFactory.isFileExist(metadataFilePath, fileType)) {
- // while drop we should refresh the schema modified time so that if any thing has changed
- // in the other beeline need to update.
- val file = FileFactory.getCarbonFile(metadataFilePath, fileType)
- CarbonUtil.deleteFoldersAndFilesSilent(file.getParentFile)
- }
CarbonHiveMetadataUtil.invalidateAndDropTable(dbName, tableName, sparkSession)
// discard cached table info in cachedDataSourceTables
sparkSession.sessionState.catalog.refreshTable(tableIdentifier)
[15/15] carbondata git commit: Rebased with master
Posted by ra...@apache.org.
Rebased with master
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f4ab1ff6
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f4ab1ff6
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f4ab1ff6
Branch: refs/heads/datamap
Commit: f4ab1ff69d51709b250efbd8bc538f6231994f54
Parents: 2015a3e
Author: Ravindra Pesala <ra...@gmail.com>
Authored: Thu Jul 13 14:18:02 2017 +0530
Committer: Ravindra Pesala <ra...@gmail.com>
Committed: Mon Jul 17 07:26:27 2017 +0530
----------------------------------------------------------------------
.../core/indexstore/UnsafeMemoryDMStore.java | 27 +++----
.../blockletindex/BlockletDataMap.java | 11 +--
.../core/memory/UnsafeMemoryManager.java | 16 ++--
.../hadoop/api/CarbonTableInputFormat.java | 81 ++++++++++----------
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 2 +-
5 files changed, 67 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4ab1ff6/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
index 8246f99..737586e 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
@@ -19,9 +19,9 @@ package org.apache.carbondata.core.indexstore;
import org.apache.carbondata.core.indexstore.row.DataMapRow;
import org.apache.carbondata.core.indexstore.row.UnsafeDataMapRow;
import org.apache.carbondata.core.indexstore.schema.DataMapSchema;
-import org.apache.carbondata.core.memory.MemoryAllocator;
-import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
import org.apache.carbondata.core.memory.MemoryBlock;
+import org.apache.carbondata.core.memory.MemoryException;
+import org.apache.carbondata.core.memory.UnsafeMemoryManager;
import static org.apache.carbondata.core.memory.CarbonUnsafe.BYTE_ARRAY_OFFSET;
import static org.apache.carbondata.core.memory.CarbonUnsafe.unsafe;
@@ -39,8 +39,6 @@ public class UnsafeMemoryDMStore {
private int runningLength;
- private MemoryAllocator memoryAllocator;
-
private boolean isMemoryFreed;
private DataMapSchema[] schema;
@@ -49,11 +47,10 @@ public class UnsafeMemoryDMStore {
private int rowCount;
- public UnsafeMemoryDMStore(DataMapSchema[] schema) {
+ public UnsafeMemoryDMStore(DataMapSchema[] schema) throws MemoryException {
this.schema = schema;
- this.memoryAllocator = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator();
this.allocatedSize = capacity;
- this.memoryBlock = memoryAllocator.allocate(allocatedSize);
+ this.memoryBlock = UnsafeMemoryManager.allocateMemoryWithRetry(allocatedSize);
this.pointers = new int[1000];
}
@@ -63,13 +60,13 @@ public class UnsafeMemoryDMStore {
*
* @param rowSize
*/
- private void ensureSize(int rowSize) {
+ private void ensureSize(int rowSize) throws MemoryException {
if (runningLength + rowSize >= allocatedSize) {
MemoryBlock allocate =
- MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().allocate(allocatedSize + capacity);
+ UnsafeMemoryManager.allocateMemoryWithRetry(allocatedSize + capacity);
unsafe.copyMemory(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset(),
allocate.getBaseObject(), allocate.getBaseOffset(), runningLength);
- memoryAllocator.free(memoryBlock);
+ UnsafeMemoryManager.INSTANCE.freeMemory(memoryBlock);
allocatedSize = allocatedSize + capacity;
memoryBlock = allocate;
}
@@ -86,7 +83,7 @@ public class UnsafeMemoryDMStore {
* @param indexRow
* @return
*/
- public void addIndexRowToUnsafe(DataMapRow indexRow) {
+ public void addIndexRowToUnsafe(DataMapRow indexRow) throws MemoryException {
// First calculate the required memory to keep the row in unsafe
int rowSize = indexRow.getTotalSizeInBytes();
// Check whether allocated memory is sufficient or not.
@@ -168,13 +165,13 @@ public class UnsafeMemoryDMStore {
return new UnsafeDataMapRow(schema, memoryBlock, pointers[index]);
}
- public void finishWriting() {
+ public void finishWriting() throws MemoryException {
if (runningLength < allocatedSize) {
MemoryBlock allocate =
- MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().allocate(runningLength);
+ UnsafeMemoryManager.allocateMemoryWithRetry(runningLength);
unsafe.copyMemory(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset(),
allocate.getBaseObject(), allocate.getBaseOffset(), runningLength);
- memoryAllocator.free(memoryBlock);
+ UnsafeMemoryManager.INSTANCE.freeMemory(memoryBlock);
memoryBlock = allocate;
}
// Compact pointers.
@@ -187,7 +184,7 @@ public class UnsafeMemoryDMStore {
public void freeMemory() {
if (!isMemoryFreed) {
- memoryAllocator.free(memoryBlock);
+ UnsafeMemoryManager.INSTANCE.freeMemory(memoryBlock);
isMemoryFreed = true;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4ab1ff6/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
index 79aa091..680852d 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -44,6 +44,7 @@ import org.apache.carbondata.core.indexstore.row.DataMapRow;
import org.apache.carbondata.core.indexstore.row.DataMapRowImpl;
import org.apache.carbondata.core.indexstore.schema.DataMapSchema;
import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.memory.MemoryException;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex;
@@ -110,7 +111,7 @@ public class BlockletDataMap implements DataMap, Cacheable {
if (unsafeMemoryDMStore != null) {
unsafeMemoryDMStore.finishWriting();
}
- } catch (IOException e) {
+ } catch (Exception e) {
throw new RuntimeException(e);
}
}
@@ -156,11 +157,11 @@ public class BlockletDataMap implements DataMap, Cacheable {
DataOutput dataOutput = new DataOutputStream(stream);
blockletInfo.write(dataOutput);
serializedData = stream.toByteArray();
- } catch (IOException e) {
+ row.setByteArray(serializedData, ordinal);
+ unsafeMemoryDMStore.addIndexRowToUnsafe(row);
+ } catch (Exception e) {
throw new RuntimeException(e);
}
- row.setByteArray(serializedData, ordinal);
- unsafeMemoryDMStore.addIndexRowToUnsafe(row);
}
}
@@ -176,7 +177,7 @@ public class BlockletDataMap implements DataMap, Cacheable {
return minRow;
}
- private void createSchema(SegmentProperties segmentProperties) {
+ private void createSchema(SegmentProperties segmentProperties) throws MemoryException {
List<DataMapSchema> indexSchemas = new ArrayList<>();
// Index key
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4ab1ff6/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
index 28e63a9..c491908 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/UnsafeMemoryManager.java
@@ -101,11 +101,9 @@ public class UnsafeMemoryManager {
if (memoryUsed + memoryRequested <= totalMemory) {
MemoryBlock allocate = allocator.allocate(memoryRequested);
memoryUsed += allocate.size();
- if (LOGGER.isDebugEnabled()) {
- set.add(allocate);
- LOGGER.error("Memory block (" + allocate + ") is created with size " + allocate.size() +
- ". Total memory used " + memoryUsed + "Bytes, left " + getAvailableMemory() + "Bytes");
- }
+ set.add(allocate);
+ LOGGER.info("Memory block (" + allocate + ") is created with size " + allocate.size() +
+ ". Total memory used " + memoryUsed + "Bytes, left " + getAvailableMemory() + "Bytes");
return allocate;
}
return null;
@@ -115,11 +113,9 @@ public class UnsafeMemoryManager {
allocator.free(memoryBlock);
memoryUsed -= memoryBlock.size();
memoryUsed = memoryUsed < 0 ? 0 : memoryUsed;
- if (LOGGER.isDebugEnabled()) {
- set.remove(memoryBlock);
- LOGGER.error("Memory block (" + memoryBlock + ") released. Total memory used " + memoryUsed +
- "Bytes, left " + getAvailableMemory() + "Bytes. Total allocated block: " + set.size());
- }
+ set.remove(memoryBlock);
+ LOGGER.info("Memory block (" + memoryBlock + ") released. Total memory used " + memoryUsed +
+ "Bytes, left " + getAvailableMemory() + "Bytes. Total allocated block: " + set.size());
}
private synchronized long getAvailableMemory() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4ab1ff6/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index e73c04a..8938699 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -38,6 +38,7 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.schema.PartitionInfo;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
import org.apache.carbondata.core.mutate.SegmentUpdateDetails;
import org.apache.carbondata.core.mutate.UpdateVO;
@@ -99,59 +100,56 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
private static final String FILTER_PREDICATE =
"mapreduce.input.carboninputformat.filter.predicate";
private static final String COLUMN_PROJECTION = "mapreduce.input.carboninputformat.projection";
- private static final String CARBON_TABLE = "mapreduce.input.carboninputformat.table";
+ private static final String TABLE_INFO = "mapreduce.input.carboninputformat.tableinfo";
private static final String CARBON_READ_SUPPORT = "mapreduce.input.carboninputformat.readsupport";
+ // a cache for carbon table, it will be used in task side
+ private CarbonTable carbonTable;
+
/**
- * It is optional, if user does not set then it reads from store
- *
- * @param configuration
- * @param carbonTable
- * @throws IOException
+ * Set the `tableInfo` in `configuration`
*/
- public static void setCarbonTable(Configuration configuration, CarbonTable carbonTable)
+ public static void setTableInfo(Configuration configuration, TableInfo tableInfo)
throws IOException {
- if (null != carbonTable) {
- configuration.set(CARBON_TABLE, ObjectSerializationUtil.convertObjectToString(carbonTable));
+ if (null != tableInfo) {
+ configuration.set(TABLE_INFO, ObjectSerializationUtil.convertObjectToString(tableInfo));
}
}
- public static CarbonTable getCarbonTable(Configuration configuration) throws IOException {
- String carbonTableStr = configuration.get(CARBON_TABLE);
- if (carbonTableStr == null) {
- populateCarbonTable(configuration);
- // read it from schema file in the store
- carbonTableStr = configuration.get(CARBON_TABLE);
- return (CarbonTable) ObjectSerializationUtil.convertStringToObject(carbonTableStr);
- }
- return (CarbonTable) ObjectSerializationUtil.convertStringToObject(carbonTableStr);
+ /**
+ * Get TableInfo object from `configuration`
+ */
+ private TableInfo getTableInfo(Configuration configuration) throws IOException {
+ String tableInfoStr = configuration.get(TABLE_INFO);
+ return (TableInfo) ObjectSerializationUtil.convertStringToObject(tableInfoStr);
}
/**
- * this method will read the schema from the physical file and populate into CARBON_TABLE
- *
- * @param configuration
- * @throws IOException
+ * Get the cached CarbonTable or create it by TableInfo in `configuration`
*/
- private static void populateCarbonTable(Configuration configuration) throws IOException {
- String dirs = configuration.get(INPUT_DIR, "");
- String[] inputPaths = StringUtils.split(dirs);
- if (inputPaths.length == 0) {
- throw new InvalidPathException("No input paths specified in job");
+ private CarbonTable getOrCreateCarbonTable(Configuration configuration) throws IOException {
+ if (carbonTable == null) {
+ // carbon table should be created either from deserialized table info (schema saved in
+ // hive metastore) or by reading schema in HDFS (schema saved in HDFS)
+ TableInfo tableInfo = getTableInfo(configuration);
+ CarbonTable carbonTable;
+ if (tableInfo != null) {
+ carbonTable = CarbonTable.buildFromTableInfo(tableInfo);
+ } else {
+ carbonTable = SchemaReader.readCarbonTableFromStore(
+ getAbsoluteTableIdentifier(configuration));
+ }
+ this.carbonTable = carbonTable;
+ return carbonTable;
+ } else {
+ return this.carbonTable;
}
- AbsoluteTableIdentifier absoluteTableIdentifier =
- AbsoluteTableIdentifier.fromTablePath(inputPaths[0]);
- // read the schema file to get the absoluteTableIdentifier having the correct table id
- // persisted in the schema
- CarbonTable carbonTable = SchemaReader.readCarbonTableFromStore(absoluteTableIdentifier);
- setCarbonTable(configuration, carbonTable);
}
public static void setTablePath(Configuration configuration, String tablePath)
throws IOException {
configuration.set(FileInputFormat.INPUT_DIR, tablePath);
}
-
/**
* It sets unresolved filter expression.
*
@@ -213,9 +211,14 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
configuration.set(INPUT_FILES, CarbonUtil.getSegmentString(validFiles));
}
- private static AbsoluteTableIdentifier getAbsoluteTableIdentifier(Configuration configuration)
+ private AbsoluteTableIdentifier getAbsoluteTableIdentifier(Configuration configuration)
throws IOException {
- return getCarbonTable(configuration).getAbsoluteTableIdentifier();
+ String dirs = configuration.get(INPUT_DIR, "");
+ String[] inputPaths = StringUtils.split(dirs);
+ if (inputPaths.length == 0) {
+ throw new InvalidPathException("No input paths specified in job");
+ }
+ return AbsoluteTableIdentifier.fromTablePath(inputPaths[0]);
}
/**
@@ -262,7 +265,7 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
// process and resolve the expression
Expression filter = getFilterPredicates(job.getConfiguration());
- CarbonTable carbonTable = getCarbonTable(job.getConfiguration());
+ CarbonTable carbonTable = getOrCreateCarbonTable(job.getConfiguration());
// this will be null in case of corrupt schema file.
if (null == carbonTable) {
throw new IOException("Missing/Corrupt schema file for table.");
@@ -320,7 +323,7 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
Boolean isIUDTable = false;
AbsoluteTableIdentifier absoluteTableIdentifier =
- getCarbonTable(job.getConfiguration()).getAbsoluteTableIdentifier();
+ getOrCreateCarbonTable(job.getConfiguration()).getAbsoluteTableIdentifier();
SegmentUpdateStatusManager updateStatusManager =
new SegmentUpdateStatusManager(absoluteTableIdentifier);
@@ -432,7 +435,7 @@ public class CarbonTableInputFormat<T> extends FileInputFormat<Void, T> {
public QueryModel getQueryModel(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
throws IOException {
Configuration configuration = taskAttemptContext.getConfiguration();
- CarbonTable carbonTable = getCarbonTable(configuration);
+ CarbonTable carbonTable = getOrCreateCarbonTable(configuration);
// getting the table absoluteTableIdentifier from the carbonTable
// to avoid unnecessary deserialization
AbsoluteTableIdentifier identifier = carbonTable.getAbsoluteTableIdentifier();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f4ab1ff6/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index 6bc7564..2e737ab 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -265,7 +265,7 @@ class CarbonMergerRDD[K, V](
val jobConf: JobConf = new JobConf(new Configuration)
val job: Job = new Job(jobConf)
val format = CarbonInputFormatUtil.createCarbonInputFormat(absoluteTableIdentifier, job)
- CarbonInputFormat.setTableInfo(job.getConfiguration,
+ CarbonTableInputFormat.setTableInfo(job.getConfiguration,
carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable.getTableInfo)
var updateDetails: UpdateVO = null
// initialise query_id for job
[03/15] carbondata git commit: [CARBONDATA-1261] Load data sql add
'header' option
Posted by ra...@apache.org.
[CARBONDATA-1261] Load data sql add 'header' option
This closes #1133
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/0481340d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/0481340d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/0481340d
Branch: refs/heads/datamap
Commit: 0481340dbef487c84a924f51282b9b30895204c3
Parents: 31a6ec6
Author: QiangCai <da...@gmail.com>
Authored: Tue Jul 4 12:11:33 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Tue Jul 11 16:24:58 2017 +0800
----------------------------------------------------------------------
.../TestLoadDataWithFileHeaderException.scala | 101 ++++++++++++++++++-
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 2 +-
.../execution/command/carbonTableSchema.scala | 31 +++++-
.../execution/command/carbonTableSchema.scala | 32 +++++-
4 files changed, 162 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0481340d/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithFileHeaderException.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithFileHeaderException.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithFileHeaderException.scala
index dbde455..e36969b 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithFileHeaderException.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithFileHeaderException.scala
@@ -43,7 +43,7 @@ class TestLoadDataWithFileHeaderException extends QueryTest with BeforeAndAfterA
}
}
- test("test load data ddl provided wrong file header exception") {
+ test("test load data ddl provided wrong file header exception") {
try {
sql(s"""
LOAD DATA LOCAL INPATH '$resourcesPath/source_without_header.csv' into table t3
@@ -56,6 +56,105 @@ class TestLoadDataWithFileHeaderException extends QueryTest with BeforeAndAfterA
}
}
+ test("test load data with wrong header , but without fileheader") {
+ try {
+ sql(s"""
+ LOAD DATA LOCAL INPATH '$resourcesPath/source.csv' into table t3
+ options('header'='abc')
+ """)
+ assert(false)
+ } catch {
+ case e: Exception =>
+ assert(e.getMessage.contains("'header' option should be either 'true' or 'false'"))
+ }
+ }
+
+ test("test load data with wrong header and fileheader") {
+ try {
+ sql(s"""
+ LOAD DATA LOCAL INPATH '$resourcesPath/source_without_header.csv' into table t3
+ options('header'='', 'fileheader'='ID,date,country,name,phonetype,serialname,salary')
+ """)
+ assert(false)
+ } catch {
+ case e: Exception =>
+ assert(e.getMessage.contains("'header' option should be either 'true' or 'false'"))
+ }
+ }
+
+ test("test load data with header=false, but without fileheader") {
+ sql(s"""
+ LOAD DATA LOCAL INPATH '$resourcesPath/source_without_header.csv' into table t3
+ options('header'='False')
+ """)
+ }
+
+ test("test load data with header=false and fileheader") {
+ sql(s"""
+ LOAD DATA LOCAL INPATH '$resourcesPath/source_without_header.csv' into table t3
+ options('header'='false', 'fileheader'='ID,date,country,name,phonetype,serialname,salary')
+ """)
+ }
+
+ test("test load data with header=false and wrong fileheader") {
+ try {
+ sql(s"""
+ LOAD DATA LOCAL INPATH '$resourcesPath/source_without_header.csv' into table t3
+ options('header'='false', 'fileheader'='ID1,date2,country,name,phonetype,serialname,salary')
+ """)
+ assert(false)
+ } catch {
+ case e: Exception =>
+ assert(e.getMessage.contains("CSV header in DDL is not proper. Column names in schema and CSV header are not the same"))
+ }
+ }
+
+ test("test load data with header=true, but without fileheader") {
+ sql(s"""
+ LOAD DATA LOCAL INPATH '$resourcesPath/source.csv' into table t3
+ options('header'='True')
+ """)
+ }
+
+ test("test load data with header=true and fileheader") {
+ try {
+ sql(s"""
+ LOAD DATA LOCAL INPATH '$resourcesPath/source.csv' into table t3
+ options('header'='true', 'fileheader'='ID,date,country,name,phonetype,serialname,salary')
+ """)
+ assert(false)
+ } catch {
+ case e: Exception =>
+ assert(e.getMessage.contains("When 'header' option is true, 'fileheader' option is not required."))
+ }
+ }
+
+ test("test load data with header=true and wrong fileheader") {
+ try {
+ sql(s"""
+ LOAD DATA LOCAL INPATH '$resourcesPath/source.csv' into table t3
+ options('header'='true', 'fileheader'='ID1,date1,country,name,phonetype,serialname,salary')
+ """)
+ assert(false)
+ } catch {
+ case e: Exception =>
+ assert(e.getMessage.contains("When 'header' option is true, 'fileheader' option is not required."))
+ }
+ }
+
+ test("test load data without header and fileheader") {
+ sql(s"""
+ LOAD DATA LOCAL INPATH '$resourcesPath/source.csv' into table t3
+ """)
+ }
+
+ test("test load data without header, but with fileheader") {
+ sql(s"""
+ LOAD DATA LOCAL INPATH '$resourcesPath/source_without_header.csv' into table t3
+ options('fileheader'='ID,date,country,name,phonetype,serialname,salary')
+ """)
+ }
+
override def afterAll {
sql("DROP TABLE IF EXISTS t3")
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0481340d/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 4dbdc8d..7bf9765 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -839,7 +839,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
"SERIALIZATION_NULL_FORMAT", "BAD_RECORDS_LOGGER_ENABLE", "BAD_RECORDS_ACTION",
"ALL_DICTIONARY_PATH", "MAXCOLUMNS", "COMMENTCHAR", "DATEFORMAT", "BAD_RECORD_PATH",
"SINGLE_PASS", "IS_EMPTY_DATA_BAD_RECORD", "SORT_SCOPE", "BATCH_SORT_SIZE_INMB",
- "GLOBAL_SORT_PARTITIONS"
+ "GLOBAL_SORT_PARTITIONS", "HEADER"
)
var isSupported = true
val invalidOptions = StringBuilder.newBuilder
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0481340d/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 70c8407..44d5efb 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -419,7 +419,7 @@ case class LoadTable(
val delimiter = options.getOrElse("delimiter", ",")
val quoteChar = options.getOrElse("quotechar", "\"")
- val fileHeader = options.getOrElse("fileheader", "")
+ var fileHeader = options.getOrElse("fileheader", "")
val escapeChar = options.getOrElse("escapechar", "\\")
val commentchar = options.getOrElse("commentchar", "#")
val columnDict = options.getOrElse("columndict", null)
@@ -441,6 +441,35 @@ case class LoadTable(
val batchSortSizeInMB = options.getOrElse("batch_sort_size_inmb", null)
val globalSortPartitions = options.getOrElse("global_sort_partitions", null)
ValidateUtil.validateGlobalSortPartitions(globalSortPartitions)
+
+ // if there isn't file header in csv file and load sql doesn't provide FILEHEADER option,
+ // we should use table schema to generate file header.
+ val headerOption = options.get("header")
+ if (headerOption.isDefined) {
+ // whether the csv file has file header
+ // the default value is true
+ val header = try {
+ headerOption.get.toBoolean
+ } catch {
+ case ex: IllegalArgumentException =>
+ throw new MalformedCarbonCommandException(
+ "'header' option should be either 'true' or 'false'. " + ex.getMessage)
+ }
+ header match {
+ case true =>
+ if (fileHeader.nonEmpty) {
+ throw new MalformedCarbonCommandException(
+ "When 'header' option is true, 'fileheader' option is not required.")
+ }
+ case false =>
+ // generate file header
+ if (fileHeader.isEmpty) {
+ fileHeader = table.getCreateOrderColumn(table.getFactTableName)
+ .asScala.map(_.getColName).mkString(",")
+ }
+ }
+ }
+
val bad_record_path = options.getOrElse("bad_record_path",
CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/0481340d/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 2e5812c..4b22cea 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -537,6 +537,36 @@ case class LoadTable(
carbonLoadModel.setEscapeChar(checkDefaultValue(optionsFinal.get("escapechar").get, "\\"))
carbonLoadModel.setQuoteChar(checkDefaultValue(optionsFinal.get("quotechar").get, "\""))
carbonLoadModel.setCommentChar(checkDefaultValue(optionsFinal.get("commentchar").get, "#"))
+
+ // if there isn't file header in csv file and load sql doesn't provide FILEHEADER option,
+ // we should use table schema to generate file header.
+ var fileHeader = optionsFinal.get("fileheader").get
+ val headerOption = options.get("header")
+ if (headerOption.isDefined) {
+ // whether the csv file has file header
+ // the default value is true
+ val header = try {
+ headerOption.get.toBoolean
+ } catch {
+ case ex: IllegalArgumentException =>
+ throw new MalformedCarbonCommandException(
+ "'header' option should be either 'true' or 'false'. " + ex.getMessage)
+ }
+ header match {
+ case true =>
+ if (fileHeader.nonEmpty) {
+ throw new MalformedCarbonCommandException(
+ "When 'header' option is true, 'fileheader' option is not required.")
+ }
+ case false =>
+ // generate file header
+ if (fileHeader.isEmpty) {
+ fileHeader = table.getCreateOrderColumn(table.getFactTableName)
+ .asScala.map(_.getColName).mkString(",")
+ }
+ }
+ }
+
carbonLoadModel.setDateFormat(dateFormat)
carbonLoadModel.setDefaultTimestampFormat(carbonProperty.getProperty(
CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
@@ -582,7 +612,7 @@ case class LoadTable(
LOGGER.info(s"Initiating Direct Load for the Table : ($dbName.$tableName)")
carbonLoadModel.setFactFilePath(factPath)
carbonLoadModel.setCsvDelimiter(CarbonUtil.unescapeChar(delimeter))
- carbonLoadModel.setCsvHeader(optionsFinal.get("fileheader").get)
+ carbonLoadModel.setCsvHeader(fileHeader)
carbonLoadModel.setColDictFilePath(column_dict)
carbonLoadModel.setDirectLoad(true)
carbonLoadModel.setCsvHeaderColumns(CommonUtil.getCsvHeaderColumns(carbonLoadModel))
[05/15] carbondata git commit: [CARBONDATA-1286] Change Query related
RDD to use TableInfo
Posted by ra...@apache.org.
[CARBONDATA-1286] Change Query related RDD to use TableInfo
This closes #1146
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4b69c9d8
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4b69c9d8
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4b69c9d8
Branch: refs/heads/datamap
Commit: 4b69c9d8e00da4053196250357e67add5088d639
Parents: 191dfb3
Author: jackylk <ja...@huawei.com>
Authored: Sat Jul 8 17:34:44 2017 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Tue Jul 11 15:06:38 2017 +0530
----------------------------------------------------------------------
.../core/metadata/CarbonMetadata.java | 3 +-
.../ThriftWrapperSchemaConverterImpl.java | 8 +-
.../core/metadata/datatype/DataType.java | 44 +++++++-
.../core/metadata/encoder/Encoding.java | 20 ++++
.../core/metadata/schema/SchemaEvolution.java | 1 +
.../core/metadata/schema/table/CarbonTable.java | 105 ++++--------------
.../core/metadata/schema/table/TableInfo.java | 106 +++++++++++++++----
.../core/metadata/schema/table/TableSchema.java | 29 ++++-
.../core/metadata/schema/table/Writable.java | 31 ++++++
.../metadata/schema/table/WritableUtil.java | 45 ++++++++
.../schema/table/column/ColumnSchema.java | 84 ++++++++++++---
.../dictionary/client/DictionaryClientTest.java | 3 +-
...ncrementalColumnDictionaryGeneratorTest.java | 3 +-
.../ServerDictionaryGeneratorTest.java | 3 +-
.../generator/TableDictionaryGeneratorTest.java | 3 +-
.../core/metadata/CarbonMetadataTest.java | 6 +-
.../ThriftWrapperSchemaConverterImplTest.java | 9 +-
.../metadata/schema/table/CarbonTableTest.java | 3 +-
.../table/CarbonTableWithComplexTypesTest.java | 3 +-
.../carbondata/hadoop/CarbonInputFormat.java | 93 ++++++++--------
.../hadoop/ft/CarbonInputMapperTest.java | 2 +-
.../hadoop/test/util/StoreCreator.java | 1 -
.../hive/MapredCarbonInputFormat.java | 16 +++
.../testsuite/createTable/TestTableIdTest.scala | 76 -------------
.../apache/carbondata/spark/rdd/CarbonRDD.scala | 17 +++
.../carbondata/spark/rdd/CarbonScanRDD.scala | 14 +--
.../spark/rdd/DataManagementFunc.scala | 1 -
.../execution/command/carbonTableSchema.scala | 1 -
.../scala/org/apache/spark/sql/CarbonScan.scala | 4 +-
.../execution/command/carbonTableSchema.scala | 2 -
.../sql/CarbonDatasourceHadoopRelation.scala | 42 +++++---
.../spark/sql/CarbonDictionaryDecoder.scala | 15 ++-
.../execution/CarbonLateDecodeStrategy.scala | 19 ++--
.../sql/execution/command/IUDCommands.scala | 8 +-
.../execution/command/carbonTableSchema.scala | 2 -
.../processing/model/CarbonLoadModel.java | 7 --
.../carbondata/processing/StoreCreator.java | 1 -
37 files changed, 498 insertions(+), 332 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/core/src/main/java/org/apache/carbondata/core/metadata/CarbonMetadata.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/CarbonMetadata.java b/core/src/main/java/org/apache/carbondata/core/metadata/CarbonMetadata.java
index 079540f..75fe78b 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/CarbonMetadata.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/CarbonMetadata.java
@@ -78,8 +78,7 @@ public final class CarbonMetadata {
CarbonTable carbonTable = tableInfoMap.get(convertToLowerCase(tableInfo.getTableUniqueName()));
if (null == carbonTable || carbonTable.getTableLastUpdatedTime() < tableInfo
.getLastUpdatedTime()) {
- carbonTable = new CarbonTable();
- carbonTable.loadCarbonTable(tableInfo);
+ carbonTable = CarbonTable.buildFromTableInfo(tableInfo);
tableInfoMap.put(convertToLowerCase(tableInfo.getTableUniqueName()), carbonTable);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index 2d5f395..5f71582 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -269,12 +269,8 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
org.apache.carbondata.format.TableSchema thriftFactTable =
fromWrapperToExternalTableSchema(wrapperTableInfo.getFactTable());
- List<org.apache.carbondata.format.TableSchema> thriftAggTables =
- new ArrayList<org.apache.carbondata.format.TableSchema>();
- for (TableSchema wrapperAggTableSchema : wrapperTableInfo.getAggregateTableList()) {
- thriftAggTables.add(fromWrapperToExternalTableSchema(wrapperAggTableSchema));
- }
- return new org.apache.carbondata.format.TableInfo(thriftFactTable, thriftAggTables);
+ return new org.apache.carbondata.format.TableInfo(thriftFactTable, new ArrayList<org.apache
+ .carbondata.format.TableSchema>());
}
/* (non-Javadoc)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
index 52e63dc..3a7f75d 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
@@ -46,8 +46,8 @@ public enum DataType {
// size of the value of this data type, negative value means variable length
private int sizeInBytes;
- DataType(int value ,String name, int sizeInBytes) {
- this.precedenceOrder = value;
+ DataType(int precedenceOrder, String name, int sizeInBytes) {
+ this.precedenceOrder = precedenceOrder;
this.name = name;
this.sizeInBytes = sizeInBytes;
}
@@ -71,4 +71,44 @@ public enum DataType {
public int getSizeBits() {
return (int) (Math.log(getSizeInBytes()) / Math.log(2));
}
+
+ public static DataType valueOf(int ordinal) {
+ if (ordinal == STRING.ordinal()) {
+ return STRING;
+ } else if (ordinal == DATE.ordinal()) {
+ return DATE;
+ } else if (ordinal == TIMESTAMP.ordinal()) {
+ return TIMESTAMP;
+ } else if (ordinal == BOOLEAN.ordinal()) {
+ return BOOLEAN;
+ } else if (ordinal == SHORT.ordinal()) {
+ return SHORT;
+ } else if (ordinal == INT.ordinal()) {
+ return INT;
+ } else if (ordinal == FLOAT.ordinal()) {
+ return FLOAT;
+ } else if (ordinal == LONG.ordinal()) {
+ return LONG;
+ } else if (ordinal == DOUBLE.ordinal()) {
+ return DOUBLE;
+ } else if (ordinal == NULL.ordinal()) {
+ return NULL;
+ } else if (ordinal == DECIMAL.ordinal()) {
+ return DECIMAL;
+ } else if (ordinal == ARRAY.ordinal()) {
+ return ARRAY;
+ } else if (ordinal == STRUCT.ordinal()) {
+ return STRUCT;
+ } else if (ordinal == MAP.ordinal()) {
+ return MAP;
+ } else if (ordinal == BYTE.ordinal()) {
+ return BYTE;
+ } else if (ordinal == BYTE_ARRAY.ordinal()) {
+ return BYTE_ARRAY;
+ } else if (ordinal == SHORT_INT.ordinal()) {
+ return SHORT_INT;
+ } else {
+ throw new RuntimeException("create DataType with invalid ordinal: " + ordinal);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/core/src/main/java/org/apache/carbondata/core/metadata/encoder/Encoding.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/encoder/Encoding.java b/core/src/main/java/org/apache/carbondata/core/metadata/encoder/Encoding.java
index fe24975..e3d7a9a 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/encoder/Encoding.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/encoder/Encoding.java
@@ -27,4 +27,24 @@ public enum Encoding {
BIT_PACKED,
DIRECT_DICTIONARY,
IMPLICIT;
+
+ public static Encoding valueOf(int ordinal) {
+ if (ordinal == DICTIONARY.ordinal()) {
+ return DICTIONARY;
+ } else if (ordinal == DELTA.ordinal()) {
+ return DELTA;
+ } else if (ordinal == RLE.ordinal()) {
+ return RLE;
+ } else if (ordinal == INVERTED_INDEX.ordinal()) {
+ return INVERTED_INDEX;
+ } else if (ordinal == BIT_PACKED.ordinal()) {
+ return BIT_PACKED;
+ } else if (ordinal == DIRECT_DICTIONARY.ordinal()) {
+ return DIRECT_DICTIONARY;
+ } else if (ordinal == IMPLICIT.ordinal()) {
+ return IMPLICIT;
+ } else {
+ throw new RuntimeException("create Encoding with invalid ordinal: " + ordinal);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/core/src/main/java/org/apache/carbondata/core/metadata/schema/SchemaEvolution.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/SchemaEvolution.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/SchemaEvolution.java
index 4bdbe8d..6960736 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/SchemaEvolution.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/SchemaEvolution.java
@@ -47,4 +47,5 @@ public class SchemaEvolution implements Serializable {
public void setSchemaEvolutionEntryList(List<SchemaEvolutionEntry> schemaEvolutionEntryList) {
this.schemaEvolutionEntryList = schemaEvolutionEntryList;
}
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 16ded57..01b3022 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -20,8 +20,6 @@ package org.apache.carbondata.core.metadata.schema.table;
import java.io.Serializable;
import java.util.*;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
@@ -33,7 +31,6 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonImplicitDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.stats.PartitionStatistic;
/**
* Mapping class for Carbon actual table
@@ -41,15 +38,14 @@ import org.apache.carbondata.core.stats.PartitionStatistic;
public class CarbonTable implements Serializable {
/**
- * serialization id
+ * the cached table info
*/
- private static final long serialVersionUID = 8696507171227156445L;
+ private TableInfo tableInfo;
/**
- * Attribute for Carbon table LOGGER
+ * serialization id
*/
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CarbonTable.class.getName());
+ private static final long serialVersionUID = 8696507171227156445L;
/**
* Absolute table identifier
@@ -99,20 +95,11 @@ public class CarbonTable implements Serializable {
private Map<String, PartitionInfo> tablePartitionMap;
/**
- * statistic information of partition table
- */
- private PartitionStatistic partitionStatistic;
- /**
* tableUniqueName
*/
private String tableUniqueName;
/**
- * Aggregate tables name
- */
- private List<String> aggregateTablesName;
-
- /**
* metadata file path (check if it is really required )
*/
private String metaDataFilepath;
@@ -137,13 +124,12 @@ public class CarbonTable implements Serializable {
*/
private int numberOfNoDictSortColumns;
- public CarbonTable() {
+ private CarbonTable() {
this.tableDimensionsMap = new HashMap<String, List<CarbonDimension>>();
this.tableImplicitDimensionsMap = new HashMap<String, List<CarbonDimension>>();
this.tableMeasuresMap = new HashMap<String, List<CarbonMeasure>>();
this.tableBucketMap = new HashMap<>();
this.tablePartitionMap = new HashMap<>();
- this.aggregateTablesName = new ArrayList<String>();
this.createOrderColumn = new HashMap<String, List<CarbonColumn>>();
this.tablePrimitiveDimensionsMap = new HashMap<String, List<CarbonDimension>>();
}
@@ -151,39 +137,26 @@ public class CarbonTable implements Serializable {
/**
* @param tableInfo
*/
- public void loadCarbonTable(TableInfo tableInfo) {
- this.blockSize = getTableBlockSizeInMB(tableInfo);
- this.tableLastUpdatedTime = tableInfo.getLastUpdatedTime();
- this.tableUniqueName = tableInfo.getTableUniqueName();
- this.metaDataFilepath = tableInfo.getMetaDataFilepath();
- //setting unique table identifier
- CarbonTableIdentifier carbontableIdentifier =
- new CarbonTableIdentifier(tableInfo.getDatabaseName(),
- tableInfo.getFactTable().getTableName(), tableInfo.getFactTable().getTableId());
- this.absoluteTableIdentifier =
- new AbsoluteTableIdentifier(tableInfo.getStorePath(), carbontableIdentifier);
-
- fillDimensionsAndMeasuresForTables(tableInfo.getFactTable());
- fillCreateOrderColumn(tableInfo.getFactTable().getTableName());
- List<TableSchema> aggregateTableList = tableInfo.getAggregateTableList();
- for (TableSchema aggTable : aggregateTableList) {
- this.aggregateTablesName.add(aggTable.getTableName());
- fillDimensionsAndMeasuresForTables(aggTable);
- if (aggTable.getBucketingInfo() != null) {
- tableBucketMap.put(aggTable.getTableName(), aggTable.getBucketingInfo());
- }
- if (aggTable.getPartitionInfo() != null) {
- tablePartitionMap.put(aggTable.getTableName(), aggTable.getPartitionInfo());
- }
- }
+ public static CarbonTable buildFromTableInfo(TableInfo tableInfo) {
+ CarbonTable table = new CarbonTable();
+ table.tableInfo = tableInfo;
+ table.blockSize = tableInfo.getTableBlockSizeInMB();
+ table.tableLastUpdatedTime = tableInfo.getLastUpdatedTime();
+ table.tableUniqueName = tableInfo.getTableUniqueName();
+ table.metaDataFilepath = tableInfo.getMetaDataFilepath();
+ table.absoluteTableIdentifier = tableInfo.getOrCreateAbsoluteTableIdentifier();
+
+ table.fillDimensionsAndMeasuresForTables(tableInfo.getFactTable());
+ table.fillCreateOrderColumn(tableInfo.getFactTable().getTableName());
if (tableInfo.getFactTable().getBucketingInfo() != null) {
- tableBucketMap.put(tableInfo.getFactTable().getTableName(),
+ table.tableBucketMap.put(tableInfo.getFactTable().getTableName(),
tableInfo.getFactTable().getBucketingInfo());
}
if (tableInfo.getFactTable().getPartitionInfo() != null) {
- tablePartitionMap.put(tableInfo.getFactTable().getTableName(),
+ table.tablePartitionMap.put(tableInfo.getFactTable().getTableName(),
tableInfo.getFactTable().getPartitionInfo());
}
+ return table;
}
/**
@@ -213,29 +186,6 @@ public class CarbonTable implements Serializable {
this.createOrderColumn.put(tableName, columns);
}
- /**
- * This method will return the table size. Default table block size will be considered
- * in case not specified by the user
- *
- * @param tableInfo
- * @return
- */
- private int getTableBlockSizeInMB(TableInfo tableInfo) {
- String tableBlockSize = null;
- // In case of old store there will not be any map for table properties so table properties
- // will be null
- Map<String, String> tableProperties = tableInfo.getFactTable().getTableProperties();
- if (null != tableProperties) {
- tableBlockSize = tableProperties.get(CarbonCommonConstants.TABLE_BLOCKSIZE);
- }
- if (null == tableBlockSize) {
- tableBlockSize = CarbonCommonConstants.BLOCK_SIZE_DEFAULT_VAL;
- LOGGER.info("Table block size not specified for " + tableInfo.getTableUniqueName()
- + ". Therefore considering the default value "
- + CarbonCommonConstants.BLOCK_SIZE_DEFAULT_VAL + " MB");
- }
- return Integer.parseInt(tableBlockSize);
- }
/**
* Fill allDimensions and allMeasures for carbon table
@@ -415,13 +365,6 @@ public class CarbonTable implements Serializable {
}
/**
- * @return list of aggregate TablesName
- */
- public List<String> getAggregateTablesName() {
- return aggregateTablesName;
- }
-
- /**
* @return the tableLastUpdatedTime
*/
public long getTableLastUpdatedTime() {
@@ -597,10 +540,6 @@ public class CarbonTable implements Serializable {
return null != tablePartitionMap.get(getFactTableName());
}
- public PartitionStatistic getPartitionStatistic() {
- return partitionStatistic;
- }
-
/**
* @return absolute table identifier
*/
@@ -708,8 +647,6 @@ public class CarbonTable implements Serializable {
return sort_columsList;
}
-
-
public int getNumberOfSortColumns() {
return numberOfSortColumns;
}
@@ -717,4 +654,8 @@ public class CarbonTable implements Serializable {
public int getNumberOfNoDictSortColumns() {
return numberOfNoDictSortColumns;
}
+
+ public TableInfo getTableInfo() {
+ return tableInfo;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
index 328928d..78cd97b 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableInfo.java
@@ -16,17 +16,30 @@
*/
package org.apache.carbondata.core.metadata.schema.table;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.Map;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
/**
* Store the information about the table.
* it stores the fact table as well as aggregate table present in the schema
*/
-public class TableInfo implements Serializable {
+public class TableInfo implements Serializable, Writable {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(TableInfo.class.getName());
/**
* serialization version
@@ -49,11 +62,6 @@ public class TableInfo implements Serializable {
private TableSchema factTable;
/**
- * list of aggregate table
- */
- private List<TableSchema> aggregateTableList;
-
- /**
* last updated time to update the table if any changes
*/
private long lastUpdatedTime;
@@ -68,8 +76,10 @@ public class TableInfo implements Serializable {
*/
private String storePath;
+ // this idenifier is a lazy field which will be created when it is used first time
+ private AbsoluteTableIdentifier identifier;
+
public TableInfo() {
- aggregateTableList = new ArrayList<TableSchema>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
}
/**
@@ -87,20 +97,6 @@ public class TableInfo implements Serializable {
}
/**
- * @return the aggregateTableList
- */
- public List<TableSchema> getAggregateTableList() {
- return aggregateTableList;
- }
-
- /**
- * @param aggregateTableList the aggregateTableList to set
- */
- public void setAggregateTableList(List<TableSchema> aggregateTableList) {
- this.aggregateTableList = aggregateTableList;
- }
-
- /**
* @return the databaseName
*/
public String getDatabaseName() {
@@ -206,4 +202,68 @@ public class TableInfo implements Serializable {
}
return true;
}
+
+ /**
+ * This method will return the table size. Default table block size will be considered
+ * in case not specified by the user
+ */
+ int getTableBlockSizeInMB() {
+ String tableBlockSize = null;
+ // In case of old store there will not be any map for table properties so table properties
+ // will be null
+ Map<String, String> tableProperties = getFactTable().getTableProperties();
+ if (null != tableProperties) {
+ tableBlockSize = tableProperties.get(CarbonCommonConstants.TABLE_BLOCKSIZE);
+ }
+ if (null == tableBlockSize) {
+ tableBlockSize = CarbonCommonConstants.BLOCK_SIZE_DEFAULT_VAL;
+ LOGGER.info("Table block size not specified for " + getTableUniqueName()
+ + ". Therefore considering the default value "
+ + CarbonCommonConstants.BLOCK_SIZE_DEFAULT_VAL + " MB");
+ }
+ return Integer.parseInt(tableBlockSize);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeUTF(databaseName);
+ out.writeUTF(tableUniqueName);
+ factTable.write(out);
+ out.writeLong(lastUpdatedTime);
+ out.writeUTF(metaDataFilepath);
+ out.writeUTF(storePath);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ this.databaseName = in.readUTF();
+ this.tableUniqueName = in.readUTF();
+ this.factTable = new TableSchema();
+ this.factTable.readFields(in);
+ this.lastUpdatedTime = in.readLong();
+ this.metaDataFilepath = in.readUTF();
+ this.storePath = in.readUTF();
+ }
+
+ public AbsoluteTableIdentifier getOrCreateAbsoluteTableIdentifier() {
+ if (identifier == null) {
+ CarbonTableIdentifier carbontableIdentifier =
+ new CarbonTableIdentifier(databaseName, factTable.getTableName(), factTable.getTableId());
+ identifier = new AbsoluteTableIdentifier(storePath, carbontableIdentifier);
+ }
+ return identifier;
+ }
+
+ public byte[] serialize() throws IOException {
+ ByteArrayOutputStream bao = new ByteArrayOutputStream();
+ this.write(new DataOutputStream(bao));
+ return bao.toByteArray();
+ }
+
+ public static TableInfo deserialize(byte[] bytes) throws IOException {
+ TableInfo tableInfo = new TableInfo();
+ DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes));
+ tableInfo.readFields(in);
+ return tableInfo;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
index f9d848e..a396d19 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
@@ -16,6 +16,9 @@
*/
package org.apache.carbondata.core.metadata.schema.table;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
@@ -30,7 +33,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
/**
* Persisting the table information
*/
-public class TableSchema implements Serializable {
+public class TableSchema implements Serializable, Writable {
/**
* serialization version
@@ -198,4 +201,28 @@ public class TableSchema implements Serializable {
public void setPartitionInfo(PartitionInfo partitionInfo) {
this.partitionInfo = partitionInfo;
}
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeUTF(tableId);
+ out.writeUTF(tableName);
+ out.writeInt(listOfColumns.size());
+ for (ColumnSchema column : listOfColumns) {
+ column.write(out);
+ }
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ this.tableId = in.readUTF();
+ this.tableName = in.readUTF();
+ int listSize = in.readInt();
+ this.listOfColumns = new ArrayList<>(listSize);
+ for (int i = 0; i < listSize; i++) {
+ ColumnSchema schema = new ColumnSchema();
+ schema.readFields(in);
+ this.listOfColumns.add(schema);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/Writable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/Writable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/Writable.java
new file mode 100644
index 0000000..94a3b78
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/Writable.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.metadata.schema.table;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+// The same interface as hadoop.io.Writable. We port the interface here to avoid hadoop package
+// dependency
+public interface Writable {
+
+ void write(DataOutput out) throws IOException;
+
+ void readFields(DataInput in) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/WritableUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/WritableUtil.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/WritableUtil.java
new file mode 100644
index 0000000..dbcb1a7
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/WritableUtil.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.metadata.schema.table;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+public class WritableUtil {
+
+ public static void writeByteArray(DataOutput out, byte[] bytes) throws IOException {
+ if (bytes == null) {
+ out.writeInt(-1);
+ } else {
+ out.writeInt(bytes.length);
+ out.write(bytes);
+ }
+ }
+
+ public static byte[] readByteArray(DataInput in) throws IOException {
+ int length = in.readInt();
+ if (length == -1) {
+ return null;
+ } else {
+ byte[] b = new byte[length];
+ in.readFully(b);
+ return b;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
index f5b8116..3680d53 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
@@ -16,17 +16,24 @@
*/
package org.apache.carbondata.core.metadata.schema.table.column;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.Writable;
+import org.apache.carbondata.core.metadata.schema.table.WritableUtil;
/**
* Store the information about the column meta data present the table
*/
-public class ColumnSchema implements Serializable {
+public class ColumnSchema implements Serializable, Writable {
/**
* serialization version
@@ -37,6 +44,7 @@ public class ColumnSchema implements Serializable {
* dataType
*/
private DataType dataType;
+
/**
* Name of the column. If it is a complex data type, we follow a naming rule
* grand_parent_column.parent_column.child_column
@@ -360,17 +368,6 @@ public class ColumnSchema implements Serializable {
}
/**
- * @param property
- * @return
- */
- public String getColumnProperty(String property) {
- if (null != columnProperties) {
- return columnProperties.get(property);
- }
- return null;
- }
-
- /**
* return columnproperties
*/
public Map<String, String> getColumnProperties() {
@@ -421,4 +418,67 @@ public class ColumnSchema implements Serializable {
public void setSortColumn(boolean sortColumn) {
isSortColumn = sortColumn;
}
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeShort(dataType.ordinal());
+ out.writeUTF(columnName);
+ out.writeUTF(columnUniqueId);
+ out.writeUTF(columnReferenceId);
+ if (encodingList == null) {
+ out.writeShort(0);
+ } else {
+ out.writeShort(encodingList.size());
+ for (Encoding encoding : encodingList) {
+ out.writeShort(encoding.ordinal());
+ }
+ }
+ out.writeBoolean(isDimensionColumn);
+ out.writeInt(scale);
+ out.writeInt(precision);
+ out.writeInt(schemaOrdinal);
+ out.writeInt(numberOfChild);
+ WritableUtil.writeByteArray(out, defaultValue);
+ if (columnProperties == null) {
+ out.writeShort(0);
+ } else {
+ out.writeShort(columnProperties.size());
+ for (Map.Entry<String, String> entry : columnProperties.entrySet()) {
+ out.writeUTF(entry.getKey());
+ out.writeUTF(entry.getValue());
+ }
+ }
+ out.writeBoolean(invisible);
+ out.writeBoolean(isSortColumn);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ int ordinal = in.readShort();
+ this.dataType = DataType.valueOf(ordinal);
+ this.columnName = in.readUTF();
+ this.columnUniqueId = in.readUTF();
+ this.columnReferenceId = in.readUTF();
+ int encodingListSize = in.readShort();
+ this.encodingList = new ArrayList<>(encodingListSize);
+ for (int i = 0; i < encodingListSize; i++) {
+ ordinal = in.readShort();
+ encodingList.add(Encoding.valueOf(ordinal));
+ }
+ this.isDimensionColumn = in.readBoolean();
+ this.scale = in.readInt();
+ this.precision = in.readInt();
+ this.schemaOrdinal = in.readInt();
+ this.numberOfChild = in.readInt();
+ this.defaultValue = WritableUtil.readByteArray(in);
+ int mapSize = in.readShort();
+ this.columnProperties = new HashMap<>(mapSize);
+ for (int i = 0; i < mapSize; i++) {
+ String key = in.readUTF();
+ String value = in.readUTF();
+ this.columnProperties.put(key, value);
+ }
+ this.invisible = in.readBoolean();
+ this.isSortColumn = in.readBoolean();
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java b/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
index 60d3c26..dc3b232 100644
--- a/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
@@ -88,8 +88,7 @@ public class DictionaryClientTest {
tableInfo.setDatabaseName("test");
storePath = System.getProperty("java.io.tmpdir") + "/tmp";
tableInfo.setStorePath(storePath);
- CarbonTable carbonTable = new CarbonTable();
- carbonTable.loadCarbonTable(tableInfo);
+ CarbonTable carbonTable = CarbonTable.buildFromTableInfo(tableInfo);
// Add the created table to metadata
metadata.addCarbonTable(carbonTable);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/core/src/test/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGeneratorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGeneratorTest.java b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGeneratorTest.java
index 49db6ce..0bac01a 100644
--- a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGeneratorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGeneratorTest.java
@@ -166,8 +166,7 @@ public class IncrementalColumnDictionaryGeneratorTest {
System.out.print(dictPath.mkdirs());
tableInfo.setStorePath(storePath);
- CarbonTable carbonTable = new CarbonTable();
- carbonTable.loadCarbonTable(tableInfo);
+ CarbonTable carbonTable = CarbonTable.buildFromTableInfo(tableInfo);
// Add the table to metadata
metadata.addCarbonTable(carbonTable);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java
index b66331f..d8df99a 100644
--- a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java
@@ -83,8 +83,7 @@ public class ServerDictionaryGeneratorTest {
tableInfo.setDatabaseName("test");
storePath = System.getProperty("java.io.tmpdir") + "/tmp";
tableInfo.setStorePath(storePath);
- CarbonTable carbonTable = new CarbonTable();
- carbonTable.loadCarbonTable(tableInfo);
+ CarbonTable carbonTable = CarbonTable.buildFromTableInfo(tableInfo);
// Add the created table to metadata
metadata.addCarbonTable(carbonTable);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/core/src/test/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGeneratorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGeneratorTest.java b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGeneratorTest.java
index 0cb47c4..8a68b72 100644
--- a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGeneratorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGeneratorTest.java
@@ -83,8 +83,7 @@ public class TableDictionaryGeneratorTest {
tableInfo.setDatabaseName("test");
storePath = System.getProperty("java.io.tmpdir") + "/tmp";
tableInfo.setStorePath(storePath);
- CarbonTable carbonTable = new CarbonTable();
- carbonTable.loadCarbonTable(tableInfo);
+ CarbonTable carbonTable = CarbonTable.buildFromTableInfo(tableInfo);
// Add the created table to metadata
metadata.addCarbonTable(carbonTable);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/core/src/test/java/org/apache/carbondata/core/metadata/CarbonMetadataTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/CarbonMetadataTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/CarbonMetadataTest.java
index 56d14c2..3af0bdb 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/CarbonMetadataTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/CarbonMetadataTest.java
@@ -160,7 +160,7 @@ public class CarbonMetadataTest {
}
@Test public void testGetCarbonDimensionBasedOnColIdentifier() {
- CarbonTable carbonTable = new CarbonTable();
+ CarbonTable carbonTable = CarbonTable.buildFromTableInfo(getTableInfo(1000L));
String columnIdentifier = "1";
final List<CarbonDimension> carbonDimensions = new ArrayList();
ColumnSchema colSchema1 = new ColumnSchema();
@@ -186,7 +186,7 @@ public class CarbonMetadataTest {
@Test
public void testGetCarbonDimensionBasedOnColIdentifierWhenChildDimensionColumnEqualsColumnIdentifier() {
- CarbonTable carbonTable = new CarbonTable();
+ CarbonTable carbonTable = CarbonTable.buildFromTableInfo(getTableInfo(1000L));
String columnIdentifier = "9";
final List<CarbonDimension> carbonDimensions = new ArrayList();
ColumnSchema colSchema1 = new ColumnSchema();
@@ -226,7 +226,7 @@ public class CarbonMetadataTest {
}
@Test public void testGetCarbonDimensionBasedOnColIdentifierNullCase() {
- CarbonTable carbonTable = new CarbonTable();
+ CarbonTable carbonTable = CarbonTable.buildFromTableInfo(getTableInfo(1000L));
String columnIdentifier = "3";
final List<CarbonDimension> carbonDimensions = new ArrayList();
ColumnSchema colSchema1 = new ColumnSchema();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
index e8851e0..3961d9c 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
@@ -1500,10 +1500,6 @@ public class ThriftWrapperSchemaConverterImplTest {
@Mock public TableSchema getFactTable() {
return wrapperTableSchema;
}
-
- @Mock public List<TableSchema> getAggregateTableList() {
- return tableSchemas;
- }
};
new MockUp<TableSchema>() {
@@ -1529,12 +1525,11 @@ public class ThriftWrapperSchemaConverterImplTest {
};
org.apache.carbondata.format.TableSchema thriftFactTable =
new org.apache.carbondata.format.TableSchema("tableId", thriftColumnSchemas, schemaEvol);
- List<org.apache.carbondata.format.TableSchema> thriftAggTables = new ArrayList<>();
- thriftAggTables.add(thriftFactTable);
org.apache.carbondata.format.TableInfo actualResult = thriftWrapperSchemaConverter
.fromWrapperToExternalTableInfo(wrapperTableInfo, dbName, tableName);
org.apache.carbondata.format.TableInfo expectedResult =
- new org.apache.carbondata.format.TableInfo(thriftFactTable, thriftAggTables);
+ new org.apache.carbondata.format.TableInfo(thriftFactTable, new ArrayList<org.apache
+ .carbondata.format.TableSchema>());
assertEquals(expectedResult, actualResult);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableTest.java
index 9ed8b56..f5ffe51 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableTest.java
@@ -36,8 +36,7 @@ public class CarbonTableTest extends TestCase {
private CarbonTable carbonTable;
@BeforeClass public void setUp() {
- carbonTable = new CarbonTable();
- carbonTable.loadCarbonTable(getTableInfo(1000L));
+ carbonTable = CarbonTable.buildFromTableInfo(getTableInfo(1000L));
}
@AfterClass public void tearDown() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java
index 9c3ab6a..69cab49 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java
@@ -35,8 +35,7 @@ public class CarbonTableWithComplexTypesTest extends TestCase {
private CarbonTable carbonTable;
@BeforeClass public void setUp() {
- carbonTable = new CarbonTable();
- carbonTable.loadCarbonTable(getTableInfo(1000L));
+ carbonTable = CarbonTable.buildFromTableInfo(getTableInfo(1000L));
}
@AfterClass public void tearDown() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index 1e69648..787f571 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -39,6 +39,7 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.schema.PartitionInfo;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.mutate.UpdateVO;
import org.apache.carbondata.core.mutate.data.BlockMappingVO;
import org.apache.carbondata.core.scan.expression.Expression;
@@ -101,51 +102,50 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
private static final String FILTER_PREDICATE =
"mapreduce.input.carboninputformat.filter.predicate";
private static final String COLUMN_PROJECTION = "mapreduce.input.carboninputformat.projection";
- private static final String CARBON_TABLE = "mapreduce.input.carboninputformat.table";
+ private static final String TABLE_INFO = "mapreduce.input.carboninputformat.tableinfo";
private static final String CARBON_READ_SUPPORT = "mapreduce.input.carboninputformat.readsupport";
+ // a cache for carbon table, it will be used in task side
+ private CarbonTable carbonTable;
+
/**
- * It is optional, if user does not set then it reads from store
- *
- * @param configuration
- * @param carbonTable
- * @throws IOException
+ * Set the `tableInfo` in `configuration`
*/
- public static void setCarbonTable(Configuration configuration, CarbonTable carbonTable)
+ public static void setTableInfo(Configuration configuration, TableInfo tableInfo)
throws IOException {
- if (null != carbonTable) {
- configuration.set(CARBON_TABLE, ObjectSerializationUtil.convertObjectToString(carbonTable));
+ if (null != tableInfo) {
+ configuration.set(TABLE_INFO, ObjectSerializationUtil.convertObjectToString(tableInfo));
}
}
- public static CarbonTable getCarbonTable(Configuration configuration) throws IOException {
- String carbonTableStr = configuration.get(CARBON_TABLE);
- if (carbonTableStr == null) {
- populateCarbonTable(configuration);
- // read it from schema file in the store
- carbonTableStr = configuration.get(CARBON_TABLE);
- return (CarbonTable) ObjectSerializationUtil.convertStringToObject(carbonTableStr);
- }
- return (CarbonTable) ObjectSerializationUtil.convertStringToObject(carbonTableStr);
+ /**
+ * Get TableInfo object from `configuration`
+ */
+ private TableInfo getTableInfo(Configuration configuration) throws IOException {
+ String tableInfoStr = configuration.get(TABLE_INFO);
+ return (TableInfo) ObjectSerializationUtil.convertStringToObject(tableInfoStr);
}
/**
- * this method will read the schema from the physical file and populate into CARBON_TABLE
- * @param configuration
- * @throws IOException
+ * Get the cached CarbonTable or create it by TableInfo in `configuration`
*/
- private static void populateCarbonTable(Configuration configuration) throws IOException {
- String dirs = configuration.get(INPUT_DIR, "");
- String[] inputPaths = StringUtils.split(dirs);
- if (inputPaths.length == 0) {
- throw new InvalidPathException("No input paths specified in job");
+ private CarbonTable getOrCreateCarbonTable(Configuration configuration) throws IOException {
+ if (carbonTable == null) {
+ // carbon table should be created either from deserialized table info (schema saved in
+ // hive metastore) or by reading schema in HDFS (schema saved in HDFS)
+ TableInfo tableInfo = getTableInfo(configuration);
+ CarbonTable carbonTable;
+ if (tableInfo != null) {
+ carbonTable = CarbonTable.buildFromTableInfo(tableInfo);
+ } else {
+ carbonTable = SchemaReader.readCarbonTableFromStore(
+ getAbsoluteTableIdentifier(configuration));
+ }
+ this.carbonTable = carbonTable;
+ return carbonTable;
+ } else {
+ return this.carbonTable;
}
- AbsoluteTableIdentifier absoluteTableIdentifier =
- AbsoluteTableIdentifier.fromTablePath(inputPaths[0]);
- // read the schema file to get the absoluteTableIdentifier having the correct table id
- // persisted in the schema
- CarbonTable carbonTable = SchemaReader.readCarbonTableFromStore(absoluteTableIdentifier);
- setCarbonTable(configuration, carbonTable);
}
public static void setTablePath(Configuration configuration, String tablePath)
@@ -212,13 +212,17 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
* Set list of files to access
*/
public static void setFilesToAccess(Configuration configuration, List<String> validFiles) {
- configuration
- .set(CarbonInputFormat.INPUT_FILES, CarbonUtil.getSegmentString(validFiles));
+ configuration.set(CarbonInputFormat.INPUT_FILES, CarbonUtil.getSegmentString(validFiles));
}
- private static AbsoluteTableIdentifier getAbsoluteTableIdentifier(Configuration configuration)
+ private AbsoluteTableIdentifier getAbsoluteTableIdentifier(Configuration configuration)
throws IOException {
- return getCarbonTable(configuration).getAbsoluteTableIdentifier();
+ String dirs = configuration.get(INPUT_DIR, "");
+ String[] inputPaths = StringUtils.split(dirs);
+ if (inputPaths.length == 0) {
+ throw new InvalidPathException("No input paths specified in job");
+ }
+ return AbsoluteTableIdentifier.fromTablePath(inputPaths[0]);
}
/**
@@ -265,12 +269,7 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
// process and resolve the expression
Expression filter = getFilterPredicates(job.getConfiguration());
- CarbonTable carbonTable = getCarbonTable(job.getConfiguration());
- // this will be null in case of corrupt schema file.
- if (null == carbonTable) {
- throw new IOException("Missing/Corrupt schema file for table.");
- }
-
+ CarbonTable carbonTable = getOrCreateCarbonTable(job.getConfiguration());
CarbonInputFormatUtil.processFilterExpression(filter, carbonTable);
// prune partitions for filter query on partition table
@@ -343,17 +342,15 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
UpdateVO invalidBlockVOForSegmentId = null;
Boolean isIUDTable = false;
- AbsoluteTableIdentifier absoluteTableIdentifier =
- getCarbonTable(job.getConfiguration()).getAbsoluteTableIdentifier();
- SegmentUpdateStatusManager updateStatusManager =
- new SegmentUpdateStatusManager(absoluteTableIdentifier);
+ AbsoluteTableIdentifier identifier = getAbsoluteTableIdentifier(job.getConfiguration());
+ SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(identifier);
isIUDTable = (updateStatusManager.getUpdateStatusDetails().length != 0);
//for each segment fetch blocks matching filter in Driver BTree
for (String segmentNo : getSegmentsToAccess(job)) {
List<DataRefNode> dataRefNodes =
- getDataBlocksOfSegment(job, filterExpressionProcessor, absoluteTableIdentifier,
+ getDataBlocksOfSegment(job, filterExpressionProcessor, identifier,
filterResolver, matchedPartitions, segmentNo, cacheClient, updateStatusManager);
// Get the UpdateVO for those tables on which IUD operations being performed.
@@ -699,7 +696,7 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
public QueryModel getQueryModel(InputSplit inputSplit, TaskAttemptContext taskAttemptContext)
throws IOException {
Configuration configuration = taskAttemptContext.getConfiguration();
- CarbonTable carbonTable = getCarbonTable(configuration);
+ CarbonTable carbonTable = getOrCreateCarbonTable(configuration);
// getting the table absoluteTableIdentifier from the carbonTable
// to avoid unnecessary deserialization
AbsoluteTableIdentifier identifier = carbonTable.getAbsoluteTableIdentifier();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputMapperTest.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputMapperTest.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputMapperTest.java
index 9aa1188..86a3326 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputMapperTest.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputMapperTest.java
@@ -74,8 +74,8 @@ public class CarbonInputMapperTest extends TestCase {
Assert.assertEquals("Count lines are not matching", 1000, countTheLines(outPath));
Assert.assertEquals("Column count are not matching", 7, countTheColumns(outPath));
} catch (Exception e) {
- Assert.assertTrue("failed", false);
e.printStackTrace();
+ Assert.assertTrue("failed", false);
throw e;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
index 88c182e..8cad313 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
@@ -270,7 +270,6 @@ public class StoreCreator {
+ absoluteTableIdentifier.getCarbonTableIdentifier().getTableName());
tableInfo.setLastUpdatedTime(System.currentTimeMillis());
tableInfo.setFactTable(tableSchema);
- tableInfo.setAggregateTableList(new ArrayList<TableSchema>());
CarbonTablePath carbonTablePath = CarbonStorePath
.getCarbonTablePath(absoluteTableIdentifier.getStorePath(),
absoluteTableIdentifier.getCarbonTableIdentifier());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java b/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java
index 7a1c9db..202c444 100644
--- a/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java
+++ b/integration/hive/src/main/java/org/apache/carbondata/hive/MapredCarbonInputFormat.java
@@ -31,8 +31,10 @@ import org.apache.carbondata.hadoop.CarbonInputFormat;
import org.apache.carbondata.hadoop.CarbonInputSplit;
import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport;
import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil;
+import org.apache.carbondata.hadoop.util.SchemaReader;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
import org.apache.hadoop.io.ArrayWritable;
@@ -42,6 +44,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.util.StringUtils;
public class MapredCarbonInputFormat extends CarbonInputFormat<ArrayWritable>
implements InputFormat<Void, ArrayWritable>, CombineHiveInputFormat.AvoidSplitCombination {
@@ -68,6 +71,19 @@ public class MapredCarbonInputFormat extends CarbonInputFormat<ArrayWritable>
return new CarbonHiveRecordReader(queryModel, readSupport, inputSplit, jobConf);
}
+ private CarbonTable getCarbonTable(Configuration configuration) throws IOException {
+ String dirs = configuration.get(INPUT_DIR, "");
+ String[] inputPaths = StringUtils.split(dirs);
+ if (inputPaths.length == 0) {
+ throw new InvalidPathException("No input paths specified in job");
+ }
+ AbsoluteTableIdentifier absoluteTableIdentifier =
+ AbsoluteTableIdentifier.fromTablePath(inputPaths[0]);
+ // read the schema file to get the absoluteTableIdentifier having the correct table id
+ // persisted in the schema
+ return SchemaReader.readCarbonTableFromStore(absoluteTableIdentifier);
+ }
+
private QueryModel getQueryModel(Configuration configuration) throws IOException {
CarbonTable carbonTable = getCarbonTable(configuration);
// getting the table absoluteTableIdentifier from the carbonTable
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestTableIdTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestTableIdTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestTableIdTest.scala
deleted file mode 100644
index 3ccd2e3..0000000
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestTableIdTest.scala
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark.testsuite.createTable
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.mapred.JobConf
-import org.apache.hadoop.mapreduce.Job
-import org.apache.spark.sql.common.util.QueryTest
-import org.junit.Assert
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.hadoop.CarbonInputFormat
-
-/**
- * test functionality related the case change for database name
- */
-class TestTableIdTest extends QueryTest with BeforeAndAfterAll {
-
- override def beforeAll: Unit = {
- sql("drop table if exists carbontable")
- }
-
- def validateTableId: Unit = {
- val carbonInputFormat: CarbonInputFormat[Array[Object]] = new CarbonInputFormat[Array[Object]]
- val jobConf: JobConf = new JobConf(new Configuration)
- val job: Job = Job.getInstance(jobConf)
- val storePath: String = storeLocation.replaceAll("\\\\", "/")
- job.getConfiguration
- .set("mapreduce.input.fileinputformat.inputdir",
- storePath + "/default/carbontable")
- val carbonTable: CarbonTable = CarbonInputFormat.getCarbonTable(job.getConfiguration)
- val getAbsoluteTableIdentifier = classOf[CarbonInputFormat[Array[Object]]]
- .getDeclaredMethod("getAbsoluteTableIdentifier", classOf[Configuration])
- getAbsoluteTableIdentifier.setAccessible(true)
- val absoluteTableIdentifier: AbsoluteTableIdentifier = getAbsoluteTableIdentifier
- .invoke(carbonInputFormat, job.getConfiguration).asInstanceOf[AbsoluteTableIdentifier]
-
- Assert
- .assertEquals(carbonTable.getCarbonTableIdentifier.getTableId,
- absoluteTableIdentifier.getCarbonTableIdentifier.getTableId)
- }
-
- test("test create table with database case name change") {
-
- try {
- // table creation should be successful
- sql("create table carbontable(a int, b string)stored by 'carbondata'")
- assert(true)
- } catch {
- case ex: Exception =>
- assert(false)
- }
- validateTableId
- }
-
- override def afterAll {
- sql("drop table if exists carbontable")
- }
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonRDD.scala
index 106a9fd..05b3989 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonRDD.scala
@@ -17,11 +17,14 @@
package org.apache.carbondata.spark.rdd
+import java.io.{ByteArrayInputStream, DataInputStream}
+
import scala.reflect.ClassTag
import org.apache.spark.{Dependency, OneToOneDependency, Partition, SparkContext, TaskContext}
import org.apache.spark.rdd.RDD
+import org.apache.carbondata.core.metadata.schema.table.TableInfo
import org.apache.carbondata.core.util.{CarbonSessionInfo, SessionParams, ThreadLocalSessionInfo}
/**
@@ -44,3 +47,17 @@ abstract class CarbonRDD[T: ClassTag](@transient sc: SparkContext,
internalCompute(split, context)
}
}
+
+/**
+ * This RDD contains TableInfo object which is serialized and deserialized in driver and executor
+ */
+abstract class CarbonRDDWithTableInfo[T: ClassTag](
+ @transient sc: SparkContext,
+ @transient private var deps: Seq[Dependency[_]],
+ serializedTableInfo: Array[Byte]) extends CarbonRDD[T](sc, deps) {
+
+ def this(@transient oneParent: RDD[_], serializedTableInfo: Array[Byte]) =
+ this (oneParent.context, List(new OneToOneDependency(oneParent)), serializedTableInfo)
+
+ def getTableInfo: TableInfo = TableInfo.deserialize(serializedTableInfo)
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index 3868342..85c4bc4 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -26,7 +26,6 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.mapreduce._
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
import org.apache.spark._
-import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.hive.DistributionUtil
@@ -34,7 +33,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.block.Distributable
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.metadata.schema.table.TableInfo
import org.apache.carbondata.core.scan.expression.Expression
import org.apache.carbondata.core.scan.model.QueryModel
import org.apache.carbondata.core.stats.{QueryStatistic, QueryStatisticsConstants, QueryStatisticsRecorder}
@@ -53,8 +52,9 @@ class CarbonScanRDD(
columnProjection: CarbonProjection,
filterExpression: Expression,
identifier: AbsoluteTableIdentifier,
- @transient carbonTable: CarbonTable)
- extends CarbonRDD[InternalRow](sc, Nil) {
+ serializedTableInfo: Array[Byte],
+ @transient tableInfo: TableInfo)
+ extends CarbonRDDWithTableInfo[InternalRow](sc, Nil, serializedTableInfo) {
private val queryId = sparkContext.getConf.get("queryId", System.nanoTime() + "")
private val jobTrackerId: String = {
@@ -65,7 +65,7 @@ class CarbonScanRDD(
private val readSupport = SparkReadSupport.readSupportClass
- private val bucketedTable = carbonTable.getBucketingInfo(carbonTable.getFactTableName)
+ private val bucketedTable = tableInfo.getFactTable.getBucketingInfo
@transient private val jobId = new JobID(jobTrackerId, id)
@transient val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
@@ -174,6 +174,7 @@ class CarbonScanRDD(
}
override def internalCompute(split: Partition, context: TaskContext): Iterator[InternalRow] = {
+
val carbonPropertiesFilePath = System.getProperty("carbon.properties.filepath", null)
if (null == carbonPropertiesFilePath) {
System.setProperty("carbon.properties.filepath",
@@ -246,12 +247,13 @@ class CarbonScanRDD(
}
private def prepareInputFormatForDriver(conf: Configuration): CarbonInputFormat[Object] = {
- CarbonInputFormat.setCarbonTable(conf, carbonTable)
+ CarbonInputFormat.setTableInfo(conf, tableInfo)
createInputFormat(conf)
}
private def prepareInputFormatForExecutor(conf: Configuration): CarbonInputFormat[Object] = {
CarbonInputFormat.setCarbonReadSupport(conf, readSupport)
+ CarbonInputFormat.setTableInfo(conf, getTableInfo)
createInputFormat(conf)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
index 664cbae..954303f 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
@@ -266,7 +266,6 @@ object DataManagementFunc {
def prepareCarbonLoadModel(storePath: String,
table: CarbonTable,
newCarbonLoadModel: CarbonLoadModel): Unit = {
- newCarbonLoadModel.setAggTables(table.getAggregateTablesName.asScala.toArray)
newCarbonLoadModel.setTableName(table.getFactTableName)
val dataLoadSchema = new CarbonDataLoadSchema(table)
// Need to fill dimension relation
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index ee77f35..6174f7c 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -526,7 +526,6 @@ class TableNewProcessor(cm: TableModel) {
tableInfo.setTableUniqueName(cm.databaseName + "_" + cm.tableName)
tableInfo.setLastUpdatedTime(System.currentTimeMillis())
tableInfo.setFactTable(tableSchema)
- tableInfo.setAggregateTableList(new util.ArrayList[TableSchema]())
tableInfo
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
index 17d6065..3bd92e9 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
@@ -122,12 +122,14 @@ case class CarbonScan(
columnProjection.foreach { attr =>
projection.addColumn(attr.name)
}
+
new CarbonScanRDD(
ocRaw.sparkContext,
projection,
buildCarbonPlan.getFilterExpression,
carbonTable.getAbsoluteTableIdentifier,
- carbonTable
+ carbonTable.getTableInfo.serialize(),
+ carbonTable.getTableInfo
)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 44d5efb..da6dd98 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -128,7 +128,6 @@ private[sql] case class AlterTableCompaction(alterTableModel: AlterTableModel) e
val table = relation.tableMeta.carbonTable
- carbonLoadModel.setAggTables(table.getAggregateTablesName.asScala.toArray)
carbonLoadModel.setTableName(table.getFactTableName)
val dataLoadSchema = new CarbonDataLoadSchema(table)
// Need to fill dimension relation
@@ -404,7 +403,6 @@ case class LoadTable(
carbonLoadModel.setStorePath(relation.tableMeta.storePath)
val table = relation.tableMeta.carbonTable
- carbonLoadModel.setAggTables(table.getAggregateTablesName.asScala.toArray)
carbonLoadModel.setTableName(table.getFactTableName)
val dataLoadSchema = new CarbonDataLoadSchema(table)
// Need to fill dimension relation
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
index d28044f..d1baf79 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql
+import java.io.{ByteArrayOutputStream, DataOutputStream}
+
import scala.collection.mutable.ArrayBuffer
import org.apache.spark.rdd.RDD
@@ -28,15 +30,13 @@ import org.apache.spark.sql.types.StructType
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.scan.expression.Expression
import org.apache.carbondata.core.scan.expression.logical.AndExpression
-import org.apache.carbondata.core.util.{CarbonSessionInfo, SessionParams, ThreadLocalSessionInfo}
+import org.apache.carbondata.core.util.{CarbonSessionInfo, ThreadLocalSessionInfo}
import org.apache.carbondata.hadoop.CarbonProjection
-import org.apache.carbondata.hadoop.util.SchemaReader
-import org.apache.carbondata.processing.merger.TableMeta
import org.apache.carbondata.spark.CarbonFilters
import org.apache.carbondata.spark.rdd.CarbonScanRDD
-import org.apache.carbondata.spark.util.CarbonSparkUtil
case class CarbonDatasourceHadoopRelation(
sparkSession: SparkSession,
@@ -46,15 +46,22 @@ case class CarbonDatasourceHadoopRelation(
isSubquery: ArrayBuffer[Boolean] = new ArrayBuffer[Boolean]())
extends BaseRelation with InsertableRelation {
- lazy val absIdentifier = AbsoluteTableIdentifier.fromTablePath(paths.head)
- lazy val carbonTable = carbonRelation.tableMeta.carbonTable
- lazy val carbonRelation: CarbonRelation = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .lookupRelation(Some(absIdentifier.getCarbonTableIdentifier.getDatabaseName),
- absIdentifier.getCarbonTableIdentifier.getTableName)(sparkSession)
+ lazy val identifier: AbsoluteTableIdentifier = AbsoluteTableIdentifier.fromTablePath(paths.head)
+ lazy val databaseName: String = carbonTable.getDatabaseName
+ lazy val tableName: String = carbonTable.getFactTableName
+ lazy val carbonSessionInfo : CarbonSessionInfo =
+ CarbonEnv.getInstance(sparkSession).carbonSessionInfo
+ ThreadLocalSessionInfo.setCarbonSessionInfo(carbonSessionInfo)
+
+ @transient lazy val carbonRelation: CarbonRelation =
+ CarbonEnv.getInstance(sparkSession).carbonMetastore
+ .lookupRelation(
+ Some(identifier.getCarbonTableIdentifier.getDatabaseName),
+ identifier.getCarbonTableIdentifier.getTableName)(sparkSession)
.asInstanceOf[CarbonRelation]
- val carbonSessionInfo : CarbonSessionInfo = CarbonEnv.getInstance(sparkSession).carbonSessionInfo
- ThreadLocalSessionInfo.setCarbonSessionInfo(carbonSessionInfo)
+ @transient lazy val carbonTable: CarbonTable = carbonRelation.tableMeta.carbonTable
+
override def sqlContext: SQLContext = sparkSession.sqlContext
override def schema: StructType = tableSchema.getOrElse(carbonRelation.schema)
@@ -67,15 +74,20 @@ case class CarbonDatasourceHadoopRelation(
val projection = new CarbonProjection
requiredColumns.foreach(projection.addColumn)
- new CarbonScanRDD(sqlContext.sparkContext, projection, filterExpression.orNull,
- absIdentifier, carbonTable)
+ new CarbonScanRDD(
+ sqlContext.sparkContext,
+ projection,
+ filterExpression.orNull,
+ identifier,
+ carbonTable.getTableInfo.serialize(),
+ carbonTable.getTableInfo)
}
override def unhandledFilters(filters: Array[Filter]): Array[Filter] = new Array[Filter](0)
override def toString: String = {
- "CarbonDatasourceHadoopRelation [ " + "Database name :" + carbonTable.getDatabaseName +
- ", " + "Table name :" + carbonTable.getFactTableName + ", Schema :" + tableSchema + " ]"
+ "CarbonDatasourceHadoopRelation [ " + "Database name :" + databaseName +
+ ", " + "Table name :" + tableName + ", Schema :" + tableSchema + " ]"
}
override def sizeInBytes: Long = carbonRelation.sizeInBytes
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
index bd1c8b1..33091aa 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
@@ -40,7 +40,7 @@ import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.util.DataTypeUtil
import org.apache.carbondata.spark.CarbonAliasDecoderRelation
-import org.apache.carbondata.spark.rdd.CarbonRDD
+import org.apache.carbondata.spark.rdd.{CarbonRDD, CarbonRDDWithTableInfo}
/**
* It decodes the data.
@@ -444,10 +444,9 @@ class CarbonDecoderRDD(
aliasMap: CarbonAliasDecoderRelation,
prev: RDD[InternalRow],
output: Seq[Attribute],
- sparkSession: SparkSession)
- extends CarbonRDD[InternalRow](prev) {
-
- private val storepath = CarbonEnv.getInstance(sparkSession).carbonMetastore.storePath
+ storePath: String,
+ serializedTableInfo: Array[Byte])
+ extends CarbonRDDWithTableInfo[InternalRow](prev, serializedTableInfo) {
def canBeDecoded(attr: Attribute): Boolean = {
profile match {
@@ -516,13 +515,13 @@ class CarbonDecoderRDD(
override def internalCompute(split: Partition, context: TaskContext): Iterator[InternalRow] = {
val absoluteTableIdentifiers = relations.map { relation =>
- val carbonTable = relation.carbonRelation.carbonRelation.metaData.carbonTable
- (carbonTable.getFactTableName, carbonTable.getAbsoluteTableIdentifier)
+ val tableInfo = getTableInfo
+ (tableInfo.getFactTable.getTableName, tableInfo.getOrCreateAbsoluteTableIdentifier)
}.toMap
val cacheProvider: CacheProvider = CacheProvider.getInstance
val forwardDictionaryCache: Cache[DictionaryColumnUniqueIdentifier, Dictionary] =
- cacheProvider.createCache(CacheType.FORWARD_DICTIONARY, storepath)
+ cacheProvider.createCache(CacheType.FORWARD_DICTIONARY, storePath)
val dicts: Seq[Dictionary] = getDictionary(absoluteTableIdentifiers,
forwardDictionaryCache)
val dictIndex = dicts.zipWithIndex.filter(x => x._1 != null).map(x => x._2)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala
index a206bef..1cc6668 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala
@@ -17,9 +17,6 @@
package org.apache.spark.sql.execution
-import java.text.SimpleDateFormat
-import java.util.Date
-
import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
@@ -34,14 +31,13 @@ import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, Partition
import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.sql.optimizer.CarbonDecoderRelation
import org.apache.spark.sql.sources.{BaseRelation, Filter}
-import org.apache.spark.sql.types.{AtomicType, DoubleType, IntegerType, StringType, TimestampType}
+import org.apache.spark.sql.types.{AtomicType, IntegerType, StringType}
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampDirectDictionaryGenerator
import org.apache.carbondata.core.metadata.schema.BucketingInfo
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.spark.{CarbonAliasDecoderRelation}
+import org.apache.carbondata.spark.CarbonAliasDecoderRelation
import org.apache.carbondata.spark.rdd.CarbonScanRDD
import org.apache.carbondata.spark.util.CarbonScalaUtil
@@ -99,8 +95,15 @@ private[sql] class CarbonLateDecodeStrategy extends SparkStrategy {
relation.addAttribute(newAttr)
newAttr
}
- new CarbonDecoderRDD(Seq(relation), IncludeProfile(attrs),
- CarbonAliasDecoderRelation(), rdd, output, SparkSession.getActiveSession.get)
+
+ new CarbonDecoderRDD(
+ Seq(relation),
+ IncludeProfile(attrs),
+ CarbonAliasDecoderRelation(),
+ rdd,
+ output,
+ CarbonEnv.getInstance(SparkSession.getActiveSession.get).carbonMetastore.storePath,
+ table.carbonTable.getTableInfo.serialize())
}
private[this] def toCatalystRDD(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
index 0894f23..2fccd0c 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
@@ -795,8 +795,8 @@ object UpdateExecution {
def isDestinationRelation(relation: CarbonDatasourceHadoopRelation): Boolean = {
- val tableName = relation.absIdentifier.getCarbonTableIdentifier.getTableName
- val dbName = relation.absIdentifier.getCarbonTableIdentifier.getDatabaseName
+ val tableName = relation.identifier.getCarbonTableIdentifier.getTableName
+ val dbName = relation.identifier.getCarbonTableIdentifier.getDatabaseName
(tableIdentifier.size > 1 &&
tableIdentifier(0) == dbName &&
tableIdentifier(1) == tableName) ||
@@ -841,8 +841,8 @@ object UpdateExecution {
val header = getHeader(carbonRelation, plan)
LoadTable(
- Some(carbonRelation.absIdentifier.getCarbonTableIdentifier.getDatabaseName),
- carbonRelation.absIdentifier.getCarbonTableIdentifier.getTableName,
+ Some(carbonRelation.identifier.getCarbonTableIdentifier.getDatabaseName),
+ carbonRelation.identifier.getCarbonTableIdentifier.getTableName,
null,
Seq(),
Map(("fileheader" -> header)),
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 4b22cea..ce66733 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -126,7 +126,6 @@ case class AlterTableCompaction(alterTableModel: AlterTableModel) extends Runnab
val carbonLoadModel = new CarbonLoadModel()
val table = relation.tableMeta.carbonTable
- carbonLoadModel.setAggTables(table.getAggregateTablesName.asScala.toArray)
carbonLoadModel.setTableName(table.getFactTableName)
val dataLoadSchema = new CarbonDataLoadSchema(table)
// Need to fill dimension relation
@@ -495,7 +494,6 @@ case class LoadTable(
carbonLoadModel.setStorePath(relation.tableMeta.storePath)
val table = relation.tableMeta.carbonTable
- carbonLoadModel.setAggTables(table.getAggregateTablesName.asScala.toArray)
carbonLoadModel.setTableName(table.getFactTableName)
val dataLoadSchema = new CarbonDataLoadSchema(table)
// Need to fill dimension relation
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
index bfc1be9..b60085b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
@@ -488,13 +488,6 @@ public class CarbonLoadModel implements Serializable {
}
/**
- * @param aggTables the aggTables to set
- */
- public void setAggTables(String[] aggTables) {
- this.aggTables = aggTables;
- }
-
- /**
* @param storePath The storePath to set.
*/
public void setStorePath(String storePath) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4b69c9d8/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java b/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
index 91cc195..a7c2057 100644
--- a/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
+++ b/processing/src/test/java/org/apache/carbondata/processing/StoreCreator.java
@@ -261,7 +261,6 @@ public class StoreCreator {
+ absoluteTableIdentifier.getCarbonTableIdentifier().getTableName());
tableInfo.setLastUpdatedTime(System.currentTimeMillis());
tableInfo.setFactTable(tableSchema);
- tableInfo.setAggregateTableList(new ArrayList<TableSchema>());
CarbonTablePath carbonTablePath = CarbonStorePath
.getCarbonTablePath(absoluteTableIdentifier.getStorePath(),
[12/15] carbondata git commit: [CARBONDATA-1232] Datamap
implementation for Blocklet
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
index e4d3ba5..8aa18d5 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonTableReader.java
@@ -17,14 +17,31 @@
package org.apache.carbondata.presto.impl;
-import com.facebook.presto.spi.SchemaTableName;
-import com.facebook.presto.spi.classloader.ThreadContextClassLoader;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.inject.Inject;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.*;
-import org.apache.carbondata.core.datastore.block.*;
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.DataRefNodeFinder;
+import org.apache.carbondata.core.datastore.IndexKey;
+import org.apache.carbondata.core.datastore.SegmentTaskIndexStore;
+import org.apache.carbondata.core.datastore.TableSegmentUniqueIdentifier;
+import org.apache.carbondata.core.datastore.block.AbstractIndex;
+import org.apache.carbondata.core.datastore.block.BlockletInfos;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.exception.IndexBuilderException;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -52,18 +69,24 @@ import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.path.CarbonTablePath;
import org.apache.carbondata.hadoop.CacheClient;
import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil;
+
+import com.facebook.presto.spi.SchemaTableName;
+import com.facebook.presto.spi.classloader.ThreadContextClassLoader;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.Inject;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.thrift.TBase;
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
import static java.util.Objects.requireNonNull;
import com.facebook.presto.spi.TableNotFoundException;
@@ -392,8 +415,9 @@ public class CarbonTableReader {
TableBlockInfo tableBlockInfo = leafNode.getTableBlockInfo();
if (IUDTable) {
- if (CarbonUtil.isInvalidTableBlock(tableBlockInfo, invalidBlockVOForSegmentId,
- updateStatusManager)) {
+ if (CarbonUtil
+ .isInvalidTableBlock(tableBlockInfo.getSegmentId(), tableBlockInfo.getFilePath(),
+ invalidBlockVOForSegmentId, updateStatusManager)) {
continue;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
index 277005b..da0d082 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
@@ -29,7 +29,8 @@ import org.apache.spark.sql.execution.command.CarbonMergerMapping
import org.apache.carbondata.core.datastore.block.{Distributable, TableBlockInfo}
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
-import org.apache.carbondata.hadoop.{CarbonInputFormat, CarbonInputSplit, CarbonMultiBlockSplit}
+import org.apache.carbondata.hadoop.{CarbonInputSplit, CarbonMultiBlockSplit}
+import org.apache.carbondata.hadoop.api.CarbonTableInputFormat
import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil
import org.apache.carbondata.processing.merger.CarbonDataMergerUtil
import org.apache.carbondata.processing.model.CarbonLoadModel
@@ -71,7 +72,7 @@ class CarbonIUDMergerRDD[K, V](
var blocksOfLastSegment: List[TableBlockInfo] = null
- CarbonInputFormat.setSegmentsToAccess(
+ CarbonTableInputFormat.setSegmentsToAccess(
job.getConfiguration, carbonMergerMapping.validSegments.toList.asJava)
// get splits
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index bc5ca06..6bc7564 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -43,8 +43,8 @@ import org.apache.carbondata.core.mutate.UpdateVO
import org.apache.carbondata.core.scan.result.iterator.RawResultIterator
import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
-import org.apache.carbondata.core.util.path.CarbonTablePath
-import org.apache.carbondata.hadoop.{CarbonInputFormat, CarbonInputSplit, CarbonMultiBlockSplit}
+import org.apache.carbondata.hadoop.{CarbonInputSplit, CarbonMultiBlockSplit}
+import org.apache.carbondata.hadoop.api.CarbonTableInputFormat
import org.apache.carbondata.hadoop.util.{CarbonInputFormatUtil, CarbonInputSplitTaskInfo}
import org.apache.carbondata.processing.merger._
import org.apache.carbondata.processing.model.CarbonLoadModel
@@ -293,7 +293,7 @@ class CarbonMergerRDD[K, V](
for (eachSeg <- carbonMergerMapping.validSegments) {
// map for keeping the relation of a task and its blocks.
- job.getConfiguration.set(CarbonInputFormat.INPUT_SEGMENT_NUMBERS, eachSeg)
+ job.getConfiguration.set(CarbonTableInputFormat.INPUT_SEGMENT_NUMBERS, eachSeg)
if (updateStatusManager.getUpdateStatusDetails.length != 0) {
updateDetails = updateStatusManager.getInvalidTimestampRange(eachSeg)
@@ -315,7 +315,8 @@ class CarbonMergerRDD[K, V](
updateStatusManager.getDeleteDeltaFilePath(entry.getPath.toString)
)
((!updated) || ((updated) && (!CarbonUtil
- .isInvalidTableBlock(blockInfo, updateDetails, updateStatusManager))))
+ .isInvalidTableBlock(blockInfo.getSegmentId, blockInfo.getFilePath,
+ updateDetails, updateStatusManager))))
})
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index 85c4bc4..c383779 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -39,6 +39,7 @@ import org.apache.carbondata.core.scan.model.QueryModel
import org.apache.carbondata.core.stats.{QueryStatistic, QueryStatisticsConstants, QueryStatisticsRecorder}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory}
import org.apache.carbondata.hadoop._
+import org.apache.carbondata.hadoop.api.CarbonTableInputFormat
import org.apache.carbondata.spark.load.CarbonLoaderUtil
@@ -246,22 +247,23 @@ class CarbonScanRDD(
iterator.asInstanceOf[Iterator[InternalRow]]
}
- private def prepareInputFormatForDriver(conf: Configuration): CarbonInputFormat[Object] = {
- CarbonInputFormat.setTableInfo(conf, tableInfo)
+ private def prepareInputFormatForDriver(conf: Configuration): CarbonTableInputFormat[Object] = {
+ CarbonTableInputFormat.setTableInfo(conf, tableInfo)
createInputFormat(conf)
}
- private def prepareInputFormatForExecutor(conf: Configuration): CarbonInputFormat[Object] = {
- CarbonInputFormat.setCarbonReadSupport(conf, readSupport)
- CarbonInputFormat.setTableInfo(conf, getTableInfo)
+ private def prepareInputFormatForExecutor(conf: Configuration): CarbonTableInputFormat[Object] = {
+ CarbonTableInputFormat.setCarbonReadSupport(conf, readSupport)
+ CarbonTableInputFormat.setTableInfo(conf, tableInfo)
createInputFormat(conf)
}
- private def createInputFormat(conf: Configuration): CarbonInputFormat[Object] = {
- val format = new CarbonInputFormat[Object]
- CarbonInputFormat.setTablePath(conf, identifier.appendWithLocalPrefix(identifier.getTablePath))
- CarbonInputFormat.setFilterPredicates(conf, filterExpression)
- CarbonInputFormat.setColumnProjection(conf, columnProjection)
+ private def createInputFormat(conf: Configuration): CarbonTableInputFormat[Object] = {
+ val format = new CarbonTableInputFormat[Object]
+ CarbonTableInputFormat.setTablePath(conf,
+ identifier.appendWithLocalPrefix(identifier.getTablePath))
+ CarbonTableInputFormat.setFilterPredicates(conf, filterExpression)
+ CarbonTableInputFormat.setColumnProjection(conf, columnProjection)
format
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/integration/spark/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
index 2ca3b8c..4950227 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
@@ -26,7 +26,7 @@ import org.apache.hadoop.mapreduce.Job
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.hadoop.CarbonInputFormat
+import org.apache.carbondata.hadoop.api.CarbonTableInputFormat
/**
@@ -38,8 +38,8 @@ object QueryPlanUtil {
* createCarbonInputFormat from query model
*/
def createCarbonInputFormat(absoluteTableIdentifier: AbsoluteTableIdentifier) :
- (CarbonInputFormat[Array[Object]], Job) = {
- val carbonInputFormat = new CarbonInputFormat[Array[Object]]()
+ (CarbonTableInputFormat[Array[Object]], Job) = {
+ val carbonInputFormat = new CarbonTableInputFormat[Array[Object]]()
val jobConf: JobConf = new JobConf(new Configuration)
val job: Job = new Job(jobConf)
FileInputFormat.addInputPath(job, new Path(absoluteTableIdentifier.getTablePath))
@@ -47,8 +47,8 @@ object QueryPlanUtil {
}
def createCarbonInputFormat[V: ClassTag](absoluteTableIdentifier: AbsoluteTableIdentifier,
- conf: Configuration) : CarbonInputFormat[V] = {
- val carbonInputFormat = new CarbonInputFormat[V]()
+ conf: Configuration) : CarbonTableInputFormat[V] = {
+ val carbonInputFormat = new CarbonTableInputFormat[V]()
val job: Job = new Job(conf)
FileInputFormat.addInputPath(job, new Path(absoluteTableIdentifier.getTablePath))
carbonInputFormat
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
index 2fc93e6..ba89d51 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
@@ -38,7 +38,8 @@ import org.apache.spark.util.SerializableConfiguration
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.scan.expression.logical.AndExpression
import org.apache.carbondata.core.util.path.CarbonTablePath
-import org.apache.carbondata.hadoop.{CarbonInputFormat, CarbonInputSplit, CarbonProjection}
+import org.apache.carbondata.hadoop.{CarbonInputSplit, CarbonProjection}
+import org.apache.carbondata.hadoop.api.CarbonTableInputFormat
import org.apache.carbondata.hadoop.util.{CarbonInputFormatUtil, SchemaReader}
import org.apache.carbondata.processing.merger.TableMeta
import org.apache.carbondata.spark.{CarbonFilters, CarbonOption}
@@ -89,16 +90,17 @@ private[sql] case class CarbonDatasourceHadoopRelation(
filters.flatMap { filter =>
CarbonFilters.createCarbonFilter(dataSchema, filter)
}.reduceOption(new AndExpression(_, _))
- .foreach(CarbonInputFormat.setFilterPredicates(conf, _))
+ .foreach(CarbonTableInputFormat.setFilterPredicates(conf, _))
val projection = new CarbonProjection
requiredColumns.foreach(projection.addColumn)
- CarbonInputFormat.setColumnProjection(conf, projection)
- CarbonInputFormat.setCarbonReadSupport(conf, classOf[SparkRowReadSupportImpl])
+ CarbonTableInputFormat.setColumnProjection(conf, projection)
+ CarbonTableInputFormat.setCarbonReadSupport(conf, classOf[SparkRowReadSupportImpl])
+
new CarbonHadoopFSRDD[Row](sqlContext.sparkContext,
new SerializableConfiguration(conf),
absIdentifier,
- classOf[CarbonInputFormat[Row]],
+ classOf[CarbonTableInputFormat[Row]],
classOf[Row]
)
}
@@ -118,7 +120,7 @@ class CarbonHadoopFSRDD[V: ClassTag](
@transient sc: SparkContext,
conf: SerializableConfiguration,
identifier: AbsoluteTableIdentifier,
- inputFormatClass: Class[_ <: CarbonInputFormat[V]],
+ inputFormatClass: Class[_ <: CarbonTableInputFormat[V]],
valueClass: Class[V])
extends CarbonRDD[V](sc, Nil) with SparkHadoopMapReduceUtil {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
index a292cde..c38f0e1 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
@@ -133,13 +133,6 @@ private[sql] case class ProjectForUpdateCommand(
override def run(sqlContext: SQLContext): Seq[Row] = {
-
- // sqlContext.sparkContext.setLocalProperty(org.apache.spark.sql.execution.SQLExecution
- // .EXECUTION_ID_KEY, null)
- // DataFrame(sqlContext, plan).show(truncate = false)
- // return Seq.empty
-
-
val res = plan find {
case relation: LogicalRelation if (relation.relation
.isInstanceOf[CarbonDatasourceRelation]) =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
index 70c7caf..e0a8b58 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
@@ -26,7 +26,7 @@ import org.apache.hadoop.mapreduce.Job
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
-import org.apache.carbondata.hadoop.CarbonInputFormat
+import org.apache.carbondata.hadoop.api.CarbonTableInputFormat
/**
* All the utility functions for carbon plan creation
@@ -37,8 +37,8 @@ object QueryPlanUtil {
* createCarbonInputFormat from query model
*/
def createCarbonInputFormat(absoluteTableIdentifier: AbsoluteTableIdentifier) :
- (CarbonInputFormat[Array[Object]], Job) = {
- val carbonInputFormat = new CarbonInputFormat[Array[Object]]()
+ (CarbonTableInputFormat[Array[Object]], Job) = {
+ val carbonInputFormat = new CarbonTableInputFormat[Array[Object]]()
val jobConf: JobConf = new JobConf(new Configuration)
val job: Job = new Job(jobConf)
FileInputFormat.addInputPath(job, new Path(absoluteTableIdentifier.getTablePath))
@@ -46,8 +46,8 @@ object QueryPlanUtil {
}
def createCarbonInputFormat[V: ClassTag](absoluteTableIdentifier: AbsoluteTableIdentifier,
- conf: Configuration) : CarbonInputFormat[V] = {
- val carbonInputFormat = new CarbonInputFormat[V]()
+ conf: Configuration) : CarbonTableInputFormat[V] = {
+ val carbonInputFormat = new CarbonTableInputFormat[V]()
val job: Job = new Job(conf)
FileInputFormat.addInputPath(job, new Path(absoluteTableIdentifier.getTablePath))
carbonInputFormat
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
index 1788ccb..08b8600 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
@@ -18,6 +18,7 @@ package org.apache.carbondata.processing.merger;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -378,4 +379,35 @@ public class CarbonCompactionUtil {
}
return restructuredBlockExists;
}
+
+ /**
+ * This method will check for any restructured block in the blocks selected for compaction
+ *
+ * @param segmentMapping
+ * @param tableLastUpdatedTime
+ * @return
+ */
+ public static boolean checkIfAnyRestructuredBlockExists(Map<String, TaskBlockInfo> segmentMapping,
+ long tableLastUpdatedTime) {
+ boolean restructuredBlockExists = false;
+ for (Map.Entry<String, TaskBlockInfo> taskMap : segmentMapping.entrySet()) {
+ String segmentId = taskMap.getKey();
+ TaskBlockInfo taskBlockInfo = taskMap.getValue();
+ Collection<List<TableBlockInfo>> infoList = taskBlockInfo.getAllTableBlockInfoList();
+ for (List<TableBlockInfo> listMetadata : infoList) {
+ for (TableBlockInfo blockInfo : listMetadata) {
+ // if schema modified timestamp is greater than footer stored schema timestamp,
+ // it indicates it is a restructured block
+ if (tableLastUpdatedTime > blockInfo.getDetailInfo().getSchemaUpdatedTimeStamp()) {
+ restructuredBlockExists = true;
+ break;
+ }
+ }
+ }
+ if (restructuredBlockExists) {
+ break;
+ }
+ }
+ return restructuredBlockExists;
+ }
}
[11/15] carbondata git commit: [CARBONDATA-1307] Fix bug for
TableInfo serialization problem in cluster mode
Posted by ra...@apache.org.
[CARBONDATA-1307] Fix bug for TableInfo serialization problem in cluster mode
TableInfo serialization has problem when there are scala list in ColumnSchema, this PR fixed this issue
This closes #1172
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8d3c9bf1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8d3c9bf1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8d3c9bf1
Branch: refs/heads/datamap
Commit: 8d3c9bf1b09fa3c15ff5d4e707f5a0e657d1ff74
Parents: 5b49afc
Author: jackylk <ja...@huawei.com>
Authored: Fri Jul 14 16:25:12 2017 +0800
Committer: Ravindra Pesala <ra...@gmail.com>
Committed: Sat Jul 15 15:26:11 2017 +0530
----------------------------------------------------------------------
.../carbondata/hadoop/CarbonInputFormat.java | 15 +++++++++++++--
.../hadoop/util/ObjectSerializationUtil.java | 20 ++++++++++++--------
2 files changed, 25 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d3c9bf1/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index 787f571..1673193 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -16,6 +16,8 @@
*/
package org.apache.carbondata.hadoop;
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
@@ -114,7 +116,7 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
public static void setTableInfo(Configuration configuration, TableInfo tableInfo)
throws IOException {
if (null != tableInfo) {
- configuration.set(TABLE_INFO, ObjectSerializationUtil.convertObjectToString(tableInfo));
+ configuration.set(TABLE_INFO, ObjectSerializationUtil.encodeToString(tableInfo.serialize()));
}
}
@@ -123,7 +125,16 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
*/
private TableInfo getTableInfo(Configuration configuration) throws IOException {
String tableInfoStr = configuration.get(TABLE_INFO);
- return (TableInfo) ObjectSerializationUtil.convertStringToObject(tableInfoStr);
+ if (tableInfoStr == null) {
+ return null;
+ } else {
+ TableInfo output = new TableInfo();
+ output.readFields(
+ new DataInputStream(
+ new ByteArrayInputStream(
+ ObjectSerializationUtil.decodeStringToBytes(tableInfoStr))));
+ return output;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d3c9bf1/hadoop/src/main/java/org/apache/carbondata/hadoop/util/ObjectSerializationUtil.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/ObjectSerializationUtil.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/ObjectSerializationUtil.java
index fab729e..ea3246f 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/ObjectSerializationUtil.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/ObjectSerializationUtil.java
@@ -16,11 +16,7 @@
*/
package org.apache.carbondata.hadoop.util;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
+import java.io.*;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
@@ -70,7 +66,11 @@ public class ObjectSerializationUtil {
}
}
- return new String(Base64.encodeBase64(baos.toByteArray()),
+ return encodeToString(baos.toByteArray());
+ }
+
+ public static String encodeToString(byte[] bytes) throws UnsupportedEncodingException {
+ return new String(Base64.encodeBase64(bytes),
CarbonCommonConstants.DEFAULT_CHARSET);
}
@@ -86,8 +86,7 @@ public class ObjectSerializationUtil {
return null;
}
- byte[] bytes =
- Base64.decodeBase64(objectString.getBytes(CarbonCommonConstants.DEFAULT_CHARSET));
+ byte[] bytes = decodeStringToBytes(objectString);
ByteArrayInputStream bais = null;
GZIPInputStream gis = null;
@@ -116,4 +115,9 @@ public class ObjectSerializationUtil {
}
}
}
+
+ public static byte[] decodeStringToBytes(String objectString)
+ throws UnsupportedEncodingException {
+ return Base64.decodeBase64(objectString.getBytes(CarbonCommonConstants.DEFAULT_CHARSET));
+ }
}
[02/15] carbondata git commit: [CARBONDATA-757] Big decimal
optimization This closes #1117
Posted by ra...@apache.org.
[CARBONDATA-757] Big decimal optimization This closes #1117
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/31a6ec6a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/31a6ec6a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/31a6ec6a
Branch: refs/heads/datamap
Commit: 31a6ec6a71ac72b568a99a04a0d0f187e9f07a63
Parents: c7e7f70 1291162
Author: jackylk <ja...@huawei.com>
Authored: Tue Jul 11 15:50:49 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Tue Jul 11 15:50:49 2017 +0800
----------------------------------------------------------------------
.../carbondata/core/datastore/TableSpec.java | 23 +-
.../AbstractMeasureChunkReaderV2V3Format.java | 15 +-
...CompressedMeasureChunkFileBasedReaderV1.java | 2 +-
.../core/datastore/page/ColumnPage.java | 107 ++++++---
.../core/datastore/page/LazyColumnPage.java | 13 +-
.../datastore/page/SafeFixLengthColumnPage.java | 14 +-
.../datastore/page/SafeVarLengthColumnPage.java | 11 +-
.../page/UnsafeFixLengthColumnPage.java | 13 +-
.../page/UnsafeVarLengthColumnPage.java | 19 +-
.../datastore/page/VarLengthColumnPageBase.java | 76 +++++-
.../page/encoding/AdaptiveIntegerCodec.java | 11 +-
.../page/encoding/DefaultEncodingStrategy.java | 8 +-
.../page/encoding/DeltaIntegerCodec.java | 23 +-
.../page/encoding/DirectCompressCodec.java | 16 +-
.../page/encoding/EncodingStrategy.java | 4 +-
.../page/statistics/ColumnPageStatsVO.java | 18 +-
.../carbondata/core/locks/CarbonLockUtil.java | 24 ++
.../datatype/DecimalConverterFactory.java | 237 +++++++++++++++++++
.../core/util/CarbonMetadataUtil.java | 28 +++
docs/configuration-parameters.md | 3 +-
integration/hive/pom.xml | 18 ++
.../test/resources/short_int_as_target_type.csv | 3 +
.../aggquery/IntegerDataTypeTestCase.scala | 33 ++-
.../execution/CarbonLateDecodeStrategy.scala | 4 +-
.../execution/command/carbonTableSchema.scala | 52 ++--
.../org/apache/spark/util/AlterTableUtil.scala | 25 +-
pom.xml | 6 +-
.../newflow/sort/SortStepRowUtil.java | 17 +-
.../sort/unsafe/UnsafeCarbonRowPage.java | 2 +-
.../holder/UnsafeSortTempFileChunkHolder.java | 3 +-
.../CarbonRowDataWriterProcessorStepImpl.java | 14 +-
.../sortdata/SortTempFileChunkHolder.java | 3 +-
.../carbondata/processing/store/TablePage.java | 14 +-
33 files changed, 680 insertions(+), 179 deletions(-)
----------------------------------------------------------------------
[04/15] carbondata git commit: [CARBONDATA-1273] String datatype will
be no dictionary column by default
Posted by ra...@apache.org.
[CARBONDATA-1273] String datatype will be no dictionary column by default
This closes #1144
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/191dfb35
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/191dfb35
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/191dfb35
Branch: refs/heads/datamap
Commit: 191dfb3513f54ae22a0766e7c00e66e43eb42cb4
Parents: 0481340
Author: QiangCai <da...@gmail.com>
Authored: Thu Jul 6 10:41:55 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Tue Jul 11 16:28:17 2017 +0800
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 23 ---
.../carbondata/core/util/CarbonProperties.java | 39 ----
.../InsertIntoCarbonTableTestCase.scala | 2 +-
.../dataload/TestBatchSortDataLoad.scala | 5 +-
.../dataload/TestLoadDataUseAllDictionary.scala | 1 +
.../partition/TestDDLForPartitionTable.scala | 5 +-
.../testsuite/sortcolumns/TestSortColumns.scala | 4 +-
.../sortcolumns/TestSortColumnsWithUnsafe.scala | 6 +-
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 143 ++++++--------
.../spark/util/GlobalDictionaryUtil.scala | 24 +--
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 16 +-
.../spark/util/AllDictionaryTestCase.scala | 7 +-
.../AutoHighCardinalityIdentifyTestCase.scala | 148 --------------
.../util/ExternalColumnDictionaryTestCase.scala | 4 +-
...GlobalDictionaryUtilConcurrentTestCase.scala | 3 +-
.../apache/spark/sql/TestCarbonSqlParser.scala | 191 ++++++-------------
.../spark/util/AllDictionaryTestCase.scala | 5 +-
.../util/ExternalColumnDictionaryTestCase.scala | 4 +-
.../sortdata/SortTempFileChunkHolder.java | 2 +-
.../store/writer/AbstractFactDataWriter.java | 4 +-
20 files changed, 158 insertions(+), 478 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/191dfb35/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 208bab8..6393131 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -870,29 +870,6 @@ public final class CarbonCommonConstants {
public static final String NO_INVERTED_INDEX = "no_inverted_index";
/**
- * this variable is to enable/disable identify high cardinality during first data loading
- */
- @CarbonProperty
- public static final String HIGH_CARDINALITY_IDENTIFY_ENABLE = "high.cardinality.identify.enable";
- public static final String HIGH_CARDINALITY_IDENTIFY_ENABLE_DEFAULT = "true";
-
- /**
- * threshold of high cardinality
- */
- @CarbonProperty
- public static final String HIGH_CARDINALITY_THRESHOLD = "high.cardinality.threshold";
- public static final String HIGH_CARDINALITY_THRESHOLD_DEFAULT = "1000000";
- public static final int HIGH_CARDINALITY_THRESHOLD_MIN = 10000;
-
- /**
- * percentage of cardinality in row count
- */
- @CarbonProperty
- public static final String HIGH_CARDINALITY_IN_ROW_COUNT_PERCENTAGE =
- "high.cardinality.row.count.percentage";
- public static final String HIGH_CARDINALITY_IN_ROW_COUNT_PERCENTAGE_DEFAULT = "80";
-
- /**
* 16 mb size
*/
public static final long CARBON_16MB = 16 * 1024 * 1024;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/191dfb35/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index c1e70ff..d14b7ab 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -90,8 +90,6 @@ public final class CarbonProperties {
validateNumCores();
validateNumCoresBlockSort();
validateSortSize();
- validateHighCardinalityIdentify();
- validateHighCardinalityThreshold();
validateCarbonDataFileVersion();
validateExecutorStartUpTime();
validatePrefetchBufferSize();
@@ -329,43 +327,6 @@ public final class CarbonProperties {
}
}
- private void validateHighCardinalityIdentify() {
- String highcardIdentifyStr = carbonProperties
- .getProperty(CarbonCommonConstants.HIGH_CARDINALITY_IDENTIFY_ENABLE,
- CarbonCommonConstants.HIGH_CARDINALITY_IDENTIFY_ENABLE_DEFAULT);
- try {
- Boolean.parseBoolean(highcardIdentifyStr);
- } catch (NumberFormatException e) {
- LOGGER.info("The high cardinality identify value \"" + highcardIdentifyStr
- + "\" is invalid. Using the default value \""
- + CarbonCommonConstants.HIGH_CARDINALITY_IDENTIFY_ENABLE_DEFAULT);
- carbonProperties.setProperty(CarbonCommonConstants.HIGH_CARDINALITY_IDENTIFY_ENABLE,
- CarbonCommonConstants.HIGH_CARDINALITY_IDENTIFY_ENABLE_DEFAULT);
- }
- }
-
- private void validateHighCardinalityThreshold() {
- String highcardThresholdStr = carbonProperties
- .getProperty(CarbonCommonConstants.HIGH_CARDINALITY_THRESHOLD,
- CarbonCommonConstants.HIGH_CARDINALITY_THRESHOLD_DEFAULT);
- try {
- int highcardThreshold = Integer.parseInt(highcardThresholdStr);
- if (highcardThreshold < CarbonCommonConstants.HIGH_CARDINALITY_THRESHOLD_MIN) {
- LOGGER.info("The high cardinality threshold value \"" + highcardThresholdStr
- + "\" is invalid. Using the min value \""
- + CarbonCommonConstants.HIGH_CARDINALITY_THRESHOLD_MIN);
- carbonProperties.setProperty(CarbonCommonConstants.HIGH_CARDINALITY_THRESHOLD,
- CarbonCommonConstants.HIGH_CARDINALITY_THRESHOLD_MIN + "");
- }
- } catch (NumberFormatException e) {
- LOGGER.info("The high cardinality threshold value \"" + highcardThresholdStr
- + "\" is invalid. Using the default value \""
- + CarbonCommonConstants.HIGH_CARDINALITY_THRESHOLD_DEFAULT);
- carbonProperties.setProperty(CarbonCommonConstants.HIGH_CARDINALITY_THRESHOLD,
- CarbonCommonConstants.HIGH_CARDINALITY_THRESHOLD_DEFAULT);
- }
- }
-
/**
* Below method will be used to validate the data file version parameter
* if parameter is invalid current version will be set
http://git-wip-us.apache.org/repos/asf/carbondata/blob/191dfb35/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
index c968672..7bd9094 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableTestCase.scala
@@ -39,7 +39,7 @@ class InsertIntoCarbonTableTestCase extends QueryTest with BeforeAndAfterAll {
checkAnswer(
sql("select imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_oper
atorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription from THive order by imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Late
st_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription"),
sql("select imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,Latest_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_oper
atorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription from TCarbon order by imei,deviceInformationId,MAC,deviceColor,device_backColor,modelId,marketName,AMSize,ROMSize,CUPAudit,CPIClocked,series,productionDate,bomCode,internalModels,deliveryTime,channelsId,channelsName,deliveryAreaId,deliveryCountry,deliveryProvince,deliveryCity,deliveryDistrict,deliveryStreet,oxSingleNumber,contractNumber,ActiveCheckTime,ActiveAreaId,ActiveCountry,ActiveProvince,Activecity,ActiveDistrict,ActiveStreet,ActiveOperatorId,Active_releaseId,Active_EMUIVersion,Active_operaSysVersion,Active_BacVerNumber,Active_BacFlashVer,Active_webUIVersion,Active_webUITypeCarrVer,Active_webTypeDataVerNumber,Active_operatorsVersion,Active_phonePADPartitionedVersions,Latest_YEAR,Latest_MONTH,Latest_DAY,Latest_HOUR,Latest_areaId,Latest_country,Latest_province,Latest_city,Latest_district,Latest_street,Latest_releaseId,Latest_EMUIVersion,Latest_operaSysVersion,Latest_BacVerNumber,La
test_BacFlashVer,Latest_webUIVersion,Latest_webUITypeCarrVer,Latest_webTypeDataVerNumber,Latest_operatorsVersion,Latest_phonePADPartitionedVersions,Latest_operatorId,gamePointId,gamePointDescription")
- )
+ )
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, timeStampPropOrig)
}
test("insert from hive-sum expression") {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/191dfb35/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
index d53b5e5..c5a6a27 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestBatchSortDataLoad.scala
@@ -29,7 +29,6 @@ import org.apache.carbondata.core.util.CarbonProperties
class TestBatchSortDataLoad extends QueryTest with BeforeAndAfterAll {
var filePath: String = _
-
def buildTestData() = {
filePath = s"${integrationPath}/spark-common-test/target/big.csv"
val file = new File(filePath)
@@ -79,6 +78,7 @@ class TestBatchSortDataLoad extends QueryTest with BeforeAndAfterAll {
| CREATE TABLE carbon_load1(c1 string, c2 string, c3 string, c4 string, c5 string,
| c6 string, c7 int, c8 int, c9 int, c10 int)
| STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('dictionary_include'='c1,c2,c3,c4,c5,c6')
""".stripMargin)
sql(s"LOAD DATA LOCAL INPATH '$filePath' into table carbon_load1 " +
@@ -160,6 +160,7 @@ class TestBatchSortDataLoad extends QueryTest with BeforeAndAfterAll {
| CREATE TABLE carbon_load3(c1 string, c2 string, c3 string, c4 string, c5 string,
| c6 string, c7 int, c8 int, c9 int, c10 int)
| STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('dictionary_include'='c1,c2,c3,c4,c5,c6')
""".stripMargin)
sql(s"LOAD DATA LOCAL INPATH '$filePath' into table carbon_load3 " +
@@ -182,6 +183,7 @@ class TestBatchSortDataLoad extends QueryTest with BeforeAndAfterAll {
| CREATE TABLE carbon_load4(c1 string, c2 string, c3 string, c4 string, c5 string,
| c6 string, c7 int, c8 int, c9 int, c10 int)
| STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('dictionary_include'='c1,c2,c3,c4,c5,c6')
""".stripMargin)
sql(s"LOAD DATA LOCAL INPATH '$filePath' into table carbon_load4 " )
@@ -202,6 +204,7 @@ class TestBatchSortDataLoad extends QueryTest with BeforeAndAfterAll {
| CREATE TABLE carbon_load6(c1 string, c2 string, c3 string, c4 string, c5 string,
| c6 string, c7 int, c8 int, c9 int, c10 int)
| STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('dictionary_include'='c1,c2,c3,c4,c5,c6')
""".stripMargin)
sql(s"LOAD DATA LOCAL INPATH '$filePath' into table carbon_load6 " )
http://git-wip-us.apache.org/repos/asf/carbondata/blob/191dfb35/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataUseAllDictionary.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataUseAllDictionary.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataUseAllDictionary.scala
index d6deb89..170c41b 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataUseAllDictionary.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataUseAllDictionary.scala
@@ -30,6 +30,7 @@ class TestLoadDataUseAllDictionary extends QueryTest with BeforeAndAfterAll{
(ID Int, date Timestamp, country String,
name String, phonetype String, serialname String, salary Int)
STORED BY 'carbondata'
+ TBLPROPERTIES('dictionary_include'='country,name,phonetype,serialname')
""")
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/191dfb35/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
index 9ad5959..5b77a24 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
@@ -106,9 +106,8 @@ class TestDDLForPartitionTable extends QueryTest with BeforeAndAfterAll {
assert(partitionInfo != null)
assert(partitionInfo.getColumnSchemaList.get(0).getColumnName.equalsIgnoreCase("workgroupcategory"))
assert(partitionInfo.getColumnSchemaList.get(0).getDataType == DataType.STRING)
- assert(partitionInfo.getColumnSchemaList.get(0).getEncodingList.size == 2)
- assert(partitionInfo.getColumnSchemaList.get(0).getEncodingList.get(0) == Encoding.DICTIONARY)
- assert(partitionInfo.getColumnSchemaList.get(0).getEncodingList.get(1) == Encoding.INVERTED_INDEX)
+ assert(partitionInfo.getColumnSchemaList.get(0).getEncodingList.size == 1)
+ assert(partitionInfo.getColumnSchemaList.get(0).getEncodingList.get(0) == Encoding.INVERTED_INDEX)
assert(partitionInfo.getPartitionType == PartitionType.LIST)
assert(partitionInfo.getListInfo.size == 3)
assert(partitionInfo.getListInfo.get(0).size == 1)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/191dfb35/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
index 01c8021..e25ebb8 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
@@ -72,7 +72,7 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
}
test("create table with dictionary sort_columns") {
- sql("CREATE TABLE sorttable2 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='empname')")
+ sql("CREATE TABLE sorttable2 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='empname', 'dictionary_include'='empname')")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable2 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
checkAnswer(sql("select empname from sorttable2"),sql("select empname from origintable1"))
}
@@ -178,7 +178,7 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
}
test("filter on sort_columns include no-dictionary, direct-dictionary and dictioanry") {
- sql("CREATE TABLE sorttable6 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='workgroupcategory, doj, empname')")
+ sql("CREATE TABLE sorttable6 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='workgroupcategory, doj, empname', 'dictionary_include'='empname')")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable6 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
// no dictionary
checkAnswer(sql("select * from sorttable6 where workgroupcategory = 1"), sql("select * from origintable1 where workgroupcategory = 1 order by doj"))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/191dfb35/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumnsWithUnsafe.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumnsWithUnsafe.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumnsWithUnsafe.scala
index 7ffa13f..f45f931 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumnsWithUnsafe.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumnsWithUnsafe.scala
@@ -35,11 +35,11 @@ class TestSortColumnsWithUnsafe extends QueryTest with BeforeAndAfterAll {
test("create table with no dictionary sort_columns") {
sql("CREATE TABLE sorttable1 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='empno')")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable1 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
- checkAnswer(sql("select empno from sorttable1"), sql("select empno from sorttable1 order by empno"))
+ checkAnswer(sql("select empno from sorttable1"), sql("select empno from origintable1 order by empno"))
}
test("create table with dictionary sort_columns") {
- sql("CREATE TABLE sorttable2 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='empname')")
+ sql("CREATE TABLE sorttable2 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='empname', 'dictionary_include'='empname')")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable2 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
checkAnswer(sql("select empname from sorttable2"),sql("select empname from origintable1"))
}
@@ -145,7 +145,7 @@ class TestSortColumnsWithUnsafe extends QueryTest with BeforeAndAfterAll {
}
test("filter on sort_columns include no-dictionary, direct-dictionary and dictioanry") {
- sql("CREATE TABLE sorttable6 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='workgroupcategory, doj, empname')")
+ sql("CREATE TABLE sorttable6 (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='workgroupcategory, doj, empname', 'dictionary_include'='empname')")
sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE sorttable6 OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
// no dictionary
checkAnswer(sql("select * from sorttable6 where workgroupcategory = 1"), sql("select * from origintable1 where workgroupcategory = 1 order by doj"))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/191dfb35/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index d0f9362..a7b8143 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@ -153,8 +153,6 @@ case class DictionaryLoadModel(table: CarbonTableIdentifier,
isComplexes: Array[Boolean],
primDimensions: Array[CarbonDimension],
delimiters: Array[String],
- highCardIdentifyEnable: Boolean,
- highCardThreshold: Int,
columnIdentifier: Array[ColumnIdentifier],
isFirstLoad: Boolean,
hdfsTempLocation: String,
@@ -329,18 +327,17 @@ class CarbonBlockDistinctValuesCombineRDD(
class CarbonGlobalDictionaryGenerateRDD(
prev: RDD[(Int, ColumnDistinctValues)],
model: DictionaryLoadModel)
- extends CarbonRDD[(Int, String, Boolean)](prev) {
+ extends CarbonRDD[(Int, String)](prev) {
override def getPartitions: Array[Partition] = firstParent[(Int, ColumnDistinctValues)].partitions
override def internalCompute(split: Partition,
- context: TaskContext): Iterator[(Int, String, Boolean)] = {
+ context: TaskContext): Iterator[(Int, String)] = {
val LOGGER = LogServiceFactory.getLogService(this.getClass.getName)
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.STORE_LOCATION,
model.hdfsLocation)
val status = CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS
- var isHighCardinalityColumn = false
- val iter = new Iterator[(Int, String, Boolean)] {
+ val iter = new Iterator[(Int, String)] {
var dictionaryForDistinctValueLookUp: Dictionary = _
var dictionaryForSortIndexWriting: Dictionary = _
var dictionaryForDistinctValueLookUpCleared: Boolean = false
@@ -375,90 +372,70 @@ class CarbonGlobalDictionaryGenerateRDD(
val distinctValueList = rddIter.next()._2
valuesBuffer ++= distinctValueList.values
rowCount += distinctValueList.rowCount
- // check high cardinality
- if (model.isFirstLoad && model.highCardIdentifyEnable
- && !model.isComplexes(split.index)
- && model.primDimensions(split.index).isColumnar) {
- isHighCardinalityColumn = GlobalDictionaryUtil.isHighCardinalityColumn(
- valuesBuffer.size, model)
- if (isHighCardinalityColumn) {
- break
- }
- }
}
}
val combineListTime = System.currentTimeMillis() - t1
- if (isHighCardinalityColumn) {
- LOGGER.info(s"column ${ model.table.getTableUniqueName }." +
- s"${
- model.primDimensions(split.index)
- .getColName
- } is high cardinality column")
+ isDictionaryLocked = dictLock.lockWithRetries()
+ if (isDictionaryLocked) {
+ logInfo(s"Successfully able to get the dictionary lock for ${
+ model.primDimensions(split.index).getColName
+ }")
} else {
- isDictionaryLocked = dictLock.lockWithRetries()
- if (isDictionaryLocked) {
- logInfo(s"Successfully able to get the dictionary lock for ${
+ sys.error(s"Dictionary file ${
model.primDimensions(split.index).getColName
- }")
- } else {
- sys
- .error(s"Dictionary file ${
- model.primDimensions(split.index).getColName
- } is locked for updation. Please try after some time")
- }
- val t2 = System.currentTimeMillis
- val fileType = FileFactory.getFileType(model.dictFilePaths(split.index))
- model.dictFileExists(split.index) = FileFactory
- .isFileExist(model.dictFilePaths(split.index), fileType)
- dictionaryForDistinctValueLookUp = if (model.dictFileExists(split.index)) {
- CarbonLoaderUtil.getDictionary(model.table,
- model.columnIdentifier(split.index),
- model.hdfsLocation,
- model.primDimensions(split.index).getDataType
- )
- } else {
- null
- }
- val dictCacheTime = System.currentTimeMillis - t2
- val t3 = System.currentTimeMillis()
- val dictWriteTask = new DictionaryWriterTask(valuesBuffer,
- dictionaryForDistinctValueLookUp,
- model.table,
+ } is locked for updation. Please try after some time")
+ }
+ val t2 = System.currentTimeMillis
+ val fileType = FileFactory.getFileType(model.dictFilePaths(split.index))
+ val isDictFileExists = FileFactory.isFileExist(model.dictFilePaths(split.index), fileType)
+ dictionaryForDistinctValueLookUp = if (isDictFileExists) {
+ CarbonLoaderUtil.getDictionary(model.table,
model.columnIdentifier(split.index),
model.hdfsLocation,
- model.primDimensions(split.index).getColumnSchema,
- model.dictFileExists(split.index)
+ model.primDimensions(split.index).getDataType
)
- // execute dictionary writer task to get distinct values
- val distinctValues = dictWriteTask.execute()
- val dictWriteTime = System.currentTimeMillis() - t3
- val t4 = System.currentTimeMillis()
- // if new data came than rewrite sort index file
- if (distinctValues.size() > 0) {
- val sortIndexWriteTask = new SortIndexWriterTask(model.table,
- model.columnIdentifier(split.index),
- model.primDimensions(split.index).getDataType,
- model.hdfsLocation,
- dictionaryForDistinctValueLookUp,
- distinctValues)
- sortIndexWriteTask.execute()
- }
- val sortIndexWriteTime = System.currentTimeMillis() - t4
- CarbonTimeStatisticsFactory.getLoadStatisticsInstance.recordDicShuffleAndWriteTime()
- // After sortIndex writing, update dictionaryMeta
- dictWriteTask.updateMetaData()
- // clear the value buffer after writing dictionary data
- valuesBuffer.clear
- CarbonUtil.clearDictionaryCache(dictionaryForDistinctValueLookUp)
- dictionaryForDistinctValueLookUpCleared = true
- LOGGER.info(s"\n columnName: ${ model.primDimensions(split.index).getColName }" +
- s"\n columnId: ${ model.primDimensions(split.index).getColumnId }" +
- s"\n new distinct values count: ${ distinctValues.size() }" +
- s"\n combine lists: $combineListTime" +
- s"\n create dictionary cache: $dictCacheTime" +
- s"\n sort list, distinct and write: $dictWriteTime" +
- s"\n write sort info: $sortIndexWriteTime")
+ } else {
+ null
+ }
+ val dictCacheTime = System.currentTimeMillis - t2
+ val t3 = System.currentTimeMillis()
+ val dictWriteTask = new DictionaryWriterTask(valuesBuffer,
+ dictionaryForDistinctValueLookUp,
+ model.table,
+ model.columnIdentifier(split.index),
+ model.hdfsLocation,
+ model.primDimensions(split.index).getColumnSchema,
+ isDictFileExists
+ )
+ // execute dictionary writer task to get distinct values
+ val distinctValues = dictWriteTask.execute()
+ val dictWriteTime = System.currentTimeMillis() - t3
+ val t4 = System.currentTimeMillis()
+ // if new data came than rewrite sort index file
+ if (distinctValues.size() > 0) {
+ val sortIndexWriteTask = new SortIndexWriterTask(model.table,
+ model.columnIdentifier(split.index),
+ model.primDimensions(split.index).getDataType,
+ model.hdfsLocation,
+ dictionaryForDistinctValueLookUp,
+ distinctValues)
+ sortIndexWriteTask.execute()
}
+ val sortIndexWriteTime = System.currentTimeMillis() - t4
+ CarbonTimeStatisticsFactory.getLoadStatisticsInstance.recordDicShuffleAndWriteTime()
+ // After sortIndex writing, update dictionaryMeta
+ dictWriteTask.updateMetaData()
+ // clear the value buffer after writing dictionary data
+ valuesBuffer.clear
+ CarbonUtil.clearDictionaryCache(dictionaryForDistinctValueLookUp)
+ dictionaryForDistinctValueLookUpCleared = true
+ LOGGER.info(s"\n columnName: ${ model.primDimensions(split.index).getColName }" +
+ s"\n columnId: ${ model.primDimensions(split.index).getColumnId }" +
+ s"\n new distinct values count: ${ distinctValues.size() }" +
+ s"\n combine lists: $combineListTime" +
+ s"\n create dictionary cache: $dictCacheTime" +
+ s"\n sort list, distinct and write: $dictWriteTime" +
+ s"\n write sort info: $sortIndexWriteTime")
} catch {
case ex: Exception =>
LOGGER.error(ex)
@@ -492,8 +469,8 @@ class CarbonGlobalDictionaryGenerateRDD(
}
}
- override def next(): (Int, String, Boolean) = {
- (split.index, status, isHighCardinalityColumn)
+ override def next(): (Int, String) = {
+ (split.index, status)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/191dfb35/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index bf41dca..e1c564d 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -278,11 +278,6 @@ object GlobalDictionaryUtil {
}
}
- def isHighCardinalityColumn(columnCardinality: Int,
- model: DictionaryLoadModel): Boolean = {
- columnCardinality > model.highCardThreshold
- }
-
/**
* create a instance of DictionaryLoadModel
*
@@ -319,14 +314,6 @@ object GlobalDictionaryUtil {
val lockType = CarbonProperties.getInstance
.getProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS)
val zookeeperUrl = CarbonProperties.getInstance.getProperty(CarbonCommonConstants.ZOOKEEPER_URL)
- // load high cardinality identify configure
- val highCardIdentifyEnable = CarbonProperties.getInstance().getProperty(
- CarbonCommonConstants.HIGH_CARDINALITY_IDENTIFY_ENABLE,
- CarbonCommonConstants.HIGH_CARDINALITY_IDENTIFY_ENABLE_DEFAULT).toBoolean
- val highCardThreshold = CarbonProperties.getInstance().getProperty(
- CarbonCommonConstants.HIGH_CARDINALITY_THRESHOLD,
- CarbonCommonConstants.HIGH_CARDINALITY_THRESHOLD_DEFAULT).toInt
-
val serializationNullFormat =
carbonLoadModel.getSerializationNullFormat.split(CarbonCommonConstants.COMMA, 2)(1)
// get load count
@@ -342,8 +329,6 @@ object GlobalDictionaryUtil {
isComplexes.toArray,
primDimensions,
carbonLoadModel.getDelimiters,
- highCardIdentifyEnable,
- highCardThreshold,
columnIdentifier,
carbonLoadModel.getLoadMetadataDetails.size() == 0,
hdfsTempLocation,
@@ -394,9 +379,8 @@ object GlobalDictionaryUtil {
private def checkStatus(carbonLoadModel: CarbonLoadModel,
sqlContext: SQLContext,
model: DictionaryLoadModel,
- status: Array[(Int, String, Boolean)]) = {
+ status: Array[(Int, String)]) = {
var result = false
- val noDictionaryColumns = new ArrayBuffer[CarbonDimension]
val tableName = model.table.getTableName
status.foreach { x =>
val columnName = model.primDimensions(x._1).getColName
@@ -404,12 +388,6 @@ object GlobalDictionaryUtil {
result = true
LOGGER.error(s"table:$tableName column:$columnName generate global dictionary file failed")
}
- if (x._3) {
- noDictionaryColumns += model.primDimensions(x._1)
- }
- }
- if (noDictionaryColumns.nonEmpty) {
- updateTableMetadataFunc(carbonLoadModel, sqlContext, model, noDictionaryColumns.toArray)
}
if (result) {
LOGGER.error("generate global dictionary files failed")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/191dfb35/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 7bf9765..47d282b 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -572,6 +572,8 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
// All excluded cols should be there in create table cols
if (tableProperties.get(CarbonCommonConstants.DICTIONARY_EXCLUDE).isDefined) {
+ LOGGER.warn("dictionary_exclude option was deprecated, " +
+ "by default string column does not use global dictionary.")
dictExcludeCols =
tableProperties.get(CarbonCommonConstants.DICTIONARY_EXCLUDE).get.split(',').map(_.trim)
dictExcludeCols
@@ -617,19 +619,17 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
}
}
- // by default consider all String cols as dims and if any dictionary exclude is present then
+ // by default consider all String cols as dims and if any dictionary include isn't present then
// add it to noDictionaryDims list. consider all dictionary excludes/include cols as dims
fields.foreach { field =>
- if (dictExcludeCols.toSeq.exists(x => x.equalsIgnoreCase(field.column))) {
- val dataType = DataTypeUtil.getDataType(field.dataType.get.toUpperCase())
- if (dataType != DataType.TIMESTAMP && dataType != DataType.DATE) {
- noDictionaryDims :+= field.column
- }
- dimFields += field
- } else if (dictIncludeCols.exists(x => x.equalsIgnoreCase(field.column))) {
+ if (dictIncludeCols.exists(x => x.equalsIgnoreCase(field.column))) {
dimFields += field
} else if (isDetectAsDimentionDatatype(field.dataType.get)) {
dimFields += field
+ // consider all String cols as noDicitonaryDims by default
+ if (DataType.STRING.getName.equalsIgnoreCase(field.dataType.get)) {
+ noDictionaryDims :+= field.column
+ }
} else if (sortKeyDimsTmp.exists(x => x.equalsIgnoreCase(field.column))) {
noDictionaryDims :+= field.column
dimFields += field
http://git-wip-us.apache.org/repos/asf/carbondata/blob/191dfb35/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
index a2d40b5..32f2ed1 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
@@ -16,8 +16,6 @@
*/
package org.apache.carbondata.spark.util
-import java.io.File
-
import org.apache.spark.sql.common.util.QueryTest
import org.apache.spark.sql.{CarbonEnv, CarbonRelation}
import org.scalatest.BeforeAndAfterAll
@@ -83,7 +81,8 @@ class AllDictionaryTestCase extends QueryTest with BeforeAndAfterAll {
try {
sql(
"CREATE TABLE IF NOT EXISTS sample (id STRING, name STRING, city STRING, " +
- "age INT) STORED BY 'org.apache.carbondata.format'"
+ "age INT) STORED BY 'org.apache.carbondata.format' " +
+ "tblproperties('dictionary_include'='city')"
)
} catch {
case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
@@ -96,7 +95,7 @@ class AllDictionaryTestCase extends QueryTest with BeforeAndAfterAll {
"ActiveProvince: string, Activecity: string, ActiveDistrict: string, ActiveStreet: " +
"string>>, proddate struct<productionDate: string,activeDeactivedate: array<string>>, " +
"gamePointId INT,contractNumber INT) STORED BY 'org.apache.carbondata.format'" +
- "TBLPROPERTIES('DICTIONARY_EXCLUDE'='ROMSize')"
+ "TBLPROPERTIES('DICTIONARY_EXCLUDE'='ROMSize', 'dictionary_include'='channelsId')"
)
} catch {
case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/191dfb35/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
deleted file mode 100644
index 598a90e..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.spark.util
-
-import java.io.{BufferedWriter, File, FileWriter}
-import java.util.Random
-
-import org.apache.spark.sql.common.util.QueryTest
-import org.apache.spark.sql.{CarbonEnv, CarbonRelation}
-import org.scalatest.BeforeAndAfterAll
-
-import org.apache.carbondata.core.metadata.encoder.Encoding
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.util.path.CarbonStorePath
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
-import org.apache.carbondata.processing.model.{CarbonDataLoadSchema, CarbonLoadModel}
-
-/**
- * Test Case for org.apache.carbondata.spark.util.GlobalDictionaryUtil
- */
-class AutoHighCardinalityIdentifyTestCase extends QueryTest with BeforeAndAfterAll {
-
- var filePath: String = _
-
- def buildCarbonLoadModel(relation: CarbonRelation,
- filePath: String,
- header: String): CarbonLoadModel = {
- val carbonLoadModel = new CarbonLoadModel
- carbonLoadModel.setTableName(relation.tableMeta.carbonTableIdentifier.getDatabaseName)
- carbonLoadModel.setDatabaseName(relation.tableMeta.carbonTableIdentifier.getTableName)
- // carbonLoadModel.setSchema(relation.tableMeta.schema)
- val table = relation.tableMeta.carbonTable
- val carbonSchema = new CarbonDataLoadSchema(table)
- carbonLoadModel.setDatabaseName(table.getDatabaseName)
- carbonLoadModel.setTableName(table.getFactTableName)
- carbonLoadModel.setCarbonDataLoadSchema(carbonSchema)
- carbonLoadModel.setFactFilePath(filePath)
- carbonLoadModel.setCsvHeader(header)
- carbonLoadModel.setCsvDelimiter(",")
- carbonLoadModel.setComplexDelimiterLevel1("\\$")
- carbonLoadModel.setComplexDelimiterLevel2("\\:")
- carbonLoadModel.setDefaultTimestampFormat(CarbonProperties.getInstance().getProperty(
- CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT))
- carbonLoadModel.setCsvHeaderColumns(CommonUtil.getCsvHeaderColumns(carbonLoadModel))
- carbonLoadModel
- }
-
- override def beforeAll {
- buildTestData
- buildTable
- }
-
- override def afterAll {
- sql("drop table if exists highcard")
- sql("drop table if exists colgrp_highcard")
- }
-
- def buildTestData() = {
- filePath = s"${integrationPath}/spark/target/highcarddata.csv"
- val file = new File(filePath)
- val writer = new BufferedWriter(new FileWriter(file))
- writer.write("hc1,c2,c3")
- writer.newLine()
- var i = 0
- val random = new Random
- for(i <- 0 until 2000000) {
- writer.write("a" + i + "," +
- "b" + i%1000 + "," +
- i%1000000 + "\n")
- if ( i % 10000 == 0) {
- writer.flush()
- }
- }
- writer.close
- }
-
- def buildTable() = {
- try {
- sql("drop table if exists highcard")
- sql("""create table if not exists highcard
- (hc1 string, c2 string, c3 int)
- STORED BY 'org.apache.carbondata.format'""")
- } catch {
- case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
- }
- }
-
- def relation(tableName: String): CarbonRelation = {
- CarbonEnv.get.carbonMetastore
- .lookupRelation1(Option(CarbonCommonConstants.DATABASE_DEFAULT_NAME),
- tableName)(sqlContext)
- .asInstanceOf[CarbonRelation]
- }
-
- private def checkDictFile(table: CarbonTable) = {
- val tableIdentifier = new CarbonTableIdentifier(table.getDatabaseName,
- table.getFactTableName, "1")
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(storeLocation,
- tableIdentifier)
- val newHc1 = table.getDimensionByName("highcard", "hc1")
- val newC2 = table.getDimensionByName("highcard", "c2")
- val dictFileHc1 = carbonTablePath.getDictionaryFilePath(newHc1.getColumnId)
- val dictFileC2 = carbonTablePath.getDictionaryFilePath(newC2.getColumnId)
- assert(!CarbonUtil.isFileExists(dictFileHc1))
- assert(CarbonUtil.isFileExists(dictFileC2))
- }
-
- private def checkMetaData(oldTable: CarbonTable, newTable: CarbonTable) = {
- val oldHc1 = oldTable.getDimensionByName("highcard", "hc1")
- val oldc2 = oldTable.getDimensionByName("highcard", "c2")
- val newHc1 = newTable.getDimensionByName("highcard", "hc1")
- val newC2 = newTable.getDimensionByName("highcard", "c2")
- assert(oldHc1.hasEncoding(Encoding.DICTIONARY))
- assert(oldc2.hasEncoding(Encoding.DICTIONARY))
- assert(!newHc1.hasEncoding(Encoding.DICTIONARY))
- assert(newC2.hasEncoding(Encoding.DICTIONARY))
- }
-
- test("auto identify high cardinality column in first load #396") {
- val oldTable = relation("highcard").tableMeta.carbonTable
- sql(s"LOAD DATA LOCAL INPATH '$filePath' into table highcard")
- val newTable = relation("highcard").tableMeta.carbonTable
- sql(s"select count(hc1) from highcard")
-
- // check dictionary file
- checkDictFile(newTable)
- // check the meta data
- checkMetaData(oldTable, newTable)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/191dfb35/integration/spark/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
index 4505429..7cfb06d 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
@@ -74,7 +74,7 @@ class ExternalColumnDictionaryTestCase extends QueryTest with BeforeAndAfterAll
proddate struct<productionDate:string,activeDeactivedate:array<string>>,
gamePointId double,contractNumber double)
STORED BY 'org.apache.carbondata.format'
- TBLPROPERTIES('DICTIONARY_INCLUDE' = 'deviceInformationId')
+ TBLPROPERTIES('DICTIONARY_INCLUDE' = 'deviceInformationId, channelsId')
""")
} catch {
case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
@@ -84,7 +84,7 @@ class ExternalColumnDictionaryTestCase extends QueryTest with BeforeAndAfterAll
sql("""CREATE TABLE verticalDelimitedTable (deviceInformationId int,
channelsId string,contractNumber double)
STORED BY 'org.apache.carbondata.format'
- TBLPROPERTIES('DICTIONARY_INCLUDE' = 'deviceInformationId')
+ TBLPROPERTIES('DICTIONARY_INCLUDE' = 'deviceInformationId, channelsId')
""")
} catch {
case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/191dfb35/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
index 9e0f851..ec033d7 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
@@ -76,7 +76,8 @@ class GlobalDictionaryUtilConcurrentTestCase extends QueryTest with BeforeAndAft
def buildTable() = {
try {
sql(
- "CREATE TABLE IF NOT EXISTS employee (empid STRING) STORED BY 'org.apache.carbondata.format'")
+ "CREATE TABLE IF NOT EXISTS employee (empid STRING) STORED BY 'org.apache.carbondata.format' " +
+ "tblproperties('dictionary_include'='empid')")
} catch {
case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/191dfb35/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala b/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
index b8f0a7c..f72bbb3 100644
--- a/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
+++ b/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala
@@ -28,16 +28,6 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
*/
private class TestCarbonSqlParserStub extends CarbonSqlParser {
- //val parser:CarbonSqlDDLParser = new CarbonSqlDDLParser()
-
- def updateColumnGroupsInFieldTest(fields: Seq[Field], tableProperties: Map[String, String]): Seq[String] = {
-
- var (dims, msrs, noDictionaryDims, sortkey) = extractDimAndMsrFields(fields, tableProperties)
-
- updateColumnGroupsInField(tableProperties,
- noDictionaryDims, msrs, dims)
- }
-
def extractDimAndMsrFieldsTest(fields: Seq[Field],
tableProperties: Map[String, String]): (Seq[Field], Seq[Field], Seq[String], Seq[String]) = {
extractDimAndMsrFields(fields, tableProperties)
@@ -78,113 +68,6 @@ class TestCarbonSqlParser extends QueryTest {
fields
}
- // Testing the column group Splitting method.
- test("Test-updateColumnGroupsInField") {
- val colGroupStr = "(col2,col3),(col5,col6),(col7,col8)"
- val tableProperties = Map(CarbonCommonConstants.COLUMN_GROUPS -> colGroupStr)
- val fields: Seq[Field] = loadAllFields
- val stub = new TestCarbonSqlParserStub()
- val colgrps = stub.updateColumnGroupsInFieldTest(fields, tableProperties)
- assert(colgrps.lift(0).get.equalsIgnoreCase("col2,col3"))
- assert(colgrps.lift(1).get.equalsIgnoreCase("col5,col6"))
- assert(colgrps.lift(2).get.equalsIgnoreCase("col7,col8"))
-
- }
- test("Test-updateColumnGroupsInField_disordered") {
- val colGroupStr = "(col5,col6),(col2,col3),(col7,col8)"
- val tableProperties = Map(CarbonCommonConstants.COLUMN_GROUPS -> colGroupStr)
- var fields: Seq[Field] = loadAllFields
- val stub = new TestCarbonSqlParserStub()
- val colgrps = stub.updateColumnGroupsInFieldTest(fields, tableProperties)
- assert(colgrps.lift(0).get.equalsIgnoreCase("col2,col3"))
- assert(colgrps.lift(1).get.equalsIgnoreCase("col5,col6"))
- assert(colgrps.lift(2).get.equalsIgnoreCase("col7,col8"))
-
- }
- test("Test-ColumnGroupsInvalidField_Shouldnotallow") {
- val colGroupStr = "(col1,col2),(col10,col6),(col7,col8)"
- val tableProperties = Map(CarbonCommonConstants.COLUMN_GROUPS -> colGroupStr)
- val fields: Seq[Field] = loadAllFields
- val stub = new TestCarbonSqlParserStub()
- try {
- val colgrps = stub.updateColumnGroupsInFieldTest(fields, tableProperties)
- assert(false)
- } catch {
- case e: Exception => assert(true)
- }
- }
- test("Test-MeasureInColumnGroup_ShouldNotAllow") {
- //col1 is measure
- val colGroupStr = "(col1,col2),(col5,col6),(col7,col8)"
- val tableProperties = Map(CarbonCommonConstants.COLUMN_GROUPS -> colGroupStr)
- val fields: Seq[Field] = loadAllFields
- val stub = new TestCarbonSqlParserStub()
- try {
- val colgrps = stub.updateColumnGroupsInFieldTest(fields, tableProperties)
- assert(false)
- } catch {
- case e: Exception => assert(true)
- }
- }
- test("Test-NoDictionaryInColumnGroup_ShouldNotAllow") {
- //col5 is no dictionary
- val colGroupStr = "(col2,col3),(col5,col6),(col7,col8)"
- val noDictStr = "col5"
- val tableProperties = Map(CarbonCommonConstants.COLUMN_GROUPS -> colGroupStr, CarbonCommonConstants.DICTIONARY_EXCLUDE -> noDictStr)
- val fields: Seq[Field] = loadAllFields
- val stub = new TestCarbonSqlParserStub()
- try {
- val colgrps = stub.updateColumnGroupsInFieldTest(fields, tableProperties)
- assert(false)
- } catch {
- case e: Exception => assert(true)
- }
- }
- test("Test-SameColumnInDifferentGroup_ShouldNotAllow") {
- val colGroupStr = "(col2,col3),(col5,col6),(col6,col7,col8)"
- val tableProperties = Map(CarbonCommonConstants.COLUMN_GROUPS -> colGroupStr)
- val fields: Seq[Field] = loadAllFields
- val stub = new TestCarbonSqlParserStub()
- try {
- val colgrps = stub.updateColumnGroupsInFieldTest(fields, tableProperties)
- assert(false)
- } catch {
- case e: Exception => assert(true)
- }
- }
-
- test("Test-ColumnAreNotTogetherAsInSchema_ShouldNotAllow") {
- val colGroupStr = "(col2,col3),(col5,col8)"
- val tableProperties = Map(CarbonCommonConstants.COLUMN_GROUPS -> colGroupStr)
- val fields: Seq[Field] = loadAllFields
- val stub = new TestCarbonSqlParserStub()
- try {
- val colgrps = stub.updateColumnGroupsInFieldTest(fields, tableProperties)
- assert(false)
- } catch {
- case e: Exception => assert(true)
- }
- }
- test("Test-ColumnInColumnGroupAreShuffledButInSequence") {
- val colGroupStr = "(col2,col3),(col7,col8,col6)"
- val tableProperties = Map(CarbonCommonConstants.COLUMN_GROUPS -> colGroupStr)
- val fields: Seq[Field] = loadAllFields
- val stub = new TestCarbonSqlParserStub()
-
- val colgrps = stub.updateColumnGroupsInFieldTest(fields, tableProperties)
- assert(colgrps.lift(0).get.equalsIgnoreCase("col2,col3"))
- assert(colgrps.lift(1).get.equalsIgnoreCase("col6,col7,col8"))
- }
- // Testing the column group Splitting method with empty table properties so null will be returned.
- test("Test-Empty-updateColumnGroupsInField") {
- val tableProperties = Map("" -> "")
- val fields: Seq[Field] = loadAllFields
- val stub = new TestCarbonSqlParserStub()
- val colgrps = stub.updateColumnGroupsInFieldTest(fields, Map())
- //assert( rtn === 1)
- assert(null == colgrps)
- }
-
// Testing the extracting of Dims and no Dictionary
test("Test-extractDimColsAndNoDictionaryFields") {
val tableProperties = Map(CarbonCommonConstants.DICTIONARY_EXCLUDE -> "col2", CarbonCommonConstants.DICTIONARY_INCLUDE -> "col4")
@@ -203,8 +86,15 @@ class TestCarbonSqlParser extends QueryTest {
assert(dimCols.lift(3).get.column.equalsIgnoreCase("col4"))
//No dictionary column names will be available in noDictionary list
- assert(noDictionary.size == 1)
- assert(noDictionary.lift(0).get.equalsIgnoreCase("col2"))
+ assert(noDictionary.size == 7)
+ assert(noDictionary.lift(0).get.equalsIgnoreCase("col1"))
+ assert(noDictionary.lift(1).get.equalsIgnoreCase("col2"))
+ assert(noDictionary.lift(2).get.equalsIgnoreCase("col3"))
+ assert(noDictionary.lift(3).get.equalsIgnoreCase("col5"))
+ assert(noDictionary.lift(4).get.equalsIgnoreCase("col6"))
+ assert(noDictionary.lift(5).get.equalsIgnoreCase("col7"))
+ assert(noDictionary.lift(6).get.equalsIgnoreCase("col8"))
+
}
test("Test-DimAndMsrColsWithNoDictionaryFields1") {
@@ -220,8 +110,14 @@ class TestCarbonSqlParser extends QueryTest {
assert(dimCols.lift(2).get.column.equalsIgnoreCase("col3"))
//below column names will be available in noDictionary list
- assert(noDictionary.size == 1)
+ assert(noDictionary.size == 7)
assert(noDictionary.lift(0).get.equalsIgnoreCase("col1"))
+ assert(noDictionary.lift(1).get.equalsIgnoreCase("col2"))
+ assert(noDictionary.lift(2).get.equalsIgnoreCase("col3"))
+ assert(noDictionary.lift(3).get.equalsIgnoreCase("col5"))
+ assert(noDictionary.lift(4).get.equalsIgnoreCase("col6"))
+ assert(noDictionary.lift(5).get.equalsIgnoreCase("col7"))
+ assert(noDictionary.lift(6).get.equalsIgnoreCase("col8"))
//check msr
assert(msrCols.size == 1)
@@ -241,7 +137,13 @@ class TestCarbonSqlParser extends QueryTest {
assert(dimCols.lift(2).get.column.equalsIgnoreCase("col3"))
//below column names will be available in noDictionary list
- assert(noDictionary.size == 0)
+ assert(noDictionary.size == 6)
+ assert(noDictionary.lift(0).get.equalsIgnoreCase("col2"))
+ assert(noDictionary.lift(1).get.equalsIgnoreCase("col3"))
+ assert(noDictionary.lift(2).get.equalsIgnoreCase("col5"))
+ assert(noDictionary.lift(3).get.equalsIgnoreCase("col6"))
+ assert(noDictionary.lift(4).get.equalsIgnoreCase("col7"))
+ assert(noDictionary.lift(5).get.equalsIgnoreCase("col8"))
//check msr
assert(msrCols.size == 1)
@@ -263,8 +165,14 @@ class TestCarbonSqlParser extends QueryTest {
assert(dimCols.lift(3).get.column.equalsIgnoreCase("col4"))
//below column names will be available in noDictionary list
- assert(noDictionary.size == 1)
+ assert(noDictionary.size == 7)
assert(noDictionary.lift(0).get.equalsIgnoreCase("col1"))
+ assert(noDictionary.lift(1).get.equalsIgnoreCase("col2"))
+ assert(noDictionary.lift(2).get.equalsIgnoreCase("col3"))
+ assert(noDictionary.lift(3).get.equalsIgnoreCase("col5"))
+ assert(noDictionary.lift(4).get.equalsIgnoreCase("col6"))
+ assert(noDictionary.lift(5).get.equalsIgnoreCase("col7"))
+ assert(noDictionary.lift(6).get.equalsIgnoreCase("col8"))
//check msr
assert(msrCols.size == 0)
@@ -283,8 +191,13 @@ class TestCarbonSqlParser extends QueryTest {
assert(dimCols.lift(2).get.column.equalsIgnoreCase("col3"))
//below column names will be available in noDictionary list
- assert(noDictionary.size == 1)
- assert(noDictionary.lift(0).get.equalsIgnoreCase("col3"))
+ assert(noDictionary.size == 6)
+ assert(noDictionary.lift(0).get.equalsIgnoreCase("col1"))
+ assert(noDictionary.lift(1).get.equalsIgnoreCase("col3"))
+ assert(noDictionary.lift(2).get.equalsIgnoreCase("col5"))
+ assert(noDictionary.lift(3).get.equalsIgnoreCase("col6"))
+ assert(noDictionary.lift(4).get.equalsIgnoreCase("col7"))
+ assert(noDictionary.lift(5).get.equalsIgnoreCase("col8"))
//check msr
assert(msrCols.size == 1)
@@ -304,8 +217,13 @@ class TestCarbonSqlParser extends QueryTest {
assert(dimCols.lift(2).get.column.equalsIgnoreCase("col3"))
//below column names will be available in noDictionary list
- assert(noDictionary.size == 1)
+ assert(noDictionary.size == 6)
assert(noDictionary.lift(0).get.equalsIgnoreCase("col1"))
+ assert(noDictionary.lift(1).get.equalsIgnoreCase("col3"))
+ assert(noDictionary.lift(2).get.equalsIgnoreCase("col5"))
+ assert(noDictionary.lift(3).get.equalsIgnoreCase("col6"))
+ assert(noDictionary.lift(4).get.equalsIgnoreCase("col7"))
+ assert(noDictionary.lift(5).get.equalsIgnoreCase("col8"))
//check msr
assert(msrCols.size == 1)
@@ -325,8 +243,13 @@ class TestCarbonSqlParser extends QueryTest {
assert(dimCols.lift(2).get.column.equalsIgnoreCase("col3"))
//below column names will be available in noDictionary list
- assert(noDictionary.size == 1)
+ assert(noDictionary.size == 6)
assert(noDictionary.lift(0).get.equalsIgnoreCase("col2"))
+ assert(noDictionary.lift(1).get.equalsIgnoreCase("col3"))
+ assert(noDictionary.lift(2).get.equalsIgnoreCase("col5"))
+ assert(noDictionary.lift(3).get.equalsIgnoreCase("col6"))
+ assert(noDictionary.lift(4).get.equalsIgnoreCase("col7"))
+ assert(noDictionary.lift(5).get.equalsIgnoreCase("col8"))
//check msr
assert(msrCols.size == 1)
@@ -349,9 +272,13 @@ class TestCarbonSqlParser extends QueryTest {
assert(dimCols.lift(3).get.column.equalsIgnoreCase("col4"))
//below column names will be available in noDictionary list
- assert(noDictionary.size == 2)
+ assert(noDictionary.size == 6)
assert(noDictionary.lift(0).get.equalsIgnoreCase("col1"))
assert(noDictionary.lift(1).get.equalsIgnoreCase("col2"))
+ assert(noDictionary.lift(2).get.equalsIgnoreCase("col5"))
+ assert(noDictionary.lift(3).get.equalsIgnoreCase("col6"))
+ assert(noDictionary.lift(4).get.equalsIgnoreCase("col7"))
+ assert(noDictionary.lift(5).get.equalsIgnoreCase("col8"))
//check msr
assert(msrCols.size == 0)
@@ -369,8 +296,13 @@ class TestCarbonSqlParser extends QueryTest {
assert(dimCols.lift(1).get.column.equalsIgnoreCase("col2"))
//below column names will be available in noDictionary list
- assert(noDictionary.size == 1)
- assert(noDictionary.lift(0).get.equalsIgnoreCase("col2"))
+ assert(noDictionary.size == 6)
+ assert(noDictionary.lift(0).get.equalsIgnoreCase("col1"))
+ assert(noDictionary.lift(1).get.equalsIgnoreCase("col2"))
+ assert(noDictionary.lift(2).get.equalsIgnoreCase("col5"))
+ assert(noDictionary.lift(3).get.equalsIgnoreCase("col6"))
+ assert(noDictionary.lift(4).get.equalsIgnoreCase("col7"))
+ assert(noDictionary.lift(5).get.equalsIgnoreCase("col8"))
//check msr
assert(msrCols.size == 1)
@@ -390,7 +322,6 @@ class TestCarbonSqlParser extends QueryTest {
}
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/191dfb35/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
index 23800ee..c584fc7 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
@@ -83,7 +83,8 @@ class AllDictionaryTestCase extends QueryTest with BeforeAndAfterAll {
try {
sql(
"CREATE TABLE IF NOT EXISTS sample (id STRING, name STRING, city STRING, " +
- "age INT) STORED BY 'org.apache.carbondata.format'"
+ "age INT) STORED BY 'org.apache.carbondata.format' " +
+ "TBLPROPERTIES('dictionary_include'='city')"
)
} catch {
case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
@@ -96,7 +97,7 @@ class AllDictionaryTestCase extends QueryTest with BeforeAndAfterAll {
"ActiveProvince: string, Activecity: string, ActiveDistrict: string, ActiveStreet: " +
"string>>, proddate struct<productionDate: string,activeDeactivedate: array<string>>, " +
"gamePointId INT,contractNumber INT) STORED BY 'org.apache.carbondata.format'" +
- "TBLPROPERTIES('DICTIONARY_EXCLUDE'='ROMSize')"
+ "TBLPROPERTIES('DICTIONARY_EXCLUDE'='ROMSize', 'dictionary_include'='channelsId')"
)
} catch {
case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/191dfb35/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
index 5f7ff2e..9c576ec 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
@@ -78,7 +78,7 @@ class ExternalColumnDictionaryTestCase extends QueryTest with BeforeAndAfterAll
proddate struct<productionDate:string,activeDeactivedate:array<string>>,
gamePointId double,contractNumber double)
STORED BY 'org.apache.carbondata.format'
- TBLPROPERTIES('DICTIONARY_INCLUDE' = 'deviceInformationId')
+ TBLPROPERTIES('DICTIONARY_INCLUDE' = 'deviceInformationId, channelsId')
""")
} catch {
case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
@@ -89,7 +89,7 @@ class ExternalColumnDictionaryTestCase extends QueryTest with BeforeAndAfterAll
"""CREATE TABLE verticalDelimitedTable (deviceInformationId int,
channelsId string,contractNumber double)
STORED BY 'org.apache.carbondata.format'
- TBLPROPERTIES('DICTIONARY_INCLUDE' = 'deviceInformationId')
+ TBLPROPERTIES('DICTIONARY_INCLUDE' = 'deviceInformationId, channelsId')
""")
} catch {
case ex: Throwable => LOGGER.error(ex.getMessage + "\r\n" + ex.getStackTraceString)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/191dfb35/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
index ce7b321..1bcbaa8 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortTempFileChunkHolder.java
@@ -313,7 +313,7 @@ public class SortTempFileChunkHolder implements Comparable<SortTempFileChunkHold
Object[] holder = new Object[3];
int index = 0;
int nonDicIndex = 0;
- int[] dim = new int[this.dimensionCount];
+ int[] dim = new int[this.dimensionCount - this.noDictionaryCount];
byte[][] nonDicArray = new byte[this.noDictionaryCount + this.complexDimensionCount][];
Object[] measures = new Object[this.measureCount];
try {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/191dfb35/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index 5d9e081..1bbe98c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -603,9 +603,9 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
short numberOfByteToStorLength = 2;
// as length part is removed, so each no dictionary value index
// needs to be reshuffled by 2 bytes
- int NumberOfNoDictSortColumns =
+ int numberOfNoDictSortColumns =
dataWriterVo.getSegmentProperties().getNumberOfNoDictSortColumns();
- for (int i = 0; i < NumberOfNoDictSortColumns; i++) {
+ for (int i = 0; i < numberOfNoDictSortColumns; i++) {
output.putShort((short) (buffer.getShort() - numberOfByteToStorLength));
}
// copy the data part
[14/15] carbondata git commit: [CARBONDATA-1232] Datamap
implementation for Blocklet
Posted by ra...@apache.org.
[CARBONDATA-1232] Datamap implementation for Blocklet
This closes #1099
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2015a3e2
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2015a3e2
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2015a3e2
Branch: refs/heads/datamap
Commit: 2015a3e296afa6f3bbac221f4b590045eb95c0a8
Parents: 8d3c9bf
Author: ravipesala <ra...@gmail.com>
Authored: Sat Jun 17 22:53:57 2017 +0530
Committer: Ravindra Pesala <ra...@gmail.com>
Committed: Sun Jul 16 20:43:41 2017 +0530
----------------------------------------------------------------------
.../carbondata/core/cache/CacheProvider.java | 3 +
.../apache/carbondata/core/cache/CacheType.java | 6 +
.../core/datastore/block/TableBlockInfo.java | 19 +
.../core/datastore/block/TaskBlockInfo.java | 4 +
.../carbondata/core/indexstore/Blocklet.java | 55 +-
.../indexstore/BlockletDataMapIndexStore.java | 180 ++++++
.../core/indexstore/BlockletDetailInfo.java | 117 ++++
.../carbondata/core/indexstore/DataMap.java | 8 +-
.../core/indexstore/DataMapFactory.java | 87 +++
.../core/indexstore/DataMapStoreManager.java | 90 ++-
.../carbondata/core/indexstore/DataMapType.java | 14 +-
.../TableBlockIndexUniqueIdentifier.java | 103 ++++
.../core/indexstore/TableDataMap.java | 97 +++-
.../core/indexstore/UnsafeMemoryDMStore.java | 207 +++++++
.../blockletindex/BlockletDMComparator.java | 134 +++++
.../blockletindex/BlockletDataMap.java | 445 +++++++++++++++
.../blockletindex/BlockletDataMapFactory.java | 115 ++++
.../BlockletDataRefNodeWrapper.java | 137 +++++
.../indexstore/blockletindex/IndexWrapper.java | 49 ++
.../core/indexstore/row/DataMapRow.java | 89 +++
.../core/indexstore/row/DataMapRowImpl.java | 106 ++++
.../core/indexstore/row/UnsafeDataMapRow.java | 133 +++++
.../core/indexstore/schema/DataMapSchema.java | 124 ++++
.../core/indexstore/schema/FilterType.java | 24 +
.../core/metadata/blocklet/BlockletInfo.java | 53 +-
.../core/metadata/index/BlockIndexInfo.java | 27 +
.../executor/impl/AbstractQueryExecutor.java | 52 +-
.../executer/IncludeFilterExecuterImpl.java | 2 +-
.../executer/RangeValueFilterExecuterImpl.java | 2 +-
.../RowLevelRangeGrtThanFiterExecuterImpl.java | 2 +-
...elRangeGrtrThanEquaToFilterExecuterImpl.java | 2 +-
...velRangeLessThanEqualFilterExecuterImpl.java | 2 +-
.../RowLevelRangeLessThanFiterExecuterImpl.java | 2 +-
.../processor/AbstractDataBlockIterator.java | 3 +
.../AbstractDetailQueryResultIterator.java | 34 +-
.../util/AbstractDataFileFooterConverter.java | 53 ++
.../apache/carbondata/core/util/CarbonUtil.java | 40 +-
.../core/util/DataFileFooterConverter.java | 4 +
.../core/util/DataFileFooterConverter2.java | 3 +
.../core/util/DataFileFooterConverterV3.java | 11 +
format/src/main/thrift/carbondata_index.thrift | 1 +
.../carbondata/hadoop/CarbonInputFormat.java | 14 +-
.../carbondata/hadoop/CarbonInputSplit.java | 39 +-
.../hadoop/api/CarbonTableInputFormat.java | 562 ++++++++++++++++---
.../hadoop/util/CarbonInputFormatUtil.java | 7 +-
.../presto/impl/CarbonTableReader.java | 56 +-
.../spark/rdd/CarbonIUDMergerRDD.scala | 5 +-
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 9 +-
.../carbondata/spark/rdd/CarbonScanRDD.scala | 22 +-
.../carbondata/spark/util/QueryPlanUtil.scala | 10 +-
.../sql/CarbonDatasourceHadoopRelation.scala | 14 +-
.../sql/execution/command/IUDCommands.scala | 7 -
.../carbondata/spark/util/QueryPlanUtil.scala | 10 +-
.../processing/merger/CarbonCompactionUtil.java | 32 ++
54 files changed, 3167 insertions(+), 259 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java b/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java
index 25a8976..5c4b265 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java
@@ -31,6 +31,7 @@ import org.apache.carbondata.core.datastore.BlockIndexStore;
import org.apache.carbondata.core.datastore.SegmentTaskIndexStore;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier;
+import org.apache.carbondata.core.indexstore.BlockletDataMapIndexStore;
import org.apache.carbondata.core.util.CarbonProperties;
/**
@@ -126,6 +127,8 @@ public class CacheProvider {
} else if (cacheType.equals(cacheType.DRIVER_BTREE)) {
cacheObject =
new SegmentTaskIndexStore(carbonStorePath, carbonLRUCache);
+ } else if (cacheType.equals(cacheType.DRIVER_BLOCKLET_DATAMAP)) {
+ cacheObject = new BlockletDataMapIndexStore(carbonStorePath, carbonLRUCache);
}
cacheTypeToCacheMap.put(cacheType, cacheObject);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/cache/CacheType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/CacheType.java b/core/src/main/java/org/apache/carbondata/core/cache/CacheType.java
index 2d6570d..ab51ff2 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/CacheType.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/CacheType.java
@@ -56,6 +56,12 @@ public class CacheType<K, V> {
DRIVER_BTREE = new CacheType("driver_btree");
/**
+ * Executor BTree cache which maintains size of BTree metadata
+ */
+ public static final CacheType<TableSegmentUniqueIdentifier, SegmentTaskIndexWrapper>
+ DRIVER_BLOCKLET_DATAMAP = new CacheType("driver_blocklet_datamap");
+
+ /**
* cacheName which is unique name for a cache
*/
private String cacheName;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
index 44347cf..f003882 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
@@ -22,6 +22,7 @@ import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.path.CarbonTablePath;
@@ -77,6 +78,8 @@ public class TableBlockInfo implements Distributable, Serializable {
*/
private String[] deletedDeltaFilePath;
+ private BlockletDetailInfo detailInfo;
+
public TableBlockInfo(String filePath, long blockOffset, String segmentId, String[] locations,
long blockLength, ColumnarFormatVersion version, String[] deletedDeltaFilePath) {
this.filePath = FileFactory.getUpdatedFilePath(filePath);
@@ -88,6 +91,10 @@ public class TableBlockInfo implements Distributable, Serializable {
this.deletedDeltaFilePath = deletedDeltaFilePath;
}
+ public TableBlockInfo() {
+
+ }
+
/**
* constructor to initialize the TbaleBlockInfo with BlockletInfos
*
@@ -319,4 +326,16 @@ public class TableBlockInfo implements Distributable, Serializable {
public String[] getDeletedDeltaFilePath() {
return deletedDeltaFilePath;
}
+
+ public void setFilePath(String filePath) {
+ this.filePath = filePath;
+ }
+
+ public BlockletDetailInfo getDetailInfo() {
+ return detailInfo;
+ }
+
+ public void setDetailInfo(BlockletDetailInfo detailInfo) {
+ this.detailInfo = detailInfo;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/datastore/block/TaskBlockInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TaskBlockInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TaskBlockInfo.java
index eb707c2..4fcec87 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TaskBlockInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TaskBlockInfo.java
@@ -17,6 +17,7 @@
package org.apache.carbondata.core.datastore.block;
+import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -45,6 +46,9 @@ public class TaskBlockInfo {
return taskBlockInfoMapping.keySet();
}
+ public Collection<List<TableBlockInfo>> getAllTableBlockInfoList() {
+ return taskBlockInfoMapping.values();
+ }
/**
* returns TableBlockInfoList of given task
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java b/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
index 597c46c..66da4d0 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
@@ -16,27 +16,76 @@
*/
package org.apache.carbondata.core.indexstore;
+import java.io.IOException;
import java.io.Serializable;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+
/**
* Blocklet
*/
public class Blocklet implements Serializable {
- private String path;
+ private Path path;
+
+ private String segmentId;
private String blockletId;
+ private BlockletDetailInfo detailInfo;
+
+ private long length;
+
+ private String[] location;
+
public Blocklet(String path, String blockletId) {
- this.path = path;
+ this.path = new Path(path);
this.blockletId = blockletId;
}
- public String getPath() {
+ public Path getPath() {
return path;
}
public String getBlockletId() {
return blockletId;
}
+
+ public BlockletDetailInfo getDetailInfo() {
+ return detailInfo;
+ }
+
+ public void setDetailInfo(BlockletDetailInfo detailInfo) {
+ this.detailInfo = detailInfo;
+ }
+
+ public void updateLocations() throws IOException {
+ FileSystem fs = path.getFileSystem(FileFactory.getConfiguration());
+ RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(path);
+ LocatedFileStatus fileStatus = iter.next();
+ location = fileStatus.getBlockLocations()[0].getHosts();
+ length = fileStatus.getLen();
+ }
+
+ public String[] getLocations() throws IOException {
+ return location;
+ }
+
+ public long getLength() throws IOException {
+ return length;
+ }
+
+ public String getSegmentId() {
+ return segmentId;
+ }
+
+ public void setSegmentId(String segmentId) {
+ this.segmentId = segmentId;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
new file mode 100644
index 0000000..fc8c273
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.cache.Cache;
+import org.apache.carbondata.core.cache.CarbonLRUCache;
+import org.apache.carbondata.core.datastore.exception.IndexBuilderException;
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMap;
+
+/**
+ * Class to handle loading, unloading,clearing,storing of the table
+ * blocks
+ */
+public class BlockletDataMapIndexStore
+ implements Cache<TableBlockIndexUniqueIdentifier, BlockletDataMap> {
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(BlockletDataMapIndexStore.class.getName());
+ /**
+ * carbon store path
+ */
+ protected String carbonStorePath;
+ /**
+ * CarbonLRU cache
+ */
+ protected CarbonLRUCache lruCache;
+
+ /**
+ * map of block info to lock object map, while loading the btree this will be filled
+ * and removed after loading the tree for that particular block info, this will be useful
+ * while loading the tree concurrently so only block level lock will be applied another
+ * block can be loaded concurrently
+ */
+ private Map<String, Object> segmentLockMap;
+
+ /**
+ * constructor to initialize the SegmentTaskIndexStore
+ *
+ * @param carbonStorePath
+ * @param lruCache
+ */
+ public BlockletDataMapIndexStore(String carbonStorePath, CarbonLRUCache lruCache) {
+ this.carbonStorePath = carbonStorePath;
+ this.lruCache = lruCache;
+ segmentLockMap = new ConcurrentHashMap<String, Object>();
+ }
+
+ @Override public BlockletDataMap get(TableBlockIndexUniqueIdentifier tableSegmentUniqueIdentifier)
+ throws IOException {
+ String lruCacheKey = tableSegmentUniqueIdentifier.getUniqueTableSegmentIdentifier();
+ BlockletDataMap dataMap = (BlockletDataMap) lruCache.get(lruCacheKey);
+ if (dataMap == null) {
+ try {
+ dataMap = loadAndGetDataMap(tableSegmentUniqueIdentifier);
+ } catch (IndexBuilderException e) {
+ throw new IOException(e.getMessage(), e);
+ } catch (Throwable e) {
+ throw new IOException("Problem in loading segment block.", e);
+ }
+ }
+ return dataMap;
+ }
+
+ @Override public List<BlockletDataMap> getAll(
+ List<TableBlockIndexUniqueIdentifier> tableSegmentUniqueIdentifiers) throws IOException {
+ List<BlockletDataMap> blockletDataMaps = new ArrayList<>(tableSegmentUniqueIdentifiers.size());
+ try {
+ for (TableBlockIndexUniqueIdentifier identifier : tableSegmentUniqueIdentifiers) {
+ blockletDataMaps.add(get(identifier));
+ }
+ } catch (Throwable e) {
+ for (BlockletDataMap dataMap : blockletDataMaps) {
+ dataMap.clear();
+ }
+ throw new IOException("Problem in loading segment blocks.", e);
+ }
+ return blockletDataMaps;
+ }
+
+ /**
+ * returns the SegmentTaskIndexWrapper
+ *
+ * @param tableSegmentUniqueIdentifier
+ * @return
+ */
+ @Override public BlockletDataMap getIfPresent(
+ TableBlockIndexUniqueIdentifier tableSegmentUniqueIdentifier) {
+ BlockletDataMap dataMap = (BlockletDataMap) lruCache
+ .get(tableSegmentUniqueIdentifier.getUniqueTableSegmentIdentifier());
+ return dataMap;
+ }
+
+ /**
+ * method invalidate the segment cache for segment
+ *
+ * @param tableSegmentUniqueIdentifier
+ */
+ @Override public void invalidate(TableBlockIndexUniqueIdentifier tableSegmentUniqueIdentifier) {
+ lruCache.remove(tableSegmentUniqueIdentifier.getUniqueTableSegmentIdentifier());
+ }
+
+ /**
+ * Below method will be used to load the segment of segments
+ * One segment may have multiple task , so table segment will be loaded
+ * based on task id and will return the map of taksId to table segment
+ * map
+ *
+ * @return map of taks id to segment mapping
+ * @throws IOException
+ */
+ private BlockletDataMap loadAndGetDataMap(
+ TableBlockIndexUniqueIdentifier tableSegmentUniqueIdentifier) throws IOException {
+ String uniqueTableSegmentIdentifier =
+ tableSegmentUniqueIdentifier.getUniqueTableSegmentIdentifier();
+ Object lock = segmentLockMap.get(uniqueTableSegmentIdentifier);
+ if (lock == null) {
+ lock = addAndGetSegmentLock(uniqueTableSegmentIdentifier);
+ }
+ BlockletDataMap dataMap = null;
+ synchronized (lock) {
+ dataMap = new BlockletDataMap();
+ dataMap.init(tableSegmentUniqueIdentifier.getFilePath());
+ lruCache.put(tableSegmentUniqueIdentifier.getUniqueTableSegmentIdentifier(), dataMap,
+ dataMap.getMemorySize());
+ }
+ return dataMap;
+ }
+
+ /**
+ * Below method will be used to get the segment level lock object
+ *
+ * @param uniqueIdentifier
+ * @return lock object
+ */
+ private synchronized Object addAndGetSegmentLock(String uniqueIdentifier) {
+ // get the segment lock object if it is present then return
+ // otherwise add the new lock and return
+ Object segmentLoderLockObject = segmentLockMap.get(uniqueIdentifier);
+ if (null == segmentLoderLockObject) {
+ segmentLoderLockObject = new Object();
+ segmentLockMap.put(uniqueIdentifier, segmentLoderLockObject);
+ }
+ return segmentLoderLockObject;
+ }
+
+ /**
+ * The method clears the access count of table segments
+ *
+ * @param tableSegmentUniqueIdentifiers
+ */
+ @Override public void clearAccessCount(
+ List<TableBlockIndexUniqueIdentifier> tableSegmentUniqueIdentifiers) {
+ for (TableBlockIndexUniqueIdentifier segmentUniqueIdentifier : tableSegmentUniqueIdentifiers) {
+ BlockletDataMap cacheable =
+ (BlockletDataMap) lruCache.get(segmentUniqueIdentifier.getUniqueTableSegmentIdentifier());
+ cacheable.clear();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java
new file mode 100644
index 0000000..68dedd8
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDetailInfo.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
+
+import org.apache.hadoop.io.Writable;
+
+/**
+ * Blocklet detail information to be sent to each executor
+ */
+public class BlockletDetailInfo implements Serializable, Writable {
+
+ private int rowCount;
+
+ private short pagesCount;
+
+ private short versionNumber;
+
+ private int[] dimLens;
+
+ private long schemaUpdatedTimeStamp;
+
+ private BlockletInfo blockletInfo;
+
+ public int getRowCount() {
+ return rowCount;
+ }
+
+ public void setRowCount(int rowCount) {
+ this.rowCount = rowCount;
+ }
+
+ public int getPagesCount() {
+ return pagesCount;
+ }
+
+ public void setPagesCount(short pagesCount) {
+ this.pagesCount = pagesCount;
+ }
+
+ public short getVersionNumber() {
+ return versionNumber;
+ }
+
+ public void setVersionNumber(short versionNumber) {
+ this.versionNumber = versionNumber;
+ }
+
+ public BlockletInfo getBlockletInfo() {
+ return blockletInfo;
+ }
+
+ public void setBlockletInfo(BlockletInfo blockletInfo) {
+ this.blockletInfo = blockletInfo;
+ }
+
+ public int[] getDimLens() {
+ return dimLens;
+ }
+
+ public void setDimLens(int[] dimLens) {
+ this.dimLens = dimLens;
+ }
+
+ public long getSchemaUpdatedTimeStamp() {
+ return schemaUpdatedTimeStamp;
+ }
+
+ public void setSchemaUpdatedTimeStamp(long schemaUpdatedTimeStamp) {
+ this.schemaUpdatedTimeStamp = schemaUpdatedTimeStamp;
+ }
+
+ @Override public void write(DataOutput out) throws IOException {
+ out.writeInt(rowCount);
+ out.writeShort(pagesCount);
+ out.writeShort(versionNumber);
+ out.writeShort(dimLens.length);
+ for (int i = 0; i < dimLens.length; i++) {
+ out.writeInt(dimLens[i]);
+ }
+ out.writeLong(schemaUpdatedTimeStamp);
+ blockletInfo.write(out);
+ }
+
+ @Override public void readFields(DataInput in) throws IOException {
+ rowCount = in.readInt();
+ pagesCount = in.readShort();
+ versionNumber = in.readShort();
+ dimLens = new int[in.readShort()];
+ for (int i = 0; i < dimLens.length; i++) {
+ dimLens[i] = in.readInt();
+ }
+ schemaUpdatedTimeStamp = in.readLong();
+ blockletInfo = new BlockletInfo();
+ blockletInfo.readFields(in);
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/indexstore/DataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMap.java
index 2651f15..1276494 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMap.java
@@ -21,7 +21,7 @@ import java.util.List;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
/**
- * Interface for adding and retrieving index data.
+ * Datamap is an entity which can store and retrieve index data.
*/
public interface DataMap {
@@ -47,6 +47,12 @@ public interface DataMap {
List<Blocklet> prune(FilterResolverIntf filterExp);
/**
+ * Convert datamap to distributable object
+ * @return
+ */
+ DataMapDistributable toDistributable();
+
+ /**
* Clear complete index table and release memory.
*/
void clear();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapFactory.java
new file mode 100644
index 0000000..72f714f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapFactory.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+import java.util.List;
+
+import org.apache.carbondata.core.events.ChangeEvent;
+import org.apache.carbondata.core.indexstore.schema.FilterType;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+
+/**
+ * Interface for datamap factory, it is responsible for creating the datamap.
+ */
+public interface DataMapFactory {
+
+ /**
+ * Initialization of Datamap factory
+ * @param identifier
+ * @param dataMapName
+ */
+ void init(AbsoluteTableIdentifier identifier, String dataMapName);
+ /**
+ * Get the datamap writer for each segmentid.
+ *
+ * @param identifier
+ * @param segmentId
+ * @return
+ */
+ DataMapWriter getDataMapWriter(AbsoluteTableIdentifier identifier,
+ String segmentId);
+
+ /**
+ * Get the datamap for segmentid
+ *
+ * @param segmentId
+ * @return
+ */
+ List<DataMap> getDataMaps(String segmentId);
+
+ /**
+ * Get datamap for distributable object.
+ *
+ * @param distributable
+ * @return
+ */
+ DataMap getDataMap(DataMapDistributable distributable);
+
+ /**
+ * This method checks whether the columns and the type of filters supported
+ * for this datamap or not
+ *
+ * @param filterType
+ * @return
+ */
+ boolean isFiltersSupported(FilterType filterType);
+
+ /**
+ *
+ * @param event
+ */
+ void fireEvent(ChangeEvent event);
+
+ /**
+ * Clears datamap of the segment
+ */
+ void clear(String segmentId);
+
+ /**
+ * Clear all datamaps from memory
+ */
+ void clear();
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java
index 06638ad..1a36187 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java
@@ -16,7 +16,9 @@
*/
package org.apache.carbondata.core.indexstore;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.carbondata.common.logging.LogService;
@@ -24,13 +26,16 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
/**
- * It maintains all the index tables in it.
+ * It maintains all the DataMaps in it.
*/
-public class DataMapStoreManager {
+public final class DataMapStoreManager {
private static DataMapStoreManager instance = new DataMapStoreManager();
- private Map<DataMapType, Map<String, TableDataMap>> dataMapMappping = new HashMap<>();
+ /**
+ * Contains the list of datamaps for each table.
+ */
+ private Map<AbsoluteTableIdentifier, List<TableDataMap>> dataMapMappping = new HashMap<>();
private static final LogService LOGGER =
LogServiceFactory.getLogService(DataMapStoreManager.class.getName());
@@ -48,56 +53,85 @@ public class DataMapStoreManager {
*/
public TableDataMap getDataMap(AbsoluteTableIdentifier identifier, String dataMapName,
DataMapType mapType) {
- Map<String, TableDataMap> map = dataMapMappping.get(mapType);
- TableDataMap dataMap = null;
- if (map == null) {
+ List<TableDataMap> tableDataMaps = dataMapMappping.get(identifier);
+ TableDataMap dataMap;
+ if (tableDataMaps == null) {
+ createTableDataMap(identifier, mapType, dataMapName);
+ tableDataMaps = dataMapMappping.get(identifier);
+ }
+ dataMap = getAbstractTableDataMap(dataMapName, tableDataMaps);
+ if (dataMap == null) {
throw new RuntimeException("Datamap does not exist");
- } else {
- dataMap = map.get(dataMapName);
- if (dataMap == null) {
- throw new RuntimeException("Datamap does not exist");
- }
}
- // Initialize datamap
- dataMap.init(identifier, dataMapName);
return dataMap;
}
/**
- * Create new datamap instance using datamap type and path
+ * Create new datamap instance using datamap name, datamap type and table identifier
*
* @param mapType
* @return
*/
- public TableDataMap createTableDataMap(AbsoluteTableIdentifier identifier, DataMapType mapType,
- String dataMapName) {
- Map<String, TableDataMap> map = dataMapMappping.get(mapType);
- if (map == null) {
- map = new HashMap<>();
- dataMapMappping.put(mapType, map);
+ private TableDataMap createTableDataMap(AbsoluteTableIdentifier identifier,
+ DataMapType mapType, String dataMapName) {
+ List<TableDataMap> tableDataMaps = dataMapMappping.get(identifier);
+ if (tableDataMaps == null) {
+ tableDataMaps = new ArrayList<>();
+ dataMapMappping.put(identifier, tableDataMaps);
}
- TableDataMap dataMap = map.get(dataMapName);
+ TableDataMap dataMap = getAbstractTableDataMap(dataMapName, tableDataMaps);
if (dataMap != null) {
throw new RuntimeException("Already datamap exists in that path with type " + mapType);
}
try {
- //TODO create datamap using @mapType.getClassName())
+ DataMapFactory dataMapFactory = mapType.getClassObject().newInstance();
+ dataMapFactory.init(identifier, dataMapName);
+ dataMap = new TableDataMap(identifier, dataMapName, dataMapFactory);
} catch (Exception e) {
LOGGER.error(e);
+ throw new RuntimeException(e);
+ }
+ tableDataMaps.add(dataMap);
+ return dataMap;
+ }
+
+ private TableDataMap getAbstractTableDataMap(String dataMapName,
+ List<TableDataMap> tableDataMaps) {
+ TableDataMap dataMap = null;
+ for (TableDataMap tableDataMap: tableDataMaps) {
+ if (tableDataMap.getDataMapName().equals(dataMapName)) {
+ dataMap = tableDataMap;
+ break;
+ }
}
- dataMap.init(identifier, dataMapName);
- map.put(dataMapName, dataMap);
return dataMap;
}
- public void clearDataMap(String dataMapName, DataMapType mapType) {
- Map<String, TableDataMap> map = dataMapMappping.get(mapType);
- if (map != null && map.get(dataMapName) != null) {
- map.remove(dataMapName).clear();
+ /**
+ * Clear the datamap/datamaps of a mentioned datamap name and table from memory
+ * @param identifier
+ * @param dataMapName
+ */
+ public void clearDataMap(AbsoluteTableIdentifier identifier, String dataMapName) {
+ List<TableDataMap> tableDataMaps = dataMapMappping.get(identifier);
+ if (tableDataMaps != null) {
+ int i = 0;
+ for (TableDataMap tableDataMap: tableDataMaps) {
+ if (tableDataMap != null && dataMapName.equals(tableDataMap.getDataMapName())) {
+ tableDataMap.clear(new ArrayList<String>());
+ tableDataMaps.remove(i);
+ break;
+ }
+ i++;
+ }
}
}
+ /**
+ * Returns the singleton instance
+ * @return
+ */
public static DataMapStoreManager getInstance() {
return instance;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapType.java b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapType.java
index b6a0f5b..0059b29 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapType.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapType.java
@@ -16,19 +16,21 @@
*/
package org.apache.carbondata.core.indexstore;
+import org.apache.carbondata.core.indexstore.blockletindex.BlockletDataMapFactory;
+
/**
* Datamap type
*/
public enum DataMapType {
- BLOCKLET("org.apache.carbondata.datamap.BlockletDataMap");
+ BLOCKLET(BlockletDataMapFactory.class);
- private String className;
+ private Class<? extends DataMapFactory> classObject;
- DataMapType(String className) {
- this.className = className;
+ DataMapType(Class<? extends DataMapFactory> classObject) {
+ this.classObject = classObject;
}
- public String getClassName() {
- return className;
+ public Class<? extends DataMapFactory> getClassObject() {
+ return classObject;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/indexstore/TableBlockIndexUniqueIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/TableBlockIndexUniqueIdentifier.java b/core/src/main/java/org/apache/carbondata/core/indexstore/TableBlockIndexUniqueIdentifier.java
new file mode 100644
index 0000000..7e2bc0e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/TableBlockIndexUniqueIdentifier.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.indexstore;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+
+/**
+ * Class holds the absoluteTableIdentifier and segmentId to uniquely identify a segment
+ */
+public class TableBlockIndexUniqueIdentifier {
+ /**
+ * table fully qualified identifier
+ */
+ private AbsoluteTableIdentifier absoluteTableIdentifier;
+
+ private String segmentId;
+
+ private String carbonIndexFileName;
+
+ /**
+ * Constructor to initialize the class instance
+ *
+ * @param absoluteTableIdentifier
+ * @param segmentId
+ */
+ public TableBlockIndexUniqueIdentifier(AbsoluteTableIdentifier absoluteTableIdentifier,
+ String segmentId, String carbonIndexFileName) {
+ this.absoluteTableIdentifier = absoluteTableIdentifier;
+ this.segmentId = segmentId;
+ this.carbonIndexFileName = carbonIndexFileName;
+ }
+
+ /**
+ * returns AbsoluteTableIdentifier
+ *
+ * @return
+ */
+ public AbsoluteTableIdentifier getAbsoluteTableIdentifier() {
+ return absoluteTableIdentifier;
+ }
+
+ public String getSegmentId() {
+ return segmentId;
+ }
+
+ /**
+ * method returns the id to uniquely identify a key
+ *
+ * @return
+ */
+ public String getUniqueTableSegmentIdentifier() {
+ CarbonTableIdentifier carbonTableIdentifier =
+ absoluteTableIdentifier.getCarbonTableIdentifier();
+ return carbonTableIdentifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR
+ + carbonTableIdentifier.getTableName() + CarbonCommonConstants.UNDERSCORE
+ + carbonTableIdentifier.getTableId() + CarbonCommonConstants.FILE_SEPARATOR + segmentId
+ + CarbonCommonConstants.FILE_SEPARATOR + carbonIndexFileName;
+ }
+
+ public String getFilePath() {
+ return absoluteTableIdentifier.getTablePath() + "/Fact/Part0/Segment_" + segmentId + "/"
+ + carbonIndexFileName;
+ }
+
+ @Override public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ TableBlockIndexUniqueIdentifier that = (TableBlockIndexUniqueIdentifier) o;
+
+ if (!absoluteTableIdentifier.equals(that.absoluteTableIdentifier)) {
+ return false;
+ }
+ if (!segmentId.equals(that.segmentId)) {
+ return false;
+ }
+ return carbonIndexFileName.equals(that.carbonIndexFileName);
+ }
+
+ @Override public int hashCode() {
+ int result = absoluteTableIdentifier.hashCode();
+ result = 31 * result + segmentId.hashCode();
+ result = 31 * result + carbonIndexFileName.hashCode();
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/indexstore/TableDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/TableDataMap.java
index e1532c8..39ca4c5 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/TableDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/TableDataMap.java
@@ -16,38 +16,34 @@
*/
package org.apache.carbondata.core.indexstore;
+import java.util.ArrayList;
import java.util.List;
+import org.apache.carbondata.core.events.ChangeEvent;
import org.apache.carbondata.core.events.EventListener;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
-
/**
* DataMap at the table level, user can add any number of datamaps for one table. Depends
* on the filter condition it can prune the blocklets.
*/
-public interface TableDataMap extends EventListener {
+public final class TableDataMap implements EventListener {
- /**
- * It is called to initialize and load the required table datamap metadata.
- */
- void init(AbsoluteTableIdentifier identifier, String dataMapName);
+ private AbsoluteTableIdentifier identifier;
- /**
- * Gives the writer to write the metadata information of this datamap at table level.
- *
- * @return
- */
- DataMapWriter getWriter();
+ private String dataMapName;
+
+ private DataMapFactory dataMapFactory;
/**
- * Create the datamap using the segmentid and name.
- *
- * @param identifier
- * @param segmentId
- * @return
+ * It is called to initialize and load the required table datamap metadata.
*/
- DataMap createDataMap(AbsoluteTableIdentifier identifier, String segmentId);
+ public TableDataMap(AbsoluteTableIdentifier identifier, String dataMapName,
+ DataMapFactory dataMapFactory) {
+ this.identifier = identifier;
+ this.dataMapName = dataMapName;
+ this.dataMapFactory = dataMapFactory;
+ }
/**
* Pass the valid segments and prune the datamap using filter expression
@@ -56,7 +52,24 @@ public interface TableDataMap extends EventListener {
* @param filterExp
* @return
*/
- List<Blocklet> prune(List<String> segmentIds, FilterResolverIntf filterExp);
+ public List<Blocklet> prune(List<String> segmentIds, FilterResolverIntf filterExp) {
+ List<Blocklet> blocklets = new ArrayList<>();
+ for (String segmentId : segmentIds) {
+ List<DataMap> dataMaps = dataMapFactory.getDataMaps(segmentId);
+ for (DataMap dataMap : dataMaps) {
+ List<Blocklet> pruneBlocklets = dataMap.prune(filterExp);
+ blocklets.addAll(addSegmentId(pruneBlocklets, segmentId));
+ }
+ }
+ return blocklets;
+ }
+
+ private List<Blocklet> addSegmentId(List<Blocklet> pruneBlocklets, String segmentId) {
+ for (Blocklet blocklet : pruneBlocklets) {
+ blocklet.setSegmentId(segmentId);
+ }
+ return pruneBlocklets;
+ }
/**
* This is used for making the datamap distributable.
@@ -65,7 +78,16 @@ public interface TableDataMap extends EventListener {
*
* @return
*/
- List<DataMapDistributable> toDistributable(List<String> segmentIds);
+ public List<DataMapDistributable> toDistributable(List<String> segmentIds) {
+ List<DataMapDistributable> distributables = new ArrayList<>();
+ for (String segmentsId : segmentIds) {
+ List<DataMap> dataMaps = dataMapFactory.getDataMaps(segmentsId);
+ for (DataMap dataMap : dataMaps) {
+ distributables.add(dataMap.toDistributable());
+ }
+ }
+ return distributables;
+ }
/**
* This method is used from any machine after it is distributed. It takes the distributable object
@@ -75,20 +97,37 @@ public interface TableDataMap extends EventListener {
* @param filterExp
* @return
*/
- List<Blocklet> prune(DataMapDistributable distributable, FilterResolverIntf filterExp);
+ public List<Blocklet> prune(DataMapDistributable distributable, FilterResolverIntf filterExp) {
+ return dataMapFactory.getDataMap(distributable).prune(filterExp);
+ }
+
+ @Override public void fireEvent(ChangeEvent event) {
+ dataMapFactory.fireEvent(event);
+ }
/**
- * This method checks whether the columns and the type of filters supported
- * for this datamap or not
- *
- * @param filterExp
- * @return
+ * Clear only the datamaps of the segments
+ * @param segmentIds
*/
- boolean isFiltersSupported(FilterResolverIntf filterExp);
+ public void clear(List<String> segmentIds) {
+ for (String segmentId: segmentIds) {
+ dataMapFactory.clear(segmentId);
+ }
+ }
/**
- * Clears table level datamap
+ * Clears all datamap
+ */
+ public void clear() {
+ dataMapFactory.clear();
+ }
+ /**
+ * Get the unique name of datamap
+ *
+ * @return
*/
- void clear();
+ public String getDataMapName() {
+ return dataMapName;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
new file mode 100644
index 0000000..8246f99
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore;
+
+import org.apache.carbondata.core.indexstore.row.DataMapRow;
+import org.apache.carbondata.core.indexstore.row.UnsafeDataMapRow;
+import org.apache.carbondata.core.indexstore.schema.DataMapSchema;
+import org.apache.carbondata.core.memory.MemoryAllocator;
+import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
+import org.apache.carbondata.core.memory.MemoryBlock;
+
+import static org.apache.carbondata.core.memory.CarbonUnsafe.BYTE_ARRAY_OFFSET;
+import static org.apache.carbondata.core.memory.CarbonUnsafe.unsafe;
+
+/**
+ * Store the data map row @{@link DataMapRow} data to unsafe.
+ */
+public class UnsafeMemoryDMStore {
+
+ private MemoryBlock memoryBlock;
+
+ private static int capacity = 8 * 1024 * 1024;
+
+ private int allocatedSize;
+
+ private int runningLength;
+
+ private MemoryAllocator memoryAllocator;
+
+ private boolean isMemoryFreed;
+
+ private DataMapSchema[] schema;
+
+ private int[] pointers;
+
+ private int rowCount;
+
+ public UnsafeMemoryDMStore(DataMapSchema[] schema) {
+ this.schema = schema;
+ this.memoryAllocator = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator();
+ this.allocatedSize = capacity;
+ this.memoryBlock = memoryAllocator.allocate(allocatedSize);
+ this.pointers = new int[1000];
+ }
+
+ /**
+ * Check memory is sufficient or not, if not sufficient allocate more memory and copy old data to
+ * new one.
+ *
+ * @param rowSize
+ */
+ private void ensureSize(int rowSize) {
+ if (runningLength + rowSize >= allocatedSize) {
+ MemoryBlock allocate =
+ MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().allocate(allocatedSize + capacity);
+ unsafe.copyMemory(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset(),
+ allocate.getBaseObject(), allocate.getBaseOffset(), runningLength);
+ memoryAllocator.free(memoryBlock);
+ allocatedSize = allocatedSize + capacity;
+ memoryBlock = allocate;
+ }
+ if (this.pointers.length <= rowCount + 1) {
+ int[] newPointer = new int[pointers.length + 1000];
+ System.arraycopy(pointers, 0, newPointer, 0, pointers.length);
+ this.pointers = newPointer;
+ }
+ }
+
+ /**
+ * Add the index row to unsafe.
+ *
+ * @param indexRow
+ * @return
+ */
+ public void addIndexRowToUnsafe(DataMapRow indexRow) {
+ // First calculate the required memory to keep the row in unsafe
+ int rowSize = indexRow.getTotalSizeInBytes();
+ // Check whether allocated memory is sufficient or not.
+ ensureSize(rowSize);
+ int pointer = runningLength;
+
+ for (int i = 0; i < schema.length; i++) {
+ addToUnsafe(schema[i], indexRow, i);
+ }
+ pointers[rowCount++] = pointer;
+ }
+
+ private void addToUnsafe(DataMapSchema schema, DataMapRow row, int index) {
+ switch (schema.getSchemaType()) {
+ case FIXED:
+ switch (schema.getDataType()) {
+ case BYTE:
+ unsafe.putByte(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
+ row.getByte(index));
+ runningLength += row.getSizeInBytes(index);
+ break;
+ case SHORT:
+ unsafe
+ .putShort(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
+ row.getShort(index));
+ runningLength += row.getSizeInBytes(index);
+ break;
+ case INT:
+ unsafe.putInt(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
+ row.getInt(index));
+ runningLength += row.getSizeInBytes(index);
+ break;
+ case LONG:
+ unsafe.putLong(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
+ row.getLong(index));
+ runningLength += row.getSizeInBytes(index);
+ break;
+ case FLOAT:
+ unsafe
+ .putFloat(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
+ row.getFloat(index));
+ runningLength += row.getSizeInBytes(index);
+ break;
+ case DOUBLE:
+ unsafe
+ .putDouble(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
+ row.getDouble(index));
+ runningLength += row.getSizeInBytes(index);
+ break;
+ case BYTE_ARRAY:
+ byte[] data = row.getByteArray(index);
+ unsafe.copyMemory(data, BYTE_ARRAY_OFFSET, memoryBlock.getBaseObject(),
+ memoryBlock.getBaseOffset() + runningLength, data.length);
+ runningLength += row.getSizeInBytes(index);
+ break;
+ }
+ break;
+ case VARIABLE:
+ byte[] data = row.getByteArray(index);
+ unsafe.putShort(memoryBlock.getBaseOffset() + runningLength, (short) data.length);
+ runningLength += 2;
+ unsafe.copyMemory(data, BYTE_ARRAY_OFFSET, memoryBlock.getBaseObject(),
+ memoryBlock.getBaseOffset() + runningLength, data.length);
+ runningLength += data.length;
+ break;
+ case STRUCT:
+ DataMapSchema[] childSchemas =
+ ((DataMapSchema.StructDataMapSchema) schema).getChildSchemas();
+ DataMapRow struct = row.getRow(index);
+ for (int i = 0; i < childSchemas.length; i++) {
+ addToUnsafe(childSchemas[i], struct, i);
+ }
+ break;
+ }
+ }
+
+ public DataMapRow getUnsafeRow(int index) {
+ assert (index < rowCount);
+ return new UnsafeDataMapRow(schema, memoryBlock, pointers[index]);
+ }
+
+ public void finishWriting() {
+ if (runningLength < allocatedSize) {
+ MemoryBlock allocate =
+ MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().allocate(runningLength);
+ unsafe.copyMemory(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset(),
+ allocate.getBaseObject(), allocate.getBaseOffset(), runningLength);
+ memoryAllocator.free(memoryBlock);
+ memoryBlock = allocate;
+ }
+ // Compact pointers.
+ if (rowCount < pointers.length) {
+ int[] newPointer = new int[rowCount];
+ System.arraycopy(pointers, 0, newPointer, 0, rowCount);
+ this.pointers = newPointer;
+ }
+ }
+
+ public void freeMemory() {
+ if (!isMemoryFreed) {
+ memoryAllocator.free(memoryBlock);
+ isMemoryFreed = true;
+ }
+ }
+
+ public int getMemoryUsed() {
+ return runningLength;
+ }
+
+ public DataMapSchema[] getSchema() {
+ return schema;
+ }
+
+ public int getRowCount() {
+ return rowCount;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDMComparator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDMComparator.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDMComparator.java
new file mode 100644
index 0000000..9a50600
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDMComparator.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore.blockletindex;
+
+import java.nio.ByteBuffer;
+import java.util.Comparator;
+
+import org.apache.carbondata.core.indexstore.row.DataMapRow;
+import org.apache.carbondata.core.util.ByteUtil;
+
+/**
+ * Data map comparator
+ */
+public class BlockletDMComparator implements Comparator<DataMapRow> {
+
+ /**
+ * no dictionary column value is of variable length so in each column value
+ * it will -1
+ */
+ private static final int NO_DCITIONARY_COLUMN_VALUE = -1;
+
+ /**
+ * sized of the short value in bytes
+ */
+ private static final short SHORT_SIZE_IN_BYTES = 2;
+
+ private int[] eachColumnValueSize;
+
+ /**
+ * the number of no dictionary columns in SORT_COLUMNS
+ */
+ private int numberOfNoDictSortColumns;
+
+ /**
+ * the number of columns in SORT_COLUMNS
+ */
+ private int numberOfSortColumns;
+
+ public BlockletDMComparator(int[] eachColumnValueSize, int numberOfSortColumns,
+ int numberOfNoDictSortColumns) {
+ this.eachColumnValueSize = eachColumnValueSize;
+ this.numberOfNoDictSortColumns = numberOfNoDictSortColumns;
+ this.numberOfSortColumns = numberOfSortColumns;
+ }
+
+ @Override public int compare(DataMapRow first, DataMapRow second) {
+ int dictionaryKeyOffset = 0;
+ int nonDictionaryKeyOffset = 0;
+ int compareResult = 0;
+ int processedNoDictionaryColumn = numberOfNoDictSortColumns;
+ byte[][] firstBytes = splitKey(first.getByteArray(0));
+ byte[][] secondBytes = splitKey(first.getByteArray(0));
+ byte[] firstNoDictionaryKeys = firstBytes[1];
+ ByteBuffer firstNoDictionaryKeyBuffer = ByteBuffer.wrap(firstNoDictionaryKeys);
+ byte[] secondNoDictionaryKeys = secondBytes[1];
+ ByteBuffer secondNoDictionaryKeyBuffer = ByteBuffer.wrap(secondNoDictionaryKeys);
+ int actualOffset = 0;
+ int actualOffset1 = 0;
+ int firstNoDcitionaryLength = 0;
+ int secondNodeDictionaryLength = 0;
+
+ for (int i = 0; i < numberOfSortColumns; i++) {
+
+ if (eachColumnValueSize[i] != NO_DCITIONARY_COLUMN_VALUE) {
+ byte[] firstDictionaryKeys = firstBytes[0];
+ byte[] secondDictionaryKeys = secondBytes[0];
+ compareResult = ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(firstDictionaryKeys, dictionaryKeyOffset, eachColumnValueSize[i],
+ secondDictionaryKeys, dictionaryKeyOffset, eachColumnValueSize[i]);
+ dictionaryKeyOffset += eachColumnValueSize[i];
+ } else {
+ if (processedNoDictionaryColumn > 1) {
+ actualOffset = firstNoDictionaryKeyBuffer.getShort(nonDictionaryKeyOffset);
+ firstNoDcitionaryLength =
+ firstNoDictionaryKeyBuffer.getShort(nonDictionaryKeyOffset + SHORT_SIZE_IN_BYTES)
+ - actualOffset;
+ actualOffset1 = secondNoDictionaryKeyBuffer.getShort(nonDictionaryKeyOffset);
+ secondNodeDictionaryLength =
+ secondNoDictionaryKeyBuffer.getShort(nonDictionaryKeyOffset + SHORT_SIZE_IN_BYTES)
+ - actualOffset1;
+ compareResult = ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(firstNoDictionaryKeys, actualOffset, firstNoDcitionaryLength,
+ secondNoDictionaryKeys, actualOffset1, secondNodeDictionaryLength);
+ nonDictionaryKeyOffset += SHORT_SIZE_IN_BYTES;
+ processedNoDictionaryColumn--;
+ } else {
+ actualOffset = firstNoDictionaryKeyBuffer.getShort(nonDictionaryKeyOffset);
+ actualOffset1 = secondNoDictionaryKeyBuffer.getShort(nonDictionaryKeyOffset);
+ firstNoDcitionaryLength = firstNoDictionaryKeys.length - actualOffset;
+ secondNodeDictionaryLength = secondNoDictionaryKeys.length - actualOffset1;
+ compareResult = ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(firstNoDictionaryKeys, actualOffset, firstNoDcitionaryLength,
+ secondNoDictionaryKeys, actualOffset1, secondNodeDictionaryLength);
+ }
+ }
+ if (compareResult != 0) {
+ return compareResult;
+ }
+ }
+
+ return 0;
+ }
+
+ /**
+ * Split the index key to dictionary and no dictionary.
+ * @param startKey
+ * @return
+ */
+ private byte[][] splitKey(byte[] startKey) {
+ ByteBuffer buffer = ByteBuffer.wrap(startKey);
+ buffer.rewind();
+ int dictonaryKeySize = buffer.getInt();
+ int nonDictonaryKeySize = buffer.getInt();
+ byte[] dictionaryKey = new byte[dictonaryKeySize];
+ buffer.get(dictionaryKey);
+ byte[] nonDictionaryKey = new byte[nonDictonaryKeySize];
+ buffer.get(nonDictionaryKey);
+ return new byte[][] {dictionaryKey, nonDictionaryKey};
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
new file mode 100644
index 0000000..79aa091
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -0,0 +1,445 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore.blockletindex;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.cache.Cacheable;
+import org.apache.carbondata.core.datastore.IndexKey;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.indexstore.Blocklet;
+import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
+import org.apache.carbondata.core.indexstore.DataMap;
+import org.apache.carbondata.core.indexstore.DataMapDistributable;
+import org.apache.carbondata.core.indexstore.DataMapWriter;
+import org.apache.carbondata.core.indexstore.UnsafeMemoryDMStore;
+import org.apache.carbondata.core.indexstore.row.DataMapRow;
+import org.apache.carbondata.core.indexstore.row.DataMapRowImpl;
+import org.apache.carbondata.core.indexstore.schema.DataMapSchema;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
+import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
+import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataFileFooterConverter;
+
+/**
+ * Datamap implementation for blocklet.
+ */
+public class BlockletDataMap implements DataMap, Cacheable {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(BlockletDataMap.class.getName());
+
+ private static int KEY_INDEX = 0;
+
+ private static int MIN_VALUES_INDEX = 1;
+
+ private static int MAX_VALUES_INDEX = 2;
+
+ private static int ROW_COUNT_INDEX = 3;
+
+ private static int FILE_PATH_INDEX = 4;
+
+ private static int PAGE_COUNT_INDEX = 5;
+
+ private static int VERSION_INDEX = 6;
+
+ private static int SCHEMA_UPADATED_TIME_INDEX = 7;
+
+ private static int BLOCK_INFO_INDEX = 8;
+
+ private UnsafeMemoryDMStore unsafeMemoryDMStore;
+
+ private SegmentProperties segmentProperties;
+
+ private int[] columnCardinality;
+
+ @Override public DataMapWriter getWriter() {
+ return null;
+ }
+
+ @Override public void init(String path) {
+ DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
+ try {
+ List<DataFileFooter> indexInfo = fileFooterConverter.getIndexInfo(path);
+ for (DataFileFooter fileFooter : indexInfo) {
+ List<ColumnSchema> columnInTable = fileFooter.getColumnInTable();
+ if (segmentProperties == null) {
+ columnCardinality = fileFooter.getSegmentInfo().getColumnCardinality();
+ segmentProperties = new SegmentProperties(columnInTable, columnCardinality);
+ createSchema(segmentProperties);
+ }
+ TableBlockInfo blockInfo = fileFooter.getBlockInfo().getTableBlockInfo();
+ fileFooter = CarbonUtil.readMetadatFile(blockInfo);
+
+ loadToUnsafe(fileFooter, segmentProperties, blockInfo.getFilePath());
+ }
+ if (unsafeMemoryDMStore != null) {
+ unsafeMemoryDMStore.finishWriting();
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void loadToUnsafe(DataFileFooter fileFooter, SegmentProperties segmentProperties,
+ String filePath) {
+ int[] minMaxLen = segmentProperties.getEachDimColumnValueSize();
+ List<BlockletInfo> blockletList = fileFooter.getBlockletList();
+ DataMapSchema[] schema = unsafeMemoryDMStore.getSchema();
+ for (int index = 0; index < blockletList.size(); index++) {
+ DataMapRow row = new DataMapRowImpl(schema);
+ int ordinal = 0;
+ BlockletInfo blockletInfo = blockletList.get(index);
+
+ // add start key as index key
+ row.setByteArray(blockletInfo.getBlockletIndex().getBtreeIndex().getStartKey(), ordinal++);
+
+ BlockletMinMaxIndex minMaxIndex = blockletInfo.getBlockletIndex().getMinMaxIndex();
+ row.setRow(addMinMax(minMaxLen, schema[ordinal], minMaxIndex.getMinValues()), ordinal);
+ ordinal++;
+ row.setRow(addMinMax(minMaxLen, schema[ordinal], minMaxIndex.getMaxValues()), ordinal);
+ ordinal++;
+
+ row.setInt(blockletInfo.getNumberOfRows(), ordinal++);
+
+ // add file path
+ byte[] filePathBytes = filePath.getBytes();
+ row.setByteArray(filePathBytes, ordinal++);
+
+ // add pages
+ row.setShort((short) blockletInfo.getNumberOfPages(), ordinal++);
+
+ // add version number
+ row.setShort(fileFooter.getVersionId().number(), ordinal++);
+
+ // add schema updated time
+ row.setLong(fileFooter.getSchemaUpdatedTimeStamp(), ordinal++);
+
+ // add blocklet info
+ byte[] serializedData;
+ try {
+ ByteArrayOutputStream stream = new ByteArrayOutputStream();
+ DataOutput dataOutput = new DataOutputStream(stream);
+ blockletInfo.write(dataOutput);
+ serializedData = stream.toByteArray();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ row.setByteArray(serializedData, ordinal);
+ unsafeMemoryDMStore.addIndexRowToUnsafe(row);
+ }
+ }
+
+ private DataMapRow addMinMax(int[] minMaxLen, DataMapSchema dataMapSchema, byte[][] minValues) {
+ DataMapSchema[] minSchemas =
+ ((DataMapSchema.StructDataMapSchema) dataMapSchema).getChildSchemas();
+ DataMapRow minRow = new DataMapRowImpl(minSchemas);
+ int minOrdinal = 0;
+ // min value adding
+ for (int i = 0; i < minMaxLen.length; i++) {
+ minRow.setByteArray(minValues[i], minOrdinal++);
+ }
+ return minRow;
+ }
+
+ private void createSchema(SegmentProperties segmentProperties) {
+ List<DataMapSchema> indexSchemas = new ArrayList<>();
+
+ // Index key
+ indexSchemas.add(new DataMapSchema.VariableDataMapSchema(DataType.BYTE_ARRAY));
+ int[] minMaxLen = segmentProperties.getEachDimColumnValueSize();
+ // do it 2 times, one for min and one for max.
+ for (int k = 0; k < 2; k++) {
+ DataMapSchema[] mapSchemas = new DataMapSchema[minMaxLen.length];
+ for (int i = 0; i < minMaxLen.length; i++) {
+ if (minMaxLen[i] <= 0) {
+ mapSchemas[i] = new DataMapSchema.VariableDataMapSchema(DataType.BYTE_ARRAY);
+ } else {
+ mapSchemas[i] = new DataMapSchema.FixedDataMapSchema(DataType.BYTE_ARRAY, minMaxLen[i]);
+ }
+ }
+ DataMapSchema mapSchema = new DataMapSchema.StructDataMapSchema(DataType.STRUCT, mapSchemas);
+ indexSchemas.add(mapSchema);
+ }
+
+ // for number of rows.
+ indexSchemas.add(new DataMapSchema.FixedDataMapSchema(DataType.INT));
+
+ // for table block path
+ indexSchemas.add(new DataMapSchema.VariableDataMapSchema(DataType.BYTE_ARRAY));
+
+ // for number of pages.
+ indexSchemas.add(new DataMapSchema.FixedDataMapSchema(DataType.SHORT));
+
+ // for version number.
+ indexSchemas.add(new DataMapSchema.FixedDataMapSchema(DataType.SHORT));
+
+ // for schema updated time.
+ indexSchemas.add(new DataMapSchema.FixedDataMapSchema(DataType.LONG));
+
+ //for blocklet info
+ indexSchemas.add(new DataMapSchema.VariableDataMapSchema(DataType.BYTE_ARRAY));
+
+ unsafeMemoryDMStore =
+ new UnsafeMemoryDMStore(indexSchemas.toArray(new DataMapSchema[indexSchemas.size()]));
+ }
+
+ @Override public List<Blocklet> prune(FilterResolverIntf filterExp) {
+
+ // getting the start and end index key based on filter for hitting the
+ // selected block reference nodes based on filter resolver tree.
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("preparing the start and end key for finding"
+ + "start and end block as per filter resolver");
+ }
+ List<Blocklet> blocklets = new ArrayList<>();
+ Comparator<DataMapRow> comparator =
+ new BlockletDMComparator(segmentProperties.getEachDimColumnValueSize(),
+ segmentProperties.getNumberOfSortColumns(),
+ segmentProperties.getNumberOfNoDictSortColumns());
+ List<IndexKey> listOfStartEndKeys = new ArrayList<IndexKey>(2);
+ FilterUtil
+ .traverseResolverTreeAndGetStartAndEndKey(segmentProperties, filterExp, listOfStartEndKeys);
+ // reading the first value from list which has start key
+ IndexKey searchStartKey = listOfStartEndKeys.get(0);
+ // reading the last value from list which has end key
+ IndexKey searchEndKey = listOfStartEndKeys.get(1);
+ if (null == searchStartKey && null == searchEndKey) {
+ try {
+ // TODO need to handle for no dictionary dimensions
+ searchStartKey = FilterUtil.prepareDefaultStartIndexKey(segmentProperties);
+ // TODO need to handle for no dictionary dimensions
+ searchEndKey = FilterUtil.prepareDefaultEndIndexKey(segmentProperties);
+ } catch (KeyGenException e) {
+ return null;
+ }
+ }
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug(
+ "Successfully retrieved the start and end key" + "Dictionary Start Key: " + searchStartKey
+ .getDictionaryKeys() + "No Dictionary Start Key " + searchStartKey
+ .getNoDictionaryKeys() + "Dictionary End Key: " + searchEndKey.getDictionaryKeys()
+ + "No Dictionary End Key " + searchEndKey.getNoDictionaryKeys());
+ }
+ if (filterExp == null) {
+ int rowCount = unsafeMemoryDMStore.getRowCount();
+ for (int i = 0; i < rowCount; i++) {
+ DataMapRow unsafeRow = unsafeMemoryDMStore.getUnsafeRow(i);
+ blocklets.add(createBlocklet(unsafeRow, i));
+ }
+ } else {
+ int startIndex = findStartIndex(convertToRow(searchStartKey), comparator);
+ int endIndex = findEndIndex(convertToRow(searchEndKey), comparator);
+ FilterExecuter filterExecuter =
+ FilterUtil.getFilterExecuterTree(filterExp, segmentProperties, null);
+ while (startIndex <= endIndex) {
+ DataMapRow unsafeRow = unsafeMemoryDMStore.getUnsafeRow(startIndex);
+ BitSet bitSet = filterExecuter.isScanRequired(getMinMaxValue(unsafeRow, MAX_VALUES_INDEX),
+ getMinMaxValue(unsafeRow, MIN_VALUES_INDEX));
+ if (!bitSet.isEmpty()) {
+ blocklets.add(createBlocklet(unsafeRow, startIndex));
+ }
+ startIndex++;
+ }
+ }
+
+ return blocklets;
+ }
+
+ private byte[][] getMinMaxValue(DataMapRow row, int index) {
+ DataMapRow minMaxRow = row.getRow(index);
+ byte[][] minMax = new byte[minMaxRow.getColumnCount()][];
+ for (int i = 0; i < minMax.length; i++) {
+ minMax[i] = minMaxRow.getByteArray(i);
+ }
+ return minMax;
+ }
+
+ private Blocklet createBlocklet(DataMapRow row, int blockletId) {
+ Blocklet blocklet =
+ new Blocklet(new String(row.getByteArray(FILE_PATH_INDEX)), blockletId + "");
+ BlockletDetailInfo detailInfo = new BlockletDetailInfo();
+ detailInfo.setRowCount(row.getInt(ROW_COUNT_INDEX));
+ detailInfo.setPagesCount(row.getShort(PAGE_COUNT_INDEX));
+ detailInfo.setVersionNumber(row.getShort(VERSION_INDEX));
+ detailInfo.setDimLens(columnCardinality);
+ detailInfo.setSchemaUpdatedTimeStamp(row.getLong(SCHEMA_UPADATED_TIME_INDEX));
+ BlockletInfo blockletInfo = new BlockletInfo();
+ try {
+ byte[] byteArray = row.getByteArray(BLOCK_INFO_INDEX);
+ ByteArrayInputStream stream = new ByteArrayInputStream(byteArray);
+ DataInputStream inputStream = new DataInputStream(stream);
+ blockletInfo.readFields(inputStream);
+ inputStream.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ detailInfo.setBlockletInfo(blockletInfo);
+ blocklet.setDetailInfo(detailInfo);
+ return blocklet;
+ }
+
+ /**
+ * Binary search used to get the first tentative index row based on
+ * search key
+ *
+ * @param key search key
+ * @return first tentative block
+ */
+ private int findStartIndex(DataMapRow key, Comparator<DataMapRow> comparator) {
+ int childNodeIndex;
+ int low = 0;
+ int high = unsafeMemoryDMStore.getRowCount() - 1;
+ int mid = 0;
+ int compareRes = -1;
+ //
+ while (low <= high) {
+ mid = (low + high) >>> 1;
+ // compare the entries
+ compareRes = comparator.compare(key, unsafeMemoryDMStore.getUnsafeRow(mid));
+ if (compareRes < 0) {
+ high = mid - 1;
+ } else if (compareRes > 0) {
+ low = mid + 1;
+ } else {
+ // if key is matched then get the first entry
+ int currentPos = mid;
+ while (currentPos - 1 >= 0
+ && comparator.compare(key, unsafeMemoryDMStore.getUnsafeRow(currentPos - 1)) == 0) {
+ currentPos--;
+ }
+ mid = currentPos;
+ break;
+ }
+ }
+ // if compare result is less than zero then we
+ // and mid is more than 0 then we need to previous block as duplicates
+ // record can be present
+ if (compareRes < 0) {
+ if (mid > 0) {
+ mid--;
+ }
+ childNodeIndex = mid;
+ } else {
+ childNodeIndex = mid;
+ }
+ // get the leaf child
+ return childNodeIndex;
+ }
+
+ /**
+ * Binary search used to get the last tentative block based on
+ * search key
+ *
+ * @param key search key
+ * @return first tentative block
+ */
+ private int findEndIndex(DataMapRow key, Comparator<DataMapRow> comparator) {
+ int childNodeIndex;
+ int low = 0;
+ int high = unsafeMemoryDMStore.getRowCount() - 1;
+ int mid = 0;
+ int compareRes = -1;
+ //
+ while (low <= high) {
+ mid = (low + high) >>> 1;
+ // compare the entries
+ compareRes = comparator.compare(key, unsafeMemoryDMStore.getUnsafeRow(mid));
+ if (compareRes < 0) {
+ high = mid - 1;
+ } else if (compareRes > 0) {
+ low = mid + 1;
+ } else {
+ int currentPos = mid;
+ // if key is matched then get the first entry
+ while (currentPos + 1 < unsafeMemoryDMStore.getRowCount()
+ && comparator.compare(key, unsafeMemoryDMStore.getUnsafeRow(currentPos + 1)) == 0) {
+ currentPos++;
+ }
+ mid = currentPos;
+ break;
+ }
+ }
+ // if compare result is less than zero then we
+ // and mid is more than 0 then we need to previous block as duplicates
+ // record can be present
+ if (compareRes < 0) {
+ if (mid > 0) {
+ mid--;
+ }
+ childNodeIndex = mid;
+ } else {
+ childNodeIndex = mid;
+ }
+ return childNodeIndex;
+ }
+
+ private DataMapRow convertToRow(IndexKey key) {
+ ByteBuffer buffer =
+ ByteBuffer.allocate(key.getDictionaryKeys().length + key.getNoDictionaryKeys().length + 8);
+ buffer.putInt(key.getDictionaryKeys().length);
+ buffer.putInt(key.getNoDictionaryKeys().length);
+ buffer.put(key.getDictionaryKeys());
+ buffer.put(key.getNoDictionaryKeys());
+ DataMapRowImpl dataMapRow = new DataMapRowImpl(unsafeMemoryDMStore.getSchema());
+ dataMapRow.setByteArray(buffer.array(), 0);
+ return dataMapRow;
+ }
+
+ @Override public void clear() {
+ unsafeMemoryDMStore.freeMemory();
+ unsafeMemoryDMStore = null;
+ segmentProperties = null;
+ }
+
+ @Override public long getFileTimeStamp() {
+ return 0;
+ }
+
+ @Override public int getAccessCount() {
+ return 0;
+ }
+
+ @Override public long getMemorySize() {
+ return unsafeMemoryDMStore.getMemoryUsed();
+ }
+
+ @Override public DataMapDistributable toDistributable() {
+ // TODO
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
new file mode 100644
index 0000000..2fe6643
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore.blockletindex;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.core.cache.Cache;
+import org.apache.carbondata.core.cache.CacheProvider;
+import org.apache.carbondata.core.cache.CacheType;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.events.ChangeEvent;
+import org.apache.carbondata.core.indexstore.DataMap;
+import org.apache.carbondata.core.indexstore.DataMapDistributable;
+import org.apache.carbondata.core.indexstore.DataMapFactory;
+import org.apache.carbondata.core.indexstore.DataMapWriter;
+import org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifier;
+import org.apache.carbondata.core.indexstore.schema.FilterType;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+
+/**
+ * Table map for blocklet
+ */
+public class BlockletDataMapFactory implements DataMapFactory {
+
+ private AbsoluteTableIdentifier identifier;
+
+ private Map<String, List<TableBlockIndexUniqueIdentifier>> segmentMap = new HashMap<>();
+
+ private Cache<TableBlockIndexUniqueIdentifier, DataMap> cache;
+
+ public void init(AbsoluteTableIdentifier identifier, String dataMapName) {
+ this.identifier = identifier;
+ cache = CacheProvider.getInstance()
+ .createCache(CacheType.DRIVER_BLOCKLET_DATAMAP, identifier.getStorePath());
+ }
+
+ public DataMapWriter getDataMapWriter(AbsoluteTableIdentifier identifier, String segmentId) {
+ return null;
+ }
+
+ public List<DataMap> getDataMaps(String segmentId) {
+ List<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers =
+ segmentMap.get(segmentId);
+ if (tableBlockIndexUniqueIdentifiers == null) {
+ tableBlockIndexUniqueIdentifiers = new ArrayList<>();
+ String path = identifier.getTablePath() + "/Fact/Part0/Segment_" + segmentId;
+ FileFactory.FileType fileType = FileFactory.getFileType(path);
+ CarbonFile carbonFile = FileFactory.getCarbonFile(path, fileType);
+ CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {
+ @Override public boolean accept(CarbonFile file) {
+ return file.getName().endsWith(".carbonindex");
+ }
+ });
+ for (int i = 0; i < listFiles.length; i++) {
+ tableBlockIndexUniqueIdentifiers.add(
+ new TableBlockIndexUniqueIdentifier(identifier, segmentId, listFiles[i].getName()));
+ }
+ }
+
+ try {
+ return cache.getAll(tableBlockIndexUniqueIdentifiers);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override public boolean isFiltersSupported(FilterType filterType) {
+ return true;
+ }
+
+ public void clear(String segmentId) {
+ List<TableBlockIndexUniqueIdentifier> blockIndexes = segmentMap.remove(segmentId);
+ if (blockIndexes != null) {
+ for (TableBlockIndexUniqueIdentifier blockIndex : blockIndexes) {
+ DataMap dataMap = cache.getIfPresent(blockIndex);
+ dataMap.clear();
+ cache.invalidate(blockIndex);
+ }
+ }
+ }
+
+ @Override public void clear() {
+ for (String segmentId: segmentMap.keySet()) {
+ clear(segmentId);
+ }
+ }
+
+ @Override public DataMap getDataMap(DataMapDistributable distributable) {
+ return null;
+ }
+
+ @Override public void fireEvent(ChangeEvent event) {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
new file mode 100644
index 0000000..5509c75
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore.blockletindex;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.core.cache.update.BlockletLevelDeleteDeltaDataCache;
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.datastore.chunk.reader.CarbonDataReaderFactory;
+import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
+import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReader;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
+
+/**
+ * wrapper for blocklet data map data
+ */
+public class BlockletDataRefNodeWrapper implements DataRefNode {
+
+ private List<TableBlockInfo> blockInfos;
+
+ private int index;
+
+ private int[] dimensionLens;
+
+ private BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache;
+
+ public BlockletDataRefNodeWrapper(List<TableBlockInfo> blockInfos, int index,
+ int[] dimensionLens) {
+ this.blockInfos = blockInfos;
+ this.index = index;
+ this.dimensionLens = dimensionLens;
+ }
+
+ @Override public DataRefNode getNextDataRefNode() {
+ if (index + 1 < blockInfos.size()) {
+ new BlockletDataRefNodeWrapper(blockInfos, index + 1, dimensionLens);
+ }
+ return null;
+ }
+
+ @Override public int nodeSize() {
+ return blockInfos.get(index).getDetailInfo().getRowCount();
+ }
+
+ @Override public long nodeNumber() {
+ return index;
+ }
+
+ @Override public byte[][] getColumnsMaxValue() {
+ return null;
+ }
+
+ @Override public byte[][] getColumnsMinValue() {
+ return null;
+ }
+
+ @Override
+ public DimensionRawColumnChunk[] getDimensionChunks(FileHolder fileReader, int[][] blockIndexes)
+ throws IOException {
+ DimensionColumnChunkReader dimensionChunksReader = getDimensionColumnChunkReader();
+ return dimensionChunksReader.readRawDimensionChunks(fileReader, blockIndexes);
+ }
+
+ @Override
+ public DimensionRawColumnChunk getDimensionChunk(FileHolder fileReader, int blockIndexes)
+ throws IOException {
+ DimensionColumnChunkReader dimensionChunksReader = getDimensionColumnChunkReader();
+ return dimensionChunksReader.readRawDimensionChunk(fileReader, blockIndexes);
+ }
+
+ @Override
+ public MeasureRawColumnChunk[] getMeasureChunks(FileHolder fileReader, int[][] blockIndexes)
+ throws IOException {
+ MeasureColumnChunkReader measureColumnChunkReader = getMeasureColumnChunkReader();
+ return measureColumnChunkReader.readRawMeasureChunks(fileReader, blockIndexes);
+ }
+
+ @Override public MeasureRawColumnChunk getMeasureChunk(FileHolder fileReader, int blockIndex)
+ throws IOException {
+ MeasureColumnChunkReader measureColumnChunkReader = getMeasureColumnChunkReader();
+ return measureColumnChunkReader.readRawMeasureChunk(fileReader, blockIndex);
+ }
+
+ private DimensionColumnChunkReader getDimensionColumnChunkReader() throws IOException {
+ ColumnarFormatVersion version =
+ ColumnarFormatVersion.valueOf(blockInfos.get(index).getDetailInfo().getVersionNumber());
+ DimensionColumnChunkReader dimensionColumnChunkReader = CarbonDataReaderFactory.getInstance()
+ .getDimensionColumnChunkReader(version,
+ blockInfos.get(index).getDetailInfo().getBlockletInfo(), dimensionLens,
+ blockInfos.get(index).getFilePath());
+ return dimensionColumnChunkReader;
+ }
+
+ private MeasureColumnChunkReader getMeasureColumnChunkReader() throws IOException {
+ ColumnarFormatVersion version =
+ ColumnarFormatVersion.valueOf(blockInfos.get(index).getDetailInfo().getVersionNumber());
+ return CarbonDataReaderFactory.getInstance().getMeasureColumnChunkReader(version,
+ blockInfos.get(index).getDetailInfo().getBlockletInfo(),
+ blockInfos.get(index).getFilePath());
+ }
+
+ @Override
+ public void setDeleteDeltaDataCache(BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache) {
+ this.deleteDeltaDataCache = deleteDeltaDataCache;
+ }
+
+ @Override public BlockletLevelDeleteDeltaDataCache getDeleteDeltaDataCache() {
+ return deleteDeltaDataCache;
+ }
+
+ @Override public int numberOfPages() {
+ return blockInfos.get(index).getDetailInfo().getPagesCount();
+ }
+
+ public int numberOfNodes() {
+ return blockInfos.size();
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2015a3e2/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
new file mode 100644
index 0000000..b8cffc6
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/IndexWrapper.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.indexstore.blockletindex;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.core.datastore.block.AbstractIndex;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+/**
+ * Wrapper of abstract index
+ * TODO it could be removed after refactor
+ */
+public class IndexWrapper extends AbstractIndex {
+
+ public IndexWrapper(List<TableBlockInfo> blockInfos) {
+ DataFileFooter fileFooter = null;
+ try {
+ fileFooter = CarbonUtil.readMetadatFile(blockInfos.get(0));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ segmentProperties = new SegmentProperties(fileFooter.getColumnInTable(),
+ fileFooter.getSegmentInfo().getColumnCardinality());
+ dataRefNode = new BlockletDataRefNodeWrapper(blockInfos, 0,
+ segmentProperties.getDimensionColumnsValueSize());
+ }
+
+ @Override public void buildIndex(List<DataFileFooter> footerList) {
+ }
+}
[06/15] carbondata git commit: [CARBONDATA-1284]Implement hive based
schema storage in carbon
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
new file mode 100644
index 0000000..f245df6
--- /dev/null
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonRelation.scala
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.hive
+
+import java.util.LinkedHashSet
+
+import scala.Array.canBuildFrom
+import scala.collection.JavaConverters._
+import scala.util.parsing.combinator.RegexParsers
+
+import org.apache.spark.sql.catalyst.analysis.MultiInstanceRelation
+import org.apache.spark.sql.catalyst.expressions.AttributeReference
+import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Statistics}
+import org.apache.spark.sql.types._
+
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.metadata.datatype.DataType.DECIMAL
+import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension}
+import org.apache.carbondata.core.statusmanager.SegmentStatusManager
+import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.processing.merger.TableMeta
+
+/**
+ * Represents logical plan for one carbon table
+ */
+case class CarbonRelation(
+ databaseName: String,
+ tableName: String,
+ var metaData: CarbonMetaData,
+ tableMeta: TableMeta)
+ extends LeafNode with MultiInstanceRelation {
+
+ def recursiveMethod(dimName: String, childDim: CarbonDimension): String = {
+ childDim.getDataType.toString.toLowerCase match {
+ case "array" => s"${
+ childDim.getColName.substring(dimName.length + 1)
+ }:array<${ getArrayChildren(childDim.getColName) }>"
+ case "struct" => s"${
+ childDim.getColName.substring(dimName.length + 1)
+ }:struct<${ getStructChildren(childDim.getColName) }>"
+ case dType => s"${ childDim.getColName.substring(dimName.length + 1) }:${ dType }"
+ }
+ }
+
+ def getArrayChildren(dimName: String): String = {
+ metaData.carbonTable.getChildren(dimName).asScala.map(childDim => {
+ childDim.getDataType.toString.toLowerCase match {
+ case "array" => s"array<${ getArrayChildren(childDim.getColName) }>"
+ case "struct" => s"struct<${ getStructChildren(childDim.getColName) }>"
+ case dType => addDecimalScaleAndPrecision(childDim, dType)
+ }
+ }).mkString(",")
+ }
+
+ def getStructChildren(dimName: String): String = {
+ metaData.carbonTable.getChildren(dimName).asScala.map(childDim => {
+ childDim.getDataType.toString.toLowerCase match {
+ case "array" => s"${
+ childDim.getColName.substring(dimName.length + 1)
+ }:array<${ getArrayChildren(childDim.getColName) }>"
+ case "struct" => s"${
+ childDim.getColName.substring(dimName.length + 1)
+ }:struct<${ metaData.carbonTable.getChildren(childDim.getColName)
+ .asScala.map(f => s"${ recursiveMethod(childDim.getColName, f) }").mkString(",")
+ }>"
+ case dType => s"${ childDim.getColName
+ .substring(dimName.length() + 1) }:${ addDecimalScaleAndPrecision(childDim, dType) }"
+ }
+ }).mkString(",")
+ }
+
+ override def newInstance(): LogicalPlan = {
+ CarbonRelation(databaseName, tableName, metaData, tableMeta)
+ .asInstanceOf[this.type]
+ }
+
+ val dimensionsAttr = {
+ val sett = new LinkedHashSet(
+ tableMeta.carbonTable.getDimensionByTableName(tableMeta.carbonTableIdentifier.getTableName)
+ .asScala.asJava)
+ sett.asScala.toSeq.map(dim => {
+ val dimval = metaData.carbonTable
+ .getDimensionByName(metaData.carbonTable.getFactTableName, dim.getColName)
+ val output: DataType = dimval.getDataType
+ .toString.toLowerCase match {
+ case "array" =>
+ CarbonMetastoreTypes.toDataType(s"array<${ getArrayChildren(dim.getColName) }>")
+ case "struct" =>
+ CarbonMetastoreTypes.toDataType(s"struct<${ getStructChildren(dim.getColName) }>")
+ case dType =>
+ val dataType = addDecimalScaleAndPrecision(dimval, dType)
+ CarbonMetastoreTypes.toDataType(dataType)
+ }
+
+ AttributeReference(
+ dim.getColName,
+ output,
+ nullable = true)()
+ })
+ }
+
+ val measureAttr = {
+ val factTable = tableMeta.carbonTable.getFactTableName
+ new LinkedHashSet(
+ tableMeta.carbonTable.
+ getMeasureByTableName(tableMeta.carbonTable.getFactTableName).
+ asScala.asJava).asScala.toSeq
+ .map(x => AttributeReference(x.getColName, CarbonMetastoreTypes.toDataType(
+ metaData.carbonTable.getMeasureByName(factTable, x.getColName).getDataType.toString
+ .toLowerCase match {
+ case "decimal" => "decimal(" + x.getPrecision + "," + x.getScale + ")"
+ case others => others
+ }),
+ nullable = true)())
+ }
+
+ override val output = {
+ val columns = tableMeta.carbonTable.getCreateOrderColumn(tableMeta.carbonTable.getFactTableName)
+ .asScala
+ // convert each column to Attribute
+ columns.filter(!_.isInvisible).map { column =>
+ if (column.isDimension()) {
+ val output: DataType = column.getDataType.toString.toLowerCase match {
+ case "array" =>
+ CarbonMetastoreTypes.toDataType(s"array<${getArrayChildren(column.getColName)}>")
+ case "struct" =>
+ CarbonMetastoreTypes.toDataType(s"struct<${getStructChildren(column.getColName)}>")
+ case dType =>
+ val dataType = addDecimalScaleAndPrecision(column, dType)
+ CarbonMetastoreTypes.toDataType(dataType)
+ }
+ AttributeReference(column.getColName, output, nullable = true )(
+ qualifier = Option(tableName + "." + column.getColName))
+ } else {
+ val output = CarbonMetastoreTypes.toDataType {
+ column.getDataType.toString
+ .toLowerCase match {
+ case "decimal" => "decimal(" + column.getColumnSchema.getPrecision + "," + column
+ .getColumnSchema.getScale + ")"
+ case others => others
+ }
+ }
+ AttributeReference(column.getColName, output, nullable = true)(
+ qualifier = Option(tableName + "." + column.getColName))
+ }
+ }
+ }
+
+ def addDecimalScaleAndPrecision(dimval: CarbonColumn, dataType: String): String = {
+ var dType = dataType
+ if (dimval.getDataType == DECIMAL) {
+ dType +=
+ "(" + dimval.getColumnSchema.getPrecision + "," + dimval.getColumnSchema.getScale + ")"
+ }
+ dType
+ }
+
+ // TODO: Use data from the footers.
+ override lazy val statistics = Statistics(sizeInBytes = this.sizeInBytes)
+
+ override def equals(other: Any): Boolean = {
+ other match {
+ case p: CarbonRelation =>
+ p.databaseName == databaseName && p.output == output && p.tableName == tableName
+ case _ => false
+ }
+ }
+
+ def addDecimalScaleAndPrecision(dimval: CarbonDimension, dataType: String): String = {
+ var dType = dataType
+ if (dimval.getDataType == DECIMAL) {
+ dType +=
+ "(" + dimval.getColumnSchema.getPrecision + "," + dimval.getColumnSchema.getScale + ")"
+ }
+ dType
+ }
+
+ private var tableStatusLastUpdateTime = 0L
+
+ private var sizeInBytesLocalValue = 0L
+
+ def sizeInBytes: Long = {
+ val tableStatusNewLastUpdatedTime = SegmentStatusManager.getTableStatusLastModifiedTime(
+ tableMeta.carbonTable.getAbsoluteTableIdentifier)
+
+ if (tableStatusLastUpdateTime != tableStatusNewLastUpdatedTime) {
+ val tablePath = CarbonStorePath.getCarbonTablePath(
+ tableMeta.storePath,
+ tableMeta.carbonTableIdentifier).getPath
+ val fileType = FileFactory.getFileType(tablePath)
+ if(FileFactory.isFileExist(tablePath, fileType)) {
+ tableStatusLastUpdateTime = tableStatusNewLastUpdatedTime
+ sizeInBytesLocalValue = FileFactory.getDirectorySize(tablePath)
+ }
+ }
+ sizeInBytesLocalValue
+ }
+
+}
+
+object CarbonMetastoreTypes extends RegexParsers {
+ protected lazy val primitiveType: Parser[DataType] =
+ "string" ^^^ StringType |
+ "float" ^^^ FloatType |
+ "int" ^^^ IntegerType |
+ "tinyint" ^^^ ShortType |
+ "short" ^^^ ShortType |
+ "double" ^^^ DoubleType |
+ "long" ^^^ LongType |
+ "binary" ^^^ BinaryType |
+ "boolean" ^^^ BooleanType |
+ fixedDecimalType |
+ "decimal" ^^^ "decimal" ^^^ DecimalType(10, 0) |
+ "varchar\\((\\d+)\\)".r ^^^ StringType |
+ "date" ^^^ DateType |
+ "timestamp" ^^^ TimestampType
+
+ protected lazy val fixedDecimalType: Parser[DataType] =
+ "decimal" ~> "(" ~> "^[1-9]\\d*".r ~ ("," ~> "^[0-9]\\d*".r <~ ")") ^^ {
+ case precision ~ scale =>
+ DecimalType(precision.toInt, scale.toInt)
+ }
+
+ protected lazy val arrayType: Parser[DataType] =
+ "array" ~> "<" ~> dataType <~ ">" ^^ {
+ case tpe => ArrayType(tpe)
+ }
+
+ protected lazy val mapType: Parser[DataType] =
+ "map" ~> "<" ~> dataType ~ "," ~ dataType <~ ">" ^^ {
+ case t1 ~ _ ~ t2 => MapType(t1, t2)
+ }
+
+ protected lazy val structField: Parser[StructField] =
+ "[a-zA-Z0-9_]*".r ~ ":" ~ dataType ^^ {
+ case name ~ _ ~ tpe => StructField(name, tpe, nullable = true)
+ }
+
+ protected lazy val structType: Parser[DataType] =
+ "struct" ~> "<" ~> repsep(structField, ",") <~ ">" ^^ {
+ case fields => StructType(fields)
+ }
+
+ protected lazy val dataType: Parser[DataType] =
+ arrayType |
+ mapType |
+ structType |
+ primitiveType
+
+ def toDataType(metastoreType: String): DataType = {
+ parseAll(dataType, metastoreType) match {
+ case Success(result, _) => result
+ case failure: NoSuccess => sys.error(s"Unsupported dataType: $metastoreType")
+ }
+ }
+
+ def toMetastoreType(dt: DataType): String = {
+ dt match {
+ case ArrayType(elementType, _) => s"array<${ toMetastoreType(elementType) }>"
+ case StructType(fields) =>
+ s"struct<${
+ fields.map(f => s"${ f.name }:${ toMetastoreType(f.dataType) }")
+ .mkString(",")
+ }>"
+ case StringType => "string"
+ case FloatType => "float"
+ case IntegerType => "int"
+ case ShortType => "tinyint"
+ case DoubleType => "double"
+ case LongType => "bigint"
+ case BinaryType => "binary"
+ case BooleanType => "boolean"
+ case DecimalType() => "decimal"
+ case TimestampType => "timestamp"
+ case DateType => "date"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
index 87717fb..01bdc4f 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
@@ -65,7 +65,7 @@ object AlterTableUtil {
val acquiredLocks = ListBuffer[ICarbonLock]()
try {
locksToBeAcquired.foreach { lock =>
- acquiredLocks += CarbonLockUtil.getLockObject(table, lock)
+ acquiredLocks += CarbonLockUtil.getLockObject(table.getCarbonTableIdentifier, lock)
}
acquiredLocks.toList
} catch {
@@ -133,6 +133,7 @@ object AlterTableUtil {
val tableName = carbonTable.getFactTableName
CarbonEnv.getInstance(sparkSession).carbonMetastore
.updateTableSchema(carbonTable.getCarbonTableIdentifier,
+ carbonTable.getCarbonTableIdentifier,
thriftTable,
schemaEvolutionEntry,
carbonTable.getStorePath)(sparkSession)
@@ -185,14 +186,18 @@ object AlterTableUtil {
(sparkSession: SparkSession): Unit = {
val database = oldTableIdentifier.database.getOrElse(sparkSession.catalog.currentDatabase)
val newCarbonTableIdentifier = new CarbonTableIdentifier(database, newTableName, tableId)
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(storePath,
- newCarbonTableIdentifier)
- val tableMetadataFile = carbonTablePath.getSchemaFilePath
+ val carbonTablePath = CarbonStorePath.getCarbonTablePath(storePath, newCarbonTableIdentifier)
+ val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val tableMetadataFile = carbonTablePath.getPath
val fileType = FileFactory.getFileType(tableMetadataFile)
if (FileFactory.isFileExist(tableMetadataFile, fileType)) {
- val tableInfo: org.apache.carbondata.format.TableInfo = CarbonEnv.getInstance(sparkSession)
- .carbonMetastore
- .readSchemaFile(tableMetadataFile)
+ val tableInfo = if (metastore.isReadFromHiveMetaStore) {
+ // In case of hive metastore we first update the carbonschema inside old table only.
+ metastore.getThriftTableInfo(CarbonStorePath.getCarbonTablePath(storePath,
+ new CarbonTableIdentifier(database, oldTableIdentifier.table, tableId)))(sparkSession)
+ } else {
+ metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
+ }
val evolutionEntryList = tableInfo.fact_table.schema_evolution.schema_evolution_history
val updatedTime = evolutionEntryList.get(evolutionEntryList.size() - 1).time_stamp
if (updatedTime == timeStamp) {
@@ -200,14 +205,9 @@ object AlterTableUtil {
FileFactory.getCarbonFile(carbonTablePath.getPath, fileType)
.renameForce(carbonTablePath.getParent.toString + CarbonCommonConstants.FILE_SEPARATOR +
oldTableIdentifier.table)
- val tableIdentifier = new CarbonTableIdentifier(database,
- oldTableIdentifier.table,
- tableId)
- CarbonEnv.getInstance(sparkSession).carbonMetastore.revertTableSchema(tableIdentifier,
- tableInfo,
- storePath)(sparkSession)
- CarbonEnv.getInstance(sparkSession).carbonMetastore
- .removeTableFromMetadata(database, newTableName)
+ val tableIdentifier = new CarbonTableIdentifier(database, oldTableIdentifier.table, tableId)
+ metastore.revertTableSchema(tableIdentifier, tableInfo, storePath)(sparkSession)
+ metastore.removeTableFromMetadata(database, newTableName)
}
}
}
@@ -222,22 +222,21 @@ object AlterTableUtil {
*/
def revertAddColumnChanges(dbName: String, tableName: String, timeStamp: Long)
(sparkSession: SparkSession): Unit = {
- val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val carbonTable = metastore
.lookupRelation(Some(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation].tableMeta
.carbonTable
val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getStorePath,
carbonTable.getCarbonTableIdentifier)
- val tableMetadataFile = carbonTablePath.getSchemaFilePath
- val thriftTable: TableInfo = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .readSchemaFile(tableMetadataFile)
+ val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
val evolutionEntryList = thriftTable.fact_table.schema_evolution.schema_evolution_history
val updatedTime = evolutionEntryList.get(evolutionEntryList.size() - 1).time_stamp
if (updatedTime == timeStamp) {
LOGGER.info(s"Reverting changes for $dbName.$tableName")
val addedSchemas = evolutionEntryList.get(evolutionEntryList.size() - 1).added
thriftTable.fact_table.table_columns.removeAll(addedSchemas)
- CarbonEnv.getInstance(sparkSession).carbonMetastore
+ metastore
.revertTableSchema(carbonTable.getCarbonTableIdentifier,
thriftTable, carbonTable.getStorePath)(sparkSession)
}
@@ -253,14 +252,13 @@ object AlterTableUtil {
*/
def revertDropColumnChanges(dbName: String, tableName: String, timeStamp: Long)
(sparkSession: SparkSession): Unit = {
- val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val carbonTable = metastore
.lookupRelation(Some(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation].tableMeta
.carbonTable
val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getStorePath,
carbonTable.getCarbonTableIdentifier)
- val tableMetadataFile = carbonTablePath.getSchemaFilePath
- val thriftTable: TableInfo = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .readSchemaFile(tableMetadataFile)
+ val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
val evolutionEntryList = thriftTable.fact_table.schema_evolution.schema_evolution_history
val updatedTime = evolutionEntryList.get(evolutionEntryList.size() - 1).time_stamp
if (updatedTime == timeStamp) {
@@ -273,7 +271,7 @@ object AlterTableUtil {
}
}
}
- CarbonEnv.getInstance(sparkSession).carbonMetastore
+ metastore
.revertTableSchema(carbonTable.getCarbonTableIdentifier,
thriftTable, carbonTable.getStorePath)(sparkSession)
}
@@ -289,14 +287,13 @@ object AlterTableUtil {
*/
def revertDataTypeChanges(dbName: String, tableName: String, timeStamp: Long)
(sparkSession: SparkSession): Unit = {
- val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val carbonTable = metastore
.lookupRelation(Some(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation].tableMeta
.carbonTable
val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getStorePath,
carbonTable.getCarbonTableIdentifier)
- val tableMetadataFile = carbonTablePath.getSchemaFilePath
- val thriftTable: TableInfo = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .readSchemaFile(tableMetadataFile)
+ val thriftTable: TableInfo = metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
val evolutionEntryList = thriftTable.fact_table.schema_evolution.schema_evolution_history
val updatedTime = evolutionEntryList.get(evolutionEntryList.size() - 1).time_stamp
if (updatedTime == timeStamp) {
@@ -312,7 +309,7 @@ object AlterTableUtil {
}
}
}
- CarbonEnv.getInstance(sparkSession).carbonMetastore
+ metastore
.revertTableSchema(carbonTable.getCarbonTableIdentifier,
thriftTable, carbonTable.getStorePath)(sparkSession)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala b/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
index 74e11f1..645081f 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/CleanFiles.scala
@@ -18,6 +18,7 @@
package org.apache.spark.util
import org.apache.spark.sql.{CarbonEnv, SparkSession}
+import org.apache.spark.sql.hive.CarbonRelation
import org.apache.carbondata.api.CarbonStore
@@ -30,8 +31,9 @@ object CleanFiles {
def cleanFiles(spark: SparkSession, dbName: String, tableName: String,
storePath: String): Unit = {
TableAPIUtil.validateTableExists(spark, dbName, tableName)
- val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore
- .getTableFromMetadata(dbName, tableName).map(_.carbonTable).getOrElse(null)
+ val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.
+ lookupRelation(Some(dbName), tableName)(spark).asInstanceOf[CarbonRelation].
+ tableMeta.carbonTable
CarbonStore.cleanFiles(dbName, tableName, storePath, carbonTable)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentByDate.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentByDate.scala b/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentByDate.scala
index 3dffb42..f67a5ce 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentByDate.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentByDate.scala
@@ -16,7 +16,8 @@
*/
package org.apache.spark.util
-import org.apache.spark.sql.{CarbonEnv, SparkSession}
+ import org.apache.spark.sql.{CarbonEnv, SparkSession}
+ import org.apache.spark.sql.hive.CarbonRelation
import org.apache.carbondata.api.CarbonStore
@@ -29,8 +30,9 @@ object DeleteSegmentByDate {
def deleteSegmentByDate(spark: SparkSession, dbName: String, tableName: String,
dateValue: String): Unit = {
TableAPIUtil.validateTableExists(spark, dbName, tableName)
- val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore
- .getTableFromMetadata(dbName, tableName).map(_.carbonTable).getOrElse(null)
+ val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.
+ lookupRelation(Some(dbName), tableName)(spark).asInstanceOf[CarbonRelation].
+ tableMeta.carbonTable
CarbonStore.deleteLoadByDate(dateValue, dbName, tableName, carbonTable)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentById.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentById.scala b/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentById.scala
index 35afa28..bbf386e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentById.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/DeleteSegmentById.scala
@@ -17,6 +17,7 @@
package org.apache.spark.util
import org.apache.spark.sql.{CarbonEnv, SparkSession}
+import org.apache.spark.sql.hive.CarbonRelation
import org.apache.carbondata.api.CarbonStore
@@ -33,8 +34,9 @@ object DeleteSegmentById {
def deleteSegmentById(spark: SparkSession, dbName: String, tableName: String,
segmentIds: Seq[String]): Unit = {
TableAPIUtil.validateTableExists(spark, dbName, tableName)
- val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore
- .getTableFromMetadata(dbName, tableName).map(_.carbonTable).getOrElse(null)
+ val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.
+ lookupRelation(Some(dbName), tableName)(spark).asInstanceOf[CarbonRelation].
+ tableMeta.carbonTable
CarbonStore.deleteLoadById(segmentIds, dbName, tableName, carbonTable)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/main/scala/org/apache/spark/util/ShowSegments.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/ShowSegments.scala b/integration/spark2/src/main/scala/org/apache/spark/util/ShowSegments.scala
index 07dfcc1..d5788ba 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/ShowSegments.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/ShowSegments.scala
@@ -21,6 +21,7 @@ import java.text.SimpleDateFormat
import org.apache.commons.lang3.StringUtils
import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}
+import org.apache.spark.sql.hive.CarbonRelation
import org.apache.carbondata.api.CarbonStore
@@ -30,8 +31,9 @@ object ShowSegments {
def showSegments(spark: SparkSession, dbName: String, tableName: String,
limit: Option[String]): Seq[Row] = {
TableAPIUtil.validateTableExists(spark, dbName, tableName)
- val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore
- .getTableFromMetadata(dbName, tableName).map(_.carbonTable).getOrElse(null)
+ val carbonTable = CarbonEnv.getInstance(spark).carbonMetastore.
+ lookupRelation(Some(dbName), tableName)(spark).asInstanceOf[CarbonRelation].
+ tableMeta.carbonTable
CarbonStore.showSegments(dbName, tableName, limit, carbonTable.getMetaDataFilepath)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
index c584fc7..aa5b87d 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
@@ -23,7 +23,9 @@ import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonStorePath
import org.apache.carbondata.processing.constants.TableOptionConstant
import org.apache.carbondata.processing.model.{CarbonDataLoadSchema, CarbonLoadModel}
@@ -62,6 +64,14 @@ class AllDictionaryTestCase extends QueryTest with BeforeAndAfterAll {
CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT))
carbonLoadModel.setCsvHeaderColumns(CommonUtil.getCsvHeaderColumns(carbonLoadModel))
+ // Create table and metadata folders if not exist
+ val carbonTablePath = CarbonStorePath
+ .getCarbonTablePath(table.getStorePath, table.getCarbonTableIdentifier)
+ val metadataDirectoryPath = carbonTablePath.getMetadataDirectoryPath
+ val fileType = FileFactory.getFileType(metadataDirectoryPath)
+ if (!FileFactory.isFileExist(metadataDirectoryPath, fileType)) {
+ FileFactory.mkdirs(metadataDirectoryPath, fileType)
+ }
carbonLoadModel
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
index 9c576ec..549587f 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
@@ -25,7 +25,9 @@ import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonStorePath
import org.apache.carbondata.processing.constants.TableOptionConstant
import org.apache.carbondata.processing.etl.DataLoadingException
import org.apache.carbondata.processing.model.{CarbonDataLoadSchema, CarbonLoadModel}
@@ -175,6 +177,14 @@ class ExternalColumnDictionaryTestCase extends QueryTest with BeforeAndAfterAll
CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT))
carbonLoadModel.setCsvHeaderColumns(CommonUtil.getCsvHeaderColumns(carbonLoadModel))
carbonLoadModel.setMaxColumns("100")
+ // Create table and metadata folders if not exist
+ val carbonTablePath = CarbonStorePath
+ .getCarbonTablePath(table.getStorePath, table.getCarbonTableIdentifier)
+ val metadataDirectoryPath = carbonTablePath.getMetadataDirectoryPath
+ val fileType = FileFactory.getFileType(metadataDirectoryPath)
+ if (!FileFactory.isFileExist(metadataDirectoryPath, fileType)) {
+ FileFactory.mkdirs(metadataDirectoryPath, fileType)
+ }
carbonLoadModel
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
index 2995e60..1830ed6 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala
@@ -195,6 +195,7 @@ class CarbonDataSourceSuite extends QueryTest with BeforeAndAfterAll {
}
test("test create table with complex datatype") {
+ sql("drop table if exists create_source")
sql("create table create_source(intField int, stringField string, complexField array<string>) USING org.apache.spark.sql.CarbonSource OPTIONS('tableName'='create_source')")
sql("drop table create_source")
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
index 8cdcd26..1788ccb 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionUtil.java
@@ -256,19 +256,18 @@ public class CarbonCompactionUtil {
/**
* This will check if any compaction request has been received for any table.
*
- * @param tableMetas
+ * @param carbonTables
* @return
*/
- public static TableMeta getNextTableToCompact(TableMeta[] tableMetas,
+ public static CarbonTable getNextTableToCompact(CarbonTable[] carbonTables,
List<CarbonTableIdentifier> skipList) {
- for (TableMeta table : tableMetas) {
- CarbonTable ctable = table.carbonTable;
+ for (CarbonTable ctable : carbonTables) {
String metadataPath = ctable.getMetaDataFilepath();
// check for the compaction required file and at the same time exclude the tables which are
// present in the skip list.
if (CarbonCompactionUtil.isCompactionRequiredForTable(metadataPath) && !skipList
- .contains(table.carbonTableIdentifier)) {
- return table;
+ .contains(ctable.getCarbonTableIdentifier())) {
+ return ctable;
}
}
return null;
[08/15] carbondata git commit: [CARBONDATA-1284]Implement hive based
schema storage in carbon
Posted by ra...@apache.org.
[CARBONDATA-1284]Implement hive based schema storage in carbon
This closes #1149
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2a9debfc
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2a9debfc
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2a9debfc
Branch: refs/heads/datamap
Commit: 2a9debfccfdc5633e5e00e95b860efb8b368ea12
Parents: 4b69c9d
Author: ravipesala <ra...@gmail.com>
Authored: Sat Jul 8 17:14:04 2017 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Tue Jul 11 19:48:57 2017 +0800
----------------------------------------------------------------------
.../dictionary/ManageDictionaryAndBTree.java | 39 +-
.../core/constants/CarbonCommonConstants.java | 4 +
.../carbondata/core/locks/CarbonLockUtil.java | 11 +-
.../apache/carbondata/core/util/CarbonUtil.java | 127 +++
.../core/util/path/CarbonTablePath.java | 4 +
.../partition/TestDDLForPartitionTable.scala | 1 +
.../spark/rdd/AlterTableAddColumnRDD.scala | 7 +
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 2 +
.../apache/carbondata/spark/rdd/Compactor.scala | 2 -
.../spark/rdd/DataManagementFunc.scala | 1 -
.../spark/rdd/NewCarbonDataLoadRDD.scala | 4 -
.../execution/command/carbonTableSchema.scala | 3 -
.../spark/rdd/CarbonDataRDDFactory.scala | 47 +-
.../spark/DictionaryDetailHelper.scala | 18 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 44 +-
.../carbondata/spark/util/CarbonSparkUtil.scala | 15 +-
.../sql/CarbonDatasourceHadoopRelation.scala | 8 +-
.../scala/org/apache/spark/sql/CarbonEnv.scala | 6 +-
.../org/apache/spark/sql/CarbonSource.scala | 93 +-
.../execution/command/AlterTableCommands.scala | 53 +-
.../execution/command/CarbonHiveCommands.scala | 14 +-
.../sql/execution/command/DDLStrategy.scala | 14 +-
.../sql/execution/command/IUDCommands.scala | 5 +-
.../execution/command/carbonTableSchema.scala | 87 +-
.../spark/sql/hive/CarbonFileMetastore.scala | 531 ++++++++++
.../spark/sql/hive/CarbonHiveMetaStore.scala | 287 ++++++
.../apache/spark/sql/hive/CarbonMetaStore.scala | 170 ++++
.../apache/spark/sql/hive/CarbonMetastore.scala | 960 -------------------
.../apache/spark/sql/hive/CarbonRelation.scala | 292 ++++++
.../org/apache/spark/util/AlterTableUtil.scala | 57 +-
.../org/apache/spark/util/CleanFiles.scala | 6 +-
.../apache/spark/util/DeleteSegmentByDate.scala | 8 +-
.../apache/spark/util/DeleteSegmentById.scala | 6 +-
.../org/apache/spark/util/ShowSegments.scala | 6 +-
.../spark/util/AllDictionaryTestCase.scala | 10 +
.../util/ExternalColumnDictionaryTestCase.scala | 10 +
.../carbondata/CarbonDataSourceSuite.scala | 1 +
.../processing/merger/CarbonCompactionUtil.java | 11 +-
38 files changed, 1744 insertions(+), 1220 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java
index a50bf15..bae9189 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ManageDictionaryAndBTree.java
@@ -67,25 +67,28 @@ public class ManageDictionaryAndBTree {
String metadataDirectoryPath = carbonTablePath.getMetadataDirectoryPath();
CarbonFile metadataDir = FileFactory
.getCarbonFile(metadataDirectoryPath, FileFactory.getFileType(metadataDirectoryPath));
- // sort index file is created with dictionary size appended to it. So all the files
- // with a given column ID need to be listed
- CarbonFile[] listFiles = metadataDir.listFiles(new CarbonFileFilter() {
- @Override public boolean accept(CarbonFile path) {
- if (path.getName().startsWith(columnSchema.getColumnUniqueId())) {
- return true;
+ if (metadataDir.exists()) {
+ // sort index file is created with dictionary size appended to it. So all the files
+ // with a given column ID need to be listed
+ CarbonFile[] listFiles = metadataDir.listFiles(new CarbonFileFilter() {
+ @Override public boolean accept(CarbonFile path) {
+ if (path.getName().startsWith(columnSchema.getColumnUniqueId())) {
+ return true;
+ }
+ return false;
+ }
+ });
+ for (CarbonFile file : listFiles) {
+ // try catch is inside for loop because even if one deletion fails, other files
+ // still need to be deleted
+ try {
+ FileFactory.deleteFile(file.getCanonicalPath(),
+ FileFactory.getFileType(file.getCanonicalPath()));
+ } catch (IOException e) {
+ LOGGER.error("Failed to delete dictionary or sortIndex file for column "
+ + columnSchema.getColumnName() + "with column ID "
+ + columnSchema.getColumnUniqueId());
}
- return false;
- }
- });
- for (CarbonFile file : listFiles) {
- // try catch is inside for loop because even if one deletion fails, other files
- // still need to be deleted
- try {
- FileFactory
- .deleteFile(file.getCanonicalPath(), FileFactory.getFileType(file.getCanonicalPath()));
- } catch (IOException e) {
- LOGGER.error("Failed to delete dictionary or sortIndex file for column " + columnSchema
- .getColumnName() + "with column ID " + columnSchema.getColumnUniqueId());
}
}
// remove dictionary cache
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 6393131..55a292e 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1264,6 +1264,10 @@ public final class CarbonCommonConstants {
public static final String CARBON_BAD_RECORDS_ACTION_DEFAULT = "FORCE";
+ public static final String ENABLE_HIVE_SCHEMA_META_STORE = "spark.carbon.hive.schema.store";
+
+ public static final String ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT = "false";
+
private CarbonCommonConstants() {
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
index eaaaf94..60a7564 100644
--- a/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
@@ -19,7 +19,7 @@ package org.apache.carbondata.core.locks;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
/**
* This class contains all carbon lock utilities
@@ -66,15 +66,12 @@ public class CarbonLockUtil {
* Given a lock type this method will return a new lock object if not acquired by any other
* operation
*
- * @param carbonTable
+ * @param identifier
* @param lockType
* @return
*/
- public static ICarbonLock getLockObject(CarbonTable carbonTable,
- String lockType) {
- ICarbonLock carbonLock = CarbonLockFactory
- .getCarbonLockObj(carbonTable.getAbsoluteTableIdentifier().getCarbonTableIdentifier(),
- lockType);
+ public static ICarbonLock getLockObject(CarbonTableIdentifier identifier, String lockType) {
+ ICarbonLock carbonLock = CarbonLockFactory.getCarbonLockObj(identifier, lockType);
LOGGER.info("Trying to acquire lock: " + carbonLock);
if (carbonLock.lockWithRetries()) {
LOGGER.info("Successfully acquired the lock " + carbonLock);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 8298600..b9c164a 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -31,8 +31,10 @@ import java.nio.charset.Charset;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
@@ -58,6 +60,7 @@ import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
@@ -70,9 +73,11 @@ import org.apache.carbondata.core.service.PathService;
import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
import org.apache.carbondata.core.util.path.CarbonStorePath;
import org.apache.carbondata.core.util.path.CarbonTablePath;
+import org.apache.carbondata.core.writer.ThriftWriter;
import org.apache.carbondata.format.DataChunk2;
import org.apache.carbondata.format.DataChunk3;
+import com.google.gson.Gson;
import org.apache.commons.lang.ArrayUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
@@ -1720,5 +1725,127 @@ public final class CarbonUtil {
public static boolean isValidBadStorePath(String badRecordsLocation) {
return !(null == badRecordsLocation || badRecordsLocation.length() == 0);
}
+
+ public static String convertToMultiGsonStrings(TableInfo tableInfo, String seperator,
+ String quote, String prefix) {
+ Gson gson = new Gson();
+ String schemaString = gson.toJson(tableInfo);
+ int schemaLen = schemaString.length();
+ int splitLen = 4000;
+ int parts = schemaLen / splitLen;
+ if (schemaLen % splitLen > 0) {
+ parts++;
+ }
+ StringBuilder builder =
+ new StringBuilder(prefix).append(quote).append("carbonSchemaPartsNo").append(quote)
+ .append(seperator).append("'").append(parts).append("',");
+ int runningLen = 0;
+ int endLen = splitLen;
+ for (int i = 0; i < parts; i++) {
+ if (i == parts - 1) {
+ endLen = schemaLen % splitLen;
+ }
+ builder.append(quote).append("carbonSchema").append(i).append(quote).append(seperator);
+ builder.append("'").append(schemaString.substring(runningLen, runningLen + endLen))
+ .append("'");
+ if (i < parts - 1) {
+ builder.append(",");
+ }
+ runningLen += splitLen;
+ }
+ return builder.toString();
+ }
+
+ public static Map<String, String> convertToMultiStringMap(TableInfo tableInfo) {
+ Map<String, String> map = new HashMap<>();
+ Gson gson = new Gson();
+ String schemaString = gson.toJson(tableInfo);
+ int schemaLen = schemaString.length();
+ int splitLen = 4000;
+ int parts = schemaLen / splitLen;
+ if (schemaLen % splitLen > 0) {
+ parts++;
+ }
+ map.put("carbonSchemaPartsNo", parts + "");
+ int runningLen = 0;
+ int endLen = splitLen;
+ for (int i = 0; i < parts; i++) {
+ if (i == parts - 1) {
+ endLen = schemaLen % splitLen;
+ }
+ map.put("carbonSchema" + i, schemaString.substring(runningLen, runningLen + endLen));
+ runningLen += splitLen;
+ }
+ return map;
+ }
+
+
+ public static TableInfo convertGsonToTableInfo(Map<String, String> properties) {
+ Gson gson = new Gson();
+ String partsNo = properties.get("carbonSchemaPartsNo");
+ if (partsNo == null) {
+ return null;
+ }
+ int no = Integer.parseInt(partsNo);
+ StringBuilder builder = new StringBuilder();
+ for (int i = 0; i < no; i++) {
+ String part = properties.get("carbonSchema" + i);
+ if (part == null) {
+ throw new RuntimeException("Some thing wrong in getting schema from hive metastore");
+ }
+ builder.append(part);
+ }
+ TableInfo tableInfo = gson.fromJson(builder.toString(), TableInfo.class);
+ return tableInfo;
+ }
+
+ /**
+ * This method will read the schema file from a given path
+ *
+ * @param schemaFilePath
+ * @return
+ */
+ public static org.apache.carbondata.format.TableInfo readSchemaFile(String schemaFilePath)
+ throws IOException {
+ TBaseCreator createTBase = new ThriftReader.TBaseCreator() {
+ public org.apache.thrift.TBase<org.apache.carbondata.format.TableInfo,
+ org.apache.carbondata.format.TableInfo._Fields> create() {
+ return new org.apache.carbondata.format.TableInfo();
+ }
+ };
+ ThriftReader thriftReader = new ThriftReader(schemaFilePath, createTBase);
+ thriftReader.open();
+ org.apache.carbondata.format.TableInfo tableInfo =
+ (org.apache.carbondata.format.TableInfo) thriftReader.read();
+ thriftReader.close();
+ return tableInfo;
+ }
+
+ public static void writeThriftTableToSchemaFile(String schemaFilePath,
+ org.apache.carbondata.format.TableInfo tableInfo) throws IOException {
+ ThriftWriter thriftWriter = new ThriftWriter(schemaFilePath, false);
+ try {
+ thriftWriter.open();
+ thriftWriter.write(tableInfo);
+ } finally {
+ thriftWriter.close();
+ }
+ }
+
+ public static void createDatabaseDirectory(String dbName, String storePath) throws IOException {
+ String databasePath = storePath + File.separator + dbName.toLowerCase();
+ FileFactory.FileType fileType = FileFactory.getFileType(databasePath);
+ FileFactory.mkdirs(databasePath, fileType);
+ }
+
+ public static void dropDatabaseDirectory(String dbName, String storePath)
+ throws IOException, InterruptedException {
+ String databasePath = storePath + File.separator + dbName;
+ FileFactory.FileType fileType = FileFactory.getFileType(databasePath);
+ if (FileFactory.isFileExist(databasePath, fileType)) {
+ CarbonFile dbPath = FileFactory.getCarbonFile(databasePath, fileType);
+ CarbonUtil.deleteFoldersAndFiles(dbPath);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index 101419d..5824d76 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -432,6 +432,10 @@ public class CarbonTablePath extends Path {
return tablePath + File.separator + FACT_DIR;
}
+ public CarbonTableIdentifier getCarbonTableIdentifier() {
+ return carbonTableIdentifier;
+ }
+
@Override public boolean equals(Object o) {
if (!(o instanceof CarbonTablePath)) {
return false;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
index 5b77a24..c38b249 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala
@@ -128,6 +128,7 @@ class TestDDLForPartitionTable extends QueryTest with BeforeAndAfterAll {
}
test("test describe formatted for partition column") {
+ sql("drop table if exists des")
sql(
"""create table des(a int, b string) partitioned by (c string) stored by 'carbondata'
|tblproperties ('partition_type'='list','list_info'='1,2')""".stripMargin)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
index 7eea95d..2fb72f7 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/AlterTableAddColumnRDD.scala
@@ -22,6 +22,7 @@ import org.apache.spark.rdd.RDD
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.CarbonTableIdentifier
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
@@ -80,6 +81,12 @@ class AlterTableAddColumnRDD[K, V](sc: SparkContext,
CarbonCommonConstants.DEFAULT_CHARSET_CLASS)
}
CarbonProperties.getInstance.addProperty(CarbonCommonConstants.LOCK_TYPE, lockType)
+ // Create table and metadata folders if not exist
+ val metadataDirectoryPath = carbonTablePath.getMetadataDirectoryPath
+ val fileType = FileFactory.getFileType(metadataDirectoryPath)
+ if (!FileFactory.isFileExist(metadataDirectoryPath, fileType)) {
+ FileFactory.mkdirs(metadataDirectoryPath, fileType)
+ }
GlobalDictionaryUtil
.loadDefaultDictionaryValueForNewColumn(carbonTablePath,
columnSchema,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index 815dba3..bc5ca06 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -265,6 +265,8 @@ class CarbonMergerRDD[K, V](
val jobConf: JobConf = new JobConf(new Configuration)
val job: Job = new Job(jobConf)
val format = CarbonInputFormatUtil.createCarbonInputFormat(absoluteTableIdentifier, job)
+ CarbonInputFormat.setTableInfo(job.getConfiguration,
+ carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable.getTableInfo)
var updateDetails: UpdateVO = null
// initialise query_id for job
job.getConfiguration.set("query.id", queryId)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/Compactor.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/Compactor.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/Compactor.scala
index 1a237f6..4cebcd8 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/Compactor.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/Compactor.scala
@@ -39,7 +39,6 @@ object Compactor {
val storePath = compactionCallableModel.storePath
val storeLocation = compactionCallableModel.storeLocation
val carbonTable = compactionCallableModel.carbonTable
- val cubeCreationTime = compactionCallableModel.cubeCreationTime
val loadsToMerge = compactionCallableModel.loadsToMerge
val sc = compactionCallableModel.sqlContext
val carbonLoadModel = compactionCallableModel.carbonLoadModel
@@ -57,7 +56,6 @@ object Compactor {
storePath,
carbonTable.getMetaDataFilepath,
mergedLoadName,
- cubeCreationTime,
databaseName,
factTableName,
validSegments,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
index 954303f..5ab8160 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
@@ -250,7 +250,6 @@ object DataManagementFunc {
carbonLoadModel,
storeLocation,
compactionModel.carbonTable,
- compactionModel.tableCreationTime,
loadsToMerge,
sqlContext,
compactionModel.compactionType
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index 129c642..7cbe9a4 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -388,8 +388,6 @@ class NewDataFrameLoaderRDD[K, V](
result: DataLoadResult[K, V],
carbonLoadModel: CarbonLoadModel,
loadCount: Integer,
- tableCreationTime: Long,
- schemaLastUpdatedTime: Long,
prev: DataLoadCoalescedRDD[Row]) extends CarbonRDD[(K, V)](prev) {
override def internalCompute(theSplit: Partition, context: TaskContext): Iterator[(K, V)] = {
@@ -583,8 +581,6 @@ class PartitionTableDataLoaderRDD[K, V](
result: DataLoadResult[K, V],
carbonLoadModel: CarbonLoadModel,
loadCount: Integer,
- tableCreationTime: Long,
- schemaLastUpdatedTime: Long,
prev: RDD[Row]) extends CarbonRDD[(K, V)](prev) {
override def internalCompute(theSplit: Partition, context: TaskContext): Iterator[(K, V)] = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 6174f7c..500e18e 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -90,7 +90,6 @@ case class CarbonMergerMapping(storeLocation: String,
hdfsStoreLocation: String,
metadataFilePath: String,
var mergedLoadName: String,
- tableCreationTime: Long,
databaseName: String,
factTableName: String,
validSegments: Array[String],
@@ -117,14 +116,12 @@ case class UpdateTableModel(isUpdate: Boolean,
case class CompactionModel(compactionSize: Long,
compactionType: CompactionType,
carbonTable: CarbonTable,
- tableCreationTime: Long,
isDDLTrigger: Boolean)
case class CompactionCallableModel(storePath: String,
carbonLoadModel: CarbonLoadModel,
storeLocation: String,
carbonTable: CarbonTable,
- cubeCreationTime: Long,
loadsToMerge: util.List[LoadMetadataDetails],
sqlContext: SQLContext,
compactionType: CompactionType)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 3579b8a..31b05bc 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -104,8 +104,6 @@ object CarbonDataRDDFactory {
LOGGER.audit(s"Compaction request received for table " +
s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
- val tableCreationTime = CarbonEnv.get.carbonMetastore
- .getTableCreationTime(carbonLoadModel.getDatabaseName, carbonLoadModel.getTableName)
if (null == carbonLoadModel.getLoadMetadataDetails) {
CommonUtil.readLoadMetadataDetails(carbonLoadModel, storePath)
@@ -118,7 +116,6 @@ object CarbonDataRDDFactory {
val compactionModel = CompactionModel(compactionSize,
compactionType,
carbonTable,
- tableCreationTime,
isCompactionTriggerByDDl
)
@@ -272,23 +269,19 @@ object CarbonDataRDDFactory {
if (!isConcurrentCompactionAllowed) {
LOGGER.info("System level compaction lock is enabled.")
val skipCompactionTables = ListBuffer[CarbonTableIdentifier]()
- var tableForCompaction = CarbonCompactionUtil
- .getNextTableToCompact(CarbonEnv.get.carbonMetastore.metadata.tablesMeta.toArray,
+ var table: CarbonTable = CarbonCompactionUtil
+ .getNextTableToCompact(CarbonEnv.get.carbonMetastore.metadata.
+ tablesMeta.map(_.carbonTable).toArray,
skipCompactionTables.toList.asJava)
- while (null != tableForCompaction) {
+ while (null != table) {
LOGGER.info("Compaction request has been identified for table " +
- s"${ tableForCompaction.carbonTable.getDatabaseName }." +
- s"${ tableForCompaction.carbonTableIdentifier.getTableName }")
- val table: CarbonTable = tableForCompaction.carbonTable
+ s"${ table.getDatabaseName }." +
+ s"${ table.getFactTableName }")
val metadataPath = table.getMetaDataFilepath
val compactionType = CarbonCompactionUtil.determineCompactionType(metadataPath)
val newCarbonLoadModel = new CarbonLoadModel()
DataManagementFunc.prepareCarbonLoadModel(storePath, table, newCarbonLoadModel)
- val tableCreationTime = CarbonEnv.get.carbonMetastore
- .getTableCreationTime(newCarbonLoadModel.getDatabaseName,
- newCarbonLoadModel.getTableName
- )
val compactionSize = CarbonDataMergerUtil
.getCompactionSize(CompactionType.MAJOR_COMPACTION)
@@ -296,7 +289,6 @@ object CarbonDataRDDFactory {
val newcompactionModel = CompactionModel(compactionSize,
compactionType,
table,
- tableCreationTime,
compactionModel.isDDLTrigger
)
// proceed for compaction
@@ -309,8 +301,8 @@ object CarbonDataRDDFactory {
} catch {
case e: Exception =>
LOGGER.error("Exception in compaction thread for table " +
- s"${ tableForCompaction.carbonTable.getDatabaseName }." +
- s"${ tableForCompaction.carbonTableIdentifier.getTableName }")
+ s"${ table.getDatabaseName }." +
+ s"${ table.getFactTableName }")
// not handling the exception. only logging as this is not the table triggered
// by user.
} finally {
@@ -319,16 +311,16 @@ object CarbonDataRDDFactory {
.deleteCompactionRequiredFile(metadataPath, compactionType)) {
// if the compaction request file is not been able to delete then
// add those tables details to the skip list so that it wont be considered next.
- skipCompactionTables.+=:(tableForCompaction.carbonTableIdentifier)
+ skipCompactionTables.+=:(table.getCarbonTableIdentifier)
LOGGER.error("Compaction request file can not be deleted for table " +
- s"${ tableForCompaction.carbonTable.getDatabaseName }." +
- s"${ tableForCompaction.carbonTableIdentifier.getTableName }")
+ s"${ table.getDatabaseName }." +
+ s"${ table.getFactTableName }")
}
}
// ********* check again for all the tables.
- tableForCompaction = CarbonCompactionUtil
+ table = CarbonCompactionUtil
.getNextTableToCompact(CarbonEnv.get.carbonMetastore.metadata
- .tablesMeta.toArray, skipCompactionTables.asJava
+ .tablesMeta.map(_.carbonTable).toArray, skipCompactionTables.asJava
)
}
// giving the user his error for telling in the beeline if his triggered table
@@ -360,7 +352,7 @@ object CarbonDataRDDFactory {
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
val isAgg = false
// for handling of the segment Merging.
- def handleSegmentMerging(tableCreationTime: Long): Unit = {
+ def handleSegmentMerging(): Unit = {
LOGGER.info(s"compaction need status is" +
s" ${ CarbonDataMergerUtil.checkIfAutoLoadMergingRequired() }")
if (CarbonDataMergerUtil.checkIfAutoLoadMergingRequired()) {
@@ -371,7 +363,6 @@ object CarbonDataRDDFactory {
val compactionModel = CompactionModel(compactionSize,
CompactionType.MINOR_COMPACTION,
carbonTable,
- tableCreationTime,
isCompactionTriggerByDDl
)
var storeLocation = ""
@@ -479,10 +470,6 @@ object CarbonDataRDDFactory {
// reading the start time of data load.
val loadStartTime = CarbonUpdateUtil.readCurrentTime();
carbonLoadModel.setFactTimeStamp(loadStartTime)
- val tableCreationTime = CarbonEnv.get.carbonMetastore
- .getTableCreationTime(carbonLoadModel.getDatabaseName, carbonLoadModel.getTableName)
- val schemaLastUpdatedTime = CarbonEnv.get.carbonMetastore
- .getSchemaLastUpdatedTime(carbonLoadModel.getDatabaseName, carbonLoadModel.getTableName)
// get partition way from configuration
// val isTableSplitPartition = CarbonProperties.getInstance().getProperty(
@@ -634,8 +621,6 @@ object CarbonDataRDDFactory {
new DataLoadResultImpl(),
carbonLoadModel,
currentLoadCount,
- tableCreationTime,
- schemaLastUpdatedTime,
newRdd).collect()
} catch {
case ex: Exception =>
@@ -748,8 +733,6 @@ object CarbonDataRDDFactory {
new DataLoadResultImpl(),
carbonLoadModel,
currentLoadCount,
- tableCreationTime,
- schemaLastUpdatedTime,
rdd).collect()
} catch {
case ex: Exception =>
@@ -974,7 +957,7 @@ object CarbonDataRDDFactory {
}
try {
// compaction handling
- handleSegmentMerging(tableCreationTime)
+ handleSegmentMerging()
} catch {
case e: Exception =>
throw new Exception(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
index 32ba6cf..779ace1 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
@@ -36,15 +36,17 @@ class DictionaryDetailHelper extends DictionaryDetailService {
// Metadata folder
val metadataDirectory = FileFactory.getCarbonFile(dictfolderPath, fileType)
// need list all dictionary file paths with exists flag
- val carbonFiles = metadataDirectory.listFiles(new CarbonFileFilter {
- @Override def accept(pathname: CarbonFile): Boolean = {
- CarbonTablePath.isDictionaryFile(pathname)
- }
- })
- // 2 put dictionary file names to fileNamesMap
val fileNamesMap = new HashMap[String, Int]
- for (i <- 0 until carbonFiles.length) {
- fileNamesMap.put(carbonFiles(i).getName, i)
+ if (metadataDirectory.exists()) {
+ val carbonFiles = metadataDirectory.listFiles(new CarbonFileFilter {
+ @Override def accept(pathname: CarbonFile): Boolean = {
+ CarbonTablePath.isDictionaryFile(pathname)
+ }
+ })
+ // 2 put dictionary file names to fileNamesMap
+ for (i <- 0 until carbonFiles.length) {
+ fileNamesMap.put(carbonFiles(i).getName, i)
+ }
}
// 3 lookup fileNamesMap, if file name is in fileNamesMap, file is exists, or not.
primDimensions.zipWithIndex.foreach { f =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 5c20808..fc813d1 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -105,8 +105,6 @@ object CarbonDataRDDFactory {
LOGGER.audit(s"Compaction request received for table " +
s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName }")
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
- val tableCreationTime = CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore
- .getTableCreationTime(carbonLoadModel.getDatabaseName, carbonLoadModel.getTableName)
if (null == carbonLoadModel.getLoadMetadataDetails) {
CommonUtil.readLoadMetadataDetails(carbonLoadModel, storePath)
@@ -124,7 +122,6 @@ object CarbonDataRDDFactory {
val compactionModel = CompactionModel(compactionSize,
compactionType,
carbonTable,
- tableCreationTime,
isCompactionTriggerByDDl
)
@@ -285,22 +282,18 @@ object CarbonDataRDDFactory {
val skipCompactionTables = ListBuffer[CarbonTableIdentifier]()
var tableForCompaction = CarbonCompactionUtil
.getNextTableToCompact(CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore
- .metadata.tablesMeta.toArray,
+ .listAllTables(sqlContext.sparkSession).toArray,
skipCompactionTables.toList.asJava)
while (null != tableForCompaction) {
LOGGER.info("Compaction request has been identified for table " +
- s"${ tableForCompaction.carbonTable.getDatabaseName }." +
- s"${ tableForCompaction.carbonTableIdentifier.getTableName }")
- val table: CarbonTable = tableForCompaction.carbonTable
+ s"${ tableForCompaction.getDatabaseName }." +
+ s"${ tableForCompaction.getFactTableName}")
+ val table: CarbonTable = tableForCompaction
val metadataPath = table.getMetaDataFilepath
val compactionType = CarbonCompactionUtil.determineCompactionType(metadataPath)
val newCarbonLoadModel = new CarbonLoadModel()
DataManagementFunc.prepareCarbonLoadModel(storePath, table, newCarbonLoadModel)
- val tableCreationTime = CarbonEnv.getInstance(sqlContext.sparkSession)
- .carbonMetastore.getTableCreationTime(newCarbonLoadModel.getDatabaseName,
- newCarbonLoadModel.getTableName
- )
val compactionSize = CarbonDataMergerUtil
.getCompactionSize(CompactionType.MAJOR_COMPACTION)
@@ -308,7 +301,6 @@ object CarbonDataRDDFactory {
val newcompactionModel = CompactionModel(compactionSize,
compactionType,
table,
- tableCreationTime,
compactionModel.isDDLTrigger
)
// proceed for compaction
@@ -321,8 +313,8 @@ object CarbonDataRDDFactory {
} catch {
case e: Exception =>
LOGGER.error("Exception in compaction thread for table " +
- s"${ tableForCompaction.carbonTable.getDatabaseName }." +
- s"${ tableForCompaction.carbonTableIdentifier.getTableName }")
+ s"${ tableForCompaction.getDatabaseName }." +
+ s"${ tableForCompaction.getFactTableName }")
// not handling the exception. only logging as this is not the table triggered
// by user.
} finally {
@@ -331,17 +323,17 @@ object CarbonDataRDDFactory {
.deleteCompactionRequiredFile(metadataPath, compactionType)) {
// if the compaction request file is not been able to delete then
// add those tables details to the skip list so that it wont be considered next.
- skipCompactionTables.+=:(tableForCompaction.carbonTableIdentifier)
+ skipCompactionTables.+=:(tableForCompaction.getCarbonTableIdentifier)
LOGGER.error("Compaction request file can not be deleted for table " +
- s"${ tableForCompaction.carbonTable.getDatabaseName }." +
- s"${ tableForCompaction.carbonTableIdentifier.getTableName }")
+ s"${ tableForCompaction.getDatabaseName }." +
+ s"${ tableForCompaction.getFactTableName }")
}
}
// ********* check again for all the tables.
tableForCompaction = CarbonCompactionUtil
.getNextTableToCompact(CarbonEnv.getInstance(sqlContext.sparkSession)
- .carbonMetastore.metadata
- .tablesMeta.toArray, skipCompactionTables.asJava
+ .carbonMetastore.listAllTables(sqlContext.sparkSession).toArray,
+ skipCompactionTables.asJava
)
}
}
@@ -373,7 +365,7 @@ object CarbonDataRDDFactory {
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
val isAgg = false
// for handling of the segment Merging.
- def handleSegmentMerging(tableCreationTime: Long): Unit = {
+ def handleSegmentMerging(): Unit = {
LOGGER.info(s"compaction need status is" +
s" ${ CarbonDataMergerUtil.checkIfAutoLoadMergingRequired() }")
if (CarbonDataMergerUtil.checkIfAutoLoadMergingRequired()) {
@@ -384,7 +376,6 @@ object CarbonDataRDDFactory {
val compactionModel = CompactionModel(compactionSize,
CompactionType.MINOR_COMPACTION,
carbonTable,
- tableCreationTime,
isCompactionTriggerByDDl
)
var storeLocation = ""
@@ -492,11 +483,6 @@ object CarbonDataRDDFactory {
// reading the start time of data load.
val loadStartTime = CarbonUpdateUtil.readCurrentTime();
carbonLoadModel.setFactTimeStamp(loadStartTime)
- val tableCreationTime = CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore
- .getTableCreationTime(carbonLoadModel.getDatabaseName, carbonLoadModel.getTableName)
- val schemaLastUpdatedTime = CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore
- .getSchemaLastUpdatedTime(carbonLoadModel.getDatabaseName, carbonLoadModel.getTableName)
-
// get partition way from configuration
// val isTableSplitPartition = CarbonProperties.getInstance().getProperty(
// CarbonCommonConstants.TABLE_SPLIT_PARTITION,
@@ -645,8 +631,6 @@ object CarbonDataRDDFactory {
new DataLoadResultImpl(),
carbonLoadModel,
currentLoadCount,
- tableCreationTime,
- schemaLastUpdatedTime,
newRdd).collect()
} catch {
@@ -760,8 +744,6 @@ object CarbonDataRDDFactory {
new DataLoadResultImpl(),
carbonLoadModel,
currentLoadCount,
- tableCreationTime,
- schemaLastUpdatedTime,
rdd).collect()
} catch {
case ex: Exception =>
@@ -998,7 +980,7 @@ object CarbonDataRDDFactory {
}
try {
// compaction handling
- handleSegmentMerging(tableCreationTime)
+ handleSegmentMerging()
} catch {
case e: Exception =>
throw new Exception(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala
index 2f65fbc..d1d3015 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala
@@ -19,11 +19,13 @@ package org.apache.carbondata.spark.util
import scala.collection.JavaConverters._
-import org.apache.spark.sql.hive.{CarbonMetaData, DictionaryMap}
+import org.apache.spark.sql.hive.{CarbonMetaData, CarbonRelation, DictionaryMap}
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.encoder.Encoding
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
import org.apache.carbondata.core.util.CarbonUtil
+import org.apache.carbondata.processing.merger.TableMeta
case class TransformHolder(rdd: Any, mataData: CarbonMetaData)
@@ -42,4 +44,13 @@ object CarbonSparkUtil {
}
CarbonMetaData(dimensionsAttr, measureAttr, carbonTable, DictionaryMap(dictionary.toMap))
}
+
+ def createCarbonRelation(tableInfo: TableInfo, tablePath: String): CarbonRelation = {
+ val identifier = AbsoluteTableIdentifier.fromTablePath(tablePath)
+ val table = CarbonTable.buildFromTableInfo(tableInfo)
+ val meta = new TableMeta(identifier.getCarbonTableIdentifier, identifier.getStorePath, table)
+ CarbonRelation(tableInfo.getDatabaseName, tableInfo.getFactTable.getTableName,
+ CarbonSparkUtil.createSparkMeta(table), meta)
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
index d1baf79..7411e6e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
@@ -54,11 +54,9 @@ case class CarbonDatasourceHadoopRelation(
ThreadLocalSessionInfo.setCarbonSessionInfo(carbonSessionInfo)
@transient lazy val carbonRelation: CarbonRelation =
- CarbonEnv.getInstance(sparkSession).carbonMetastore
- .lookupRelation(
- Some(identifier.getCarbonTableIdentifier.getDatabaseName),
- identifier.getCarbonTableIdentifier.getTableName)(sparkSession)
- .asInstanceOf[CarbonRelation]
+ CarbonEnv.getInstance(sparkSession).carbonMetastore.
+ createCarbonRelation(parameters, identifier, sparkSession)
+
@transient lazy val carbonTable: CarbonTable = carbonRelation.tableMeta.carbonTable
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
index 925b82b..d19eb39 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonEnv.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql
import java.util.Map
import java.util.concurrent.ConcurrentHashMap
-import org.apache.spark.sql.hive.{CarbonMetastore, CarbonSessionCatalog}
+import org.apache.spark.sql.hive.{CarbonMetaStore, CarbonMetaStoreFactory, CarbonSessionCatalog}
import org.apache.spark.sql.internal.CarbonSQLConf
import org.apache.carbondata.common.logging.LogServiceFactory
@@ -34,7 +34,7 @@ import org.apache.carbondata.spark.readsupport.SparkRowReadSupportImpl
*/
class CarbonEnv {
- var carbonMetastore: CarbonMetastore = _
+ var carbonMetastore: CarbonMetaStore = _
var sessionParams: SessionParams = _
@@ -64,7 +64,7 @@ class CarbonEnv {
val storePath =
CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION)
LOGGER.info(s"carbon env initial: $storePath")
- new CarbonMetastore(sparkSession.conf, storePath)
+ CarbonMetaStoreFactory.createCarbonMetaStore(sparkSession.conf, storePath)
}
CarbonProperties.getInstance.addProperty(CarbonCommonConstants.IS_DRIVER_INSTANCE, "true")
initialized = true
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
index 1c16143..498ea03 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
@@ -17,20 +17,23 @@
package org.apache.spark.sql
+import scala.collection.JavaConverters._
import scala.language.implicitConversions
import org.apache.commons.lang.StringUtils
import org.apache.hadoop.fs.Path
import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
+import org.apache.spark.sql.catalyst.catalog.CatalogTable
import org.apache.spark.sql.execution.CarbonLateDecodeStrategy
-import org.apache.spark.sql.execution.command.{BucketFields, CreateTable, Field}
+import org.apache.spark.sql.execution.command.{BucketFields, CreateTable, Field, TableModel, TableNewProcessor}
import org.apache.spark.sql.optimizer.CarbonLateDecodeRule
import org.apache.spark.sql.parser.CarbonSpark2SqlParser
import org.apache.spark.sql.sources._
import org.apache.spark.sql.types.{DecimalType, StructType}
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.metadata.schema.table.TableInfo
+import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.apache.carbondata.spark.CarbonOption
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
@@ -125,7 +128,7 @@ class CarbonSource extends CreatableRelationProvider with RelationProvider
}
val path = if (sqlContext.sparkSession.sessionState.catalog.listTables(dbName)
.exists(_.table.equalsIgnoreCase(tableName))) {
- getPathForTable(sqlContext.sparkSession, dbName, tableName)
+ getPathForTable(sqlContext.sparkSession, dbName, tableName, parameters)
} else {
createTableIfNotExists(sqlContext.sparkSession, parameters, dataSchema)
}
@@ -148,20 +151,22 @@ class CarbonSource extends CreatableRelationProvider with RelationProvider
val dbName: String = parameters.getOrElse("dbName",
CarbonCommonConstants.DATABASE_DEFAULT_NAME).toLowerCase
val tableName: String = parameters.getOrElse("tableName", "").toLowerCase
+
try {
- CarbonEnv.getInstance(sparkSession).carbonMetastore
- .lookupRelation(Option(dbName), tableName)(sparkSession)
- CarbonEnv.getInstance(sparkSession).carbonMetastore.storePath + s"/$dbName/$tableName"
+ if (parameters.contains("carbonSchemaPartsNo")) {
+ getPathForTable(sparkSession, dbName, tableName, parameters)
+ } else {
+ CarbonEnv.getInstance(sparkSession).carbonMetastore
+ .lookupRelation(Option(dbName), tableName)(sparkSession)
+ CarbonEnv.getInstance(sparkSession).carbonMetastore.storePath + s"/$dbName/$tableName"
+ }
} catch {
case ex: NoSuchTableException =>
- val sqlParser = new CarbonSpark2SqlParser
- val fields = sqlParser.getFields(dataSchema)
- val map = scala.collection.mutable.Map[String, String]()
- parameters.foreach { case (key, value) => map.put(key, value.toLowerCase()) }
- val options = new CarbonOption(parameters)
- val bucketFields = sqlParser.getBucketFields(map, fields, options)
- val cm = sqlParser.prepareTableModel(ifNotExistPresent = false, Option(dbName),
- tableName, fields, Nil, map, bucketFields)
+ val cm: TableModel = CarbonSource.createTableInfoFromParams(
+ parameters,
+ dataSchema,
+ dbName,
+ tableName)
CreateTable(cm, false).run(sparkSession)
CarbonEnv.getInstance(sparkSession).carbonMetastore.storePath + s"/$dbName/$tableName"
case ex: Exception =>
@@ -171,13 +176,14 @@ class CarbonSource extends CreatableRelationProvider with RelationProvider
/**
* Returns the path of the table
+ *
* @param sparkSession
* @param dbName
* @param tableName
* @return
*/
private def getPathForTable(sparkSession: SparkSession, dbName: String,
- tableName : String): String = {
+ tableName : String, parameters: Map[String, String]): String = {
if (StringUtils.isBlank(tableName)) {
throw new MalformedCarbonCommandException("The Specified Table Name is Blank")
@@ -186,9 +192,13 @@ class CarbonSource extends CreatableRelationProvider with RelationProvider
throw new MalformedCarbonCommandException("Table Name Should not have spaces ")
}
try {
- CarbonEnv.getInstance(sparkSession).carbonMetastore
- .lookupRelation(Option(dbName), tableName)(sparkSession)
- CarbonEnv.getInstance(sparkSession).carbonMetastore.storePath + s"/$dbName/$tableName"
+ if (parameters.contains("tablePath")) {
+ parameters.get("tablePath").get
+ } else {
+ CarbonEnv.getInstance(sparkSession).carbonMetastore
+ .lookupRelation(Option(dbName), tableName)(sparkSession)
+ CarbonEnv.getInstance(sparkSession).carbonMetastore.storePath + s"/$dbName/$tableName"
+ }
} catch {
case ex: Exception =>
throw new Exception(s"Do not have $dbName and $tableName", ex)
@@ -196,3 +206,50 @@ class CarbonSource extends CreatableRelationProvider with RelationProvider
}
}
+
+object CarbonSource {
+
+ def createTableInfoFromParams(parameters: Map[String, String],
+ dataSchema: StructType,
+ dbName: String,
+ tableName: String): TableModel = {
+ val sqlParser = new CarbonSpark2SqlParser
+ val fields = sqlParser.getFields(dataSchema)
+ val map = scala.collection.mutable.Map[String, String]()
+ parameters.foreach { case (key, value) => map.put(key, value.toLowerCase()) }
+ val options = new CarbonOption(parameters)
+ val bucketFields = sqlParser.getBucketFields(map, fields, options)
+ sqlParser.prepareTableModel(ifNotExistPresent = false, Option(dbName),
+ tableName, fields, Nil, map, bucketFields)
+ }
+
+ /**
+ * Update spark catalog table with schema information in case of schema storage is hive metastore
+ * @param tableDesc
+ * @param sparkSession
+ * @return
+ */
+ def updateCatalogTableWithCarbonSchema(tableDesc: CatalogTable,
+ sparkSession: SparkSession): CatalogTable = {
+ val metaStore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val storageFormat = tableDesc.storage
+ val properties = storageFormat.properties
+ if (metaStore.isReadFromHiveMetaStore && !properties.contains("carbonSchemaPartsNo")) {
+ val dbName: String = properties.getOrElse("dbName",
+ CarbonCommonConstants.DATABASE_DEFAULT_NAME).toLowerCase
+ val tableName: String = properties.getOrElse("tableName", "").toLowerCase
+ val model = createTableInfoFromParams(properties, tableDesc.schema, dbName, tableName)
+ val tableInfo: TableInfo = TableNewProcessor(model)
+ val (tablePath, carbonSchemaString) =
+ metaStore.createTableFromThrift(tableInfo, dbName, tableName)(sparkSession)
+ val map = CarbonUtil.convertToMultiStringMap(tableInfo)
+ properties.foreach(e => map.put(e._1, e._2))
+ map.put("tablePath", tablePath)
+ // updating params
+ val updatedFormat = storageFormat.copy(properties = map.asScala.toMap)
+ tableDesc.copy(storage = updatedFormat)
+ } else {
+ tableDesc
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommands.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommands.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommands.scala
index b08c113..0d5a821 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommands.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/AlterTableCommands.scala
@@ -63,16 +63,15 @@ private[sql] case class AlterTableAddColumns(
// completion of 1st operation but as look up relation is called before it will have the
// older carbon table and this can lead to inconsistent state in the system. Therefor look
// up relation should be called after acquiring the lock
- carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ carbonTable = metastore
.lookupRelation(Some(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation]
.tableMeta.carbonTable
// get the latest carbon table and check for column existence
// read the latest schema file
val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getStorePath,
carbonTable.getCarbonTableIdentifier)
- val tableMetadataFile = carbonTablePath.getSchemaFilePath
- val thriftTableInfo: TableInfo = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .readSchemaFile(tableMetadataFile)
+ val thriftTableInfo: TableInfo = metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
val schemaConverter = new ThriftWrapperSchemaConverterImpl()
val wrapperTableInfo = schemaConverter
.fromExternalToWrapperTableInfo(thriftTableInfo,
@@ -104,8 +103,8 @@ private[sql] case class AlterTableAddColumns(
LOGGER.info(s"Alter table for add columns is successful for table $dbName.$tableName")
LOGGER.audit(s"Alter table for add columns is successful for table $dbName.$tableName")
} catch {
- case e: Exception => LOGGER
- .error("Alter table add columns failed :" + e.getMessage)
+ case e: Exception =>
+ LOGGER.error(e, "Alter table add columns failed")
if (newCols.nonEmpty) {
LOGGER.info("Cleaning up the dictionary files as alter table add operation failed")
new AlterTableDropColumnRDD(sparkSession.sparkContext,
@@ -147,9 +146,9 @@ private[sql] case class AlterTableRenameTable(alterTableRenameModel: AlterTableR
val newTableName = newTableIdentifier.table.toLowerCase
LOGGER.audit(s"Rename table request has been received for $oldDatabaseName.$oldTableName")
LOGGER.info(s"Rename table request has been received for $oldDatabaseName.$oldTableName")
+ val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
val relation: CarbonRelation =
- CarbonEnv.getInstance(sparkSession).carbonMetastore
- .lookupRelation(oldTableIdentifier.database, oldTableName)(sparkSession)
+ metastore.lookupRelation(oldTableIdentifier.database, oldTableName)(sparkSession)
.asInstanceOf[CarbonRelation]
if (relation == null) {
LOGGER.audit(s"Rename table request has failed. " +
@@ -168,15 +167,14 @@ private[sql] case class AlterTableRenameTable(alterTableRenameModel: AlterTableR
locks = AlterTableUtil
.validateTableAndAcquireLock(oldDatabaseName, oldTableName, locksToBeAcquired)(
sparkSession)
- carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .lookupRelation(Some(oldDatabaseName), oldTableName)(sparkSession)
+ carbonTable = metastore.lookupRelation(Some(oldDatabaseName), oldTableName)(sparkSession)
.asInstanceOf[CarbonRelation].tableMeta.carbonTable
// get the latest carbon table and check for column existence
val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getStorePath,
carbonTable.getCarbonTableIdentifier)
- val tableMetadataFile = carbonTablePath.getSchemaFilePath
- val tableInfo: org.apache.carbondata.format.TableInfo = CarbonEnv.getInstance(sparkSession)
- .carbonMetastore.readSchemaFile(tableMetadataFile)
+ val tableMetadataFile = carbonTablePath.getPath
+ val tableInfo: org.apache.carbondata.format.TableInfo =
+ metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
val schemaEvolutionEntry = new SchemaEvolutionEntry(System.currentTimeMillis)
schemaEvolutionEntry.setTableName(newTableName)
timeStamp = System.currentTimeMillis()
@@ -193,15 +191,13 @@ private[sql] case class AlterTableRenameTable(alterTableRenameModel: AlterTableR
}
}
val newTableIdentifier = new CarbonTableIdentifier(oldDatabaseName,
- newTableName,
- carbonTable.getCarbonTableIdentifier.getTableId)
- val newTablePath = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .updateTableSchema(newTableIdentifier,
+ newTableName, carbonTable.getCarbonTableIdentifier.getTableId)
+ val newTablePath = metastore.updateTableSchema(newTableIdentifier,
+ carbonTable.getCarbonTableIdentifier,
tableInfo,
schemaEvolutionEntry,
carbonTable.getStorePath)(sparkSession)
- CarbonEnv.getInstance(sparkSession).carbonMetastore
- .removeTableFromMetadata(oldDatabaseName, oldTableName)
+ metastore.removeTableFromMetadata(oldDatabaseName, oldTableName)
sparkSession.sessionState.asInstanceOf[CarbonSessionState].metadataHive
.runSqlHive(
s"ALTER TABLE $oldDatabaseName.$oldTableName RENAME TO $oldDatabaseName.$newTableName")
@@ -213,8 +209,8 @@ private[sql] case class AlterTableRenameTable(alterTableRenameModel: AlterTableR
LOGGER.audit(s"Table $oldTableName has been successfully renamed to $newTableName")
LOGGER.info(s"Table $oldTableName has been successfully renamed to $newTableName")
} catch {
- case e: Exception => LOGGER
- .error("Rename table failed: " + e.getMessage)
+ case e: Exception =>
+ LOGGER.error(e, "Rename table failed: " + e.getMessage)
if (carbonTable != null) {
AlterTableUtil
.revertRenameTableChanges(oldTableIdentifier,
@@ -279,7 +275,8 @@ private[sql] case class AlterTableDropColumns(
try {
locks = AlterTableUtil
.validateTableAndAcquireLock(dbName, tableName, locksToBeAcquired)(sparkSession)
- carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ carbonTable = metastore
.lookupRelation(Some(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation]
.tableMeta.carbonTable
val partitionInfo = carbonTable.getPartitionInfo(tableName)
@@ -329,9 +326,8 @@ private[sql] case class AlterTableDropColumns(
// read the latest schema file
val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getStorePath,
carbonTable.getCarbonTableIdentifier)
- val tableMetadataFile = carbonTablePath.getSchemaFilePath
- val tableInfo: org.apache.carbondata.format.TableInfo = CarbonEnv.getInstance(sparkSession)
- .carbonMetastore.readSchemaFile(tableMetadataFile)
+ val tableInfo: org.apache.carbondata.format.TableInfo =
+ metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
// maintain the deleted columns for schema evolution history
var deletedColumnSchema = ListBuffer[org.apache.carbondata.format.ColumnSchema]()
val columnSchemaList = tableInfo.fact_table.table_columns.asScala
@@ -393,7 +389,8 @@ private[sql] case class AlterTableDataTypeChange(
try {
locks = AlterTableUtil
.validateTableAndAcquireLock(dbName, tableName, locksToBeAcquired)(sparkSession)
- carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
+ carbonTable = metastore
.lookupRelation(Some(dbName), tableName)(sparkSession).asInstanceOf[CarbonRelation]
.tableMeta.carbonTable
val columnName = alterTableDataTypeChangeModel.columnName
@@ -415,9 +412,7 @@ private[sql] case class AlterTableDataTypeChange(
// read the latest schema file
val carbonTablePath = CarbonStorePath.getCarbonTablePath(carbonTable.getStorePath,
carbonTable.getCarbonTableIdentifier)
- val tableMetadataFile = carbonTablePath.getSchemaFilePath
- val tableInfo: TableInfo = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .readSchemaFile(tableMetadataFile)
+ val tableInfo: TableInfo = metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
// maintain the added column for schema evolution history
var addColumnSchema: ColumnSchema = null
var deletedColumnSchema: ColumnSchema = null
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonHiveCommands.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonHiveCommands.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonHiveCommands.scala
index d2022be..609f39b 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonHiveCommands.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/CarbonHiveCommands.scala
@@ -18,9 +18,10 @@
package org.apache.spark.sql.hive.execution.command
import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}
+import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.execution.command.{CarbonDropTableCommand, DropDatabaseCommand, ResetCommand, RunnableCommand, SetCommand}
-import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
case class CarbonDropDatabaseCommand(command: DropDatabaseCommand)
extends RunnableCommand {
@@ -29,16 +30,19 @@ case class CarbonDropDatabaseCommand(command: DropDatabaseCommand)
override def run(sparkSession: SparkSession): Seq[Row] = {
val dbName = command.databaseName
+ var tablesInDB: Seq[TableIdentifier] = null
+ if (sparkSession.sessionState.catalog.listDatabases().exists(_.equalsIgnoreCase(dbName))) {
+ tablesInDB = sparkSession.sessionState.catalog.listTables(dbName)
+ }
// DropHiveDB command will fail if cascade is false and one or more table exists in database
val rows = command.run(sparkSession)
- if (command.cascade) {
- val tablesInDB = CarbonEnv.getInstance(sparkSession).carbonMetastore.getAllTables()
- .filter(_.database.exists(_.equalsIgnoreCase(dbName)))
+ if (command.cascade && tablesInDB != null) {
tablesInDB.foreach { tableName =>
CarbonDropTableCommand(true, tableName.database, tableName.table).run(sparkSession)
}
}
- CarbonEnv.getInstance(sparkSession).carbonMetastore.dropDatabaseDirectory(dbName.toLowerCase)
+ CarbonUtil.dropDatabaseDirectory(dbName.toLowerCase, CarbonEnv.getInstance(sparkSession)
+ .carbonMetastore.storePath)
rows
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/DDLStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/DDLStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/DDLStrategy.scala
index 6087736..760cb06 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/DDLStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/DDLStrategy.scala
@@ -16,13 +16,14 @@
*/
package org.apache.spark.sql.execution.command
-import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, CarbonEnv, InsertIntoCarbonTable, ShowLoadsCommand, SparkSession}
+import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.{SparkPlan, SparkStrategy}
import org.apache.spark.sql.hive.execution.command.{CarbonDropDatabaseCommand, CarbonResetCommand, CarbonSetCommand}
+import org.apache.carbondata.core.util.CarbonUtil
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
/**
@@ -61,7 +62,8 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
_, child: LogicalPlan, _, _) =>
ExecutedCommandExec(LoadTableByInsert(relation, child)) :: Nil
case createDb@CreateDatabaseCommand(dbName, ifNotExists, _, _, _) =>
- CarbonEnv.getInstance(sparkSession).carbonMetastore.createDatabaseDirectory(dbName)
+ CarbonUtil.createDatabaseDirectory(dbName, CarbonEnv.getInstance(sparkSession).
+ carbonMetastore.storePath)
ExecutedCommandExec(createDb) :: Nil
case drop@DropDatabaseCommand(dbName, ifExists, isCascade) =>
ExecutedCommandExec(CarbonDropDatabaseCommand(drop)) :: Nil
@@ -127,6 +129,14 @@ class DDLStrategy(sparkSession: SparkSession) extends SparkStrategy {
ExecutedCommandExec(CarbonSetCommand(set)) :: Nil
case reset@ResetCommand =>
ExecutedCommandExec(CarbonResetCommand()) :: Nil
+ case org.apache.spark.sql.execution.datasources.CreateTable(tableDesc, mode, None)
+ if tableDesc.provider.get != DDLUtils.HIVE_PROVIDER
+ && tableDesc.provider.get.equals("org.apache.spark.sql.CarbonSource") =>
+ val updatedCatalog =
+ CarbonSource.updateCatalogTableWithCarbonSchema(tableDesc, sparkSession)
+ val cmd =
+ CreateDataSourceTableCommand(updatedCatalog, ignoreIfExists = mode == SaveMode.Ignore)
+ ExecutedCommandExec(cmd) :: Nil
case _ => Nil
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
index 2fccd0c..2c1de52 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
@@ -42,6 +42,7 @@ import org.apache.carbondata.core.statusmanager.{SegmentStatusManager, SegmentUp
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.core.writer.CarbonDeleteDeltaWriterImpl
+import org.apache.carbondata.hadoop.CarbonInputFormat
import org.apache.carbondata.processing.exception.MultipleMatchingException
import org.apache.carbondata.processing.merger.{CarbonDataMergerUtil, CarbonDataMergerUtilResult, CompactionType}
import org.apache.carbondata.spark.DeleteDelataResultImpl
@@ -107,7 +108,7 @@ private[sql] case class ProjectForDeleteCommand(
CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, e.compactionTimeStamp.toString)
case e: Exception =>
- LOG.error("Exception in Delete data operation " + e.getMessage)
+ LOG.error(e, "Exception in Delete data operation " + e.getMessage)
// ****** start clean up.
// In case of failure , clean all related delete delta files
CarbonUpdateUtil.cleanStaleDeltaFiles(carbonTable, timestamp)
@@ -548,7 +549,7 @@ object deleteExecution {
val (carbonInputFormat, job) =
QueryPlanUtil.createCarbonInputFormat(absoluteTableIdentifier)
-
+ CarbonInputFormat.setTableInfo(job.getConfiguration, carbonTable.getTableInfo)
val keyRdd = deleteRdd.map({ row =>
val tupleId: String = row
.getString(row.fieldIndex(CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index ce66733..88e89ad 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -29,14 +29,13 @@ import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.SparkPlan
-import org.apache.spark.sql.hive.{CarbonMetastore, CarbonRelation}
+import org.apache.spark.sql.hive.CarbonRelation
import org.apache.spark.util.FileUtils
import org.codehaus.jackson.map.ObjectMapper
import org.apache.carbondata.api.CarbonStore
import org.apache.carbondata.common.constants.LoggerAction
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.cache.dictionary.ManageDictionaryAndBTree
import org.apache.carbondata.core.constants.{CarbonCommonConstants, CarbonLoadOptionConstants}
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.dictionary.server.DictionaryServer
@@ -48,6 +47,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.mutate.{CarbonUpdateUtil, TupleIdEnum}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.apache.carbondata.core.util.path.CarbonStorePath
+import org.apache.carbondata.format
import org.apache.carbondata.processing.constants.TableOptionConstant
import org.apache.carbondata.processing.etl.DataLoadingException
import org.apache.carbondata.processing.model.{CarbonDataLoadSchema, CarbonLoadModel}
@@ -186,23 +186,24 @@ case class CreateTable(cm: TableModel, createDSTable: Boolean = true) extends Ru
} else {
// Add Database to catalog and persist
val catalog = CarbonEnv.getInstance(sparkSession).carbonMetastore
- // Need to fill partitioner class when we support partition
- val tablePath = catalog.createTableFromThrift(tableInfo, dbName, tbName)(sparkSession)
+ val (tablePath, carbonSchemaString) =
+ catalog.createTableFromThrift(tableInfo, dbName, tbName)(sparkSession)
if (createDSTable) {
try {
val fields = new Array[Field](cm.dimCols.size + cm.msrCols.size)
cm.dimCols.foreach(f => fields(f.schemaOrdinal) = f)
cm.msrCols.foreach(f => fields(f.schemaOrdinal) = f)
+
sparkSession.sql(
s"""CREATE TABLE $dbName.$tbName
|(${ fields.map(f => f.rawSchema).mkString(",") })
|USING org.apache.spark.sql.CarbonSource""".stripMargin +
- s""" OPTIONS (tableName "$tbName", dbName "$dbName", tablePath "$tablePath") """)
+ s""" OPTIONS (tableName "$tbName", dbName "$dbName", tablePath """.stripMargin +
+ s""""$tablePath" $carbonSchemaString) """)
} catch {
case e: Exception =>
val identifier: TableIdentifier = TableIdentifier(tbName, Some(dbName))
// call the drop table to delete the created table.
-
CarbonEnv.getInstance(sparkSession).carbonMetastore
.dropTable(catalog.storePath, identifier)(sparkSession)
@@ -234,9 +235,9 @@ case class DeleteLoadsById(
def run(sparkSession: SparkSession): Seq[Row] = {
Checker.validateTableExists(databaseNameOp, tableName, sparkSession)
- val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .getTableFromMetadata(getDB.getDatabaseName(databaseNameOp, sparkSession), tableName)
- .map(_.carbonTable).getOrElse(null)
+ val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore.
+ lookupRelation(databaseNameOp, tableName)(sparkSession).asInstanceOf[CarbonRelation].
+ tableMeta.carbonTable
CarbonStore.deleteLoadById(
loadids,
getDB.getDatabaseName(databaseNameOp, sparkSession),
@@ -259,9 +260,9 @@ case class DeleteLoadsByLoadDate(
def run(sparkSession: SparkSession): Seq[Row] = {
Checker.validateTableExists(databaseNameOp, tableName, sparkSession)
- val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .getTableFromMetadata(getDB.getDatabaseName(databaseNameOp, sparkSession), tableName)
- .map(_.carbonTable).getOrElse(null)
+ val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore.
+ lookupRelation(databaseNameOp, tableName)(sparkSession).asInstanceOf[CarbonRelation].
+ tableMeta.carbonTable
CarbonStore.deleteLoadByDate(
loadDate,
getDB.getDatabaseName(databaseNameOp, sparkSession),
@@ -279,13 +280,13 @@ object LoadTable {
sqlContext: SQLContext,
model: DictionaryLoadModel,
noDictDimension: Array[CarbonDimension]): Unit = {
-
+ val sparkSession = sqlContext.sparkSession
val carbonTablePath = CarbonStorePath.getCarbonTablePath(model.hdfsLocation,
model.table)
- val schemaFilePath = carbonTablePath.getSchemaFilePath
+ val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
// read TableInfo
- val tableInfo = CarbonMetastore.readSchemaFileToThriftTable(schemaFilePath)
+ val tableInfo: format.TableInfo = metastore.getThriftTableInfo(carbonTablePath)(sparkSession)
// modify TableInfo
val columns = tableInfo.getFact_table.getTable_columns
@@ -294,23 +295,21 @@ object LoadTable {
columns.get(i).encoders.remove(org.apache.carbondata.format.Encoding.DICTIONARY)
}
}
+ val entry = tableInfo.getFact_table.getSchema_evolution.getSchema_evolution_history.get(0)
+ entry.setTime_stamp(System.currentTimeMillis())
// write TableInfo
- CarbonMetastore.writeThriftTableToSchemaFile(schemaFilePath, tableInfo)
-
- val catalog = CarbonEnv.getInstance(sqlContext.sparkSession).carbonMetastore
+ metastore.updateTableSchema(carbonTablePath.getCarbonTableIdentifier,
+ carbonTablePath.getCarbonTableIdentifier,
+ tableInfo, entry, carbonTablePath.getPath)(sparkSession)
// update the schema modified time
- catalog.updateSchemasUpdatedTime(catalog.touchSchemaFileSystemTime(
+ metastore.updateAndTouchSchemasUpdatedTime(
carbonLoadModel.getDatabaseName,
- carbonLoadModel.getTableName))
-
- // update Metadata
- catalog.updateMetadataByThriftTable(schemaFilePath, tableInfo,
- model.table.getDatabaseName, model.table.getTableName, carbonLoadModel.getStorePath)
+ carbonLoadModel.getTableName)
// update CarbonDataLoadSchema
- val carbonTable = catalog.lookupRelation(Option(model.table.getDatabaseName),
+ val carbonTable = metastore.lookupRelation(Option(model.table.getDatabaseName),
model.table.getTableName)(sqlContext.sparkSession).asInstanceOf[CarbonRelation].tableMeta
.carbonTable
carbonLoadModel.setCarbonDataLoadSchema(new CarbonDataLoadSchema(carbonTable))
@@ -628,6 +627,14 @@ case class LoadTable(
LOGGER.audit(s"Cannot use single_pass=true for $dbName.$tableName during the first load")
carbonLoadModel.setUseOnePass(false)
}
+ // Create table and metadata folders if not exist
+ val carbonTablePath = CarbonStorePath
+ .getCarbonTablePath(storePath, table.getCarbonTableIdentifier)
+ val metadataDirectoryPath = carbonTablePath.getMetadataDirectoryPath
+ val fileType = FileFactory.getFileType(metadataDirectoryPath)
+ if (!FileFactory.isFileExist(metadataDirectoryPath, fileType)) {
+ FileFactory.mkdirs(metadataDirectoryPath, fileType)
+ }
if (carbonLoadModel.getUseOnePass) {
val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
val carbonTableIdentifier = carbonTable.getAbsoluteTableIdentifier
@@ -691,8 +698,7 @@ case class LoadTable(
server,
dataFrame,
updateModel)
- }
- else {
+ } else {
val (dictionaryDataFrame, loadDataFrame) = if (updateModel.isDefined) {
val fields = dataFrame.get.schema.fields
import org.apache.spark.sql.functions.udf
@@ -726,6 +732,7 @@ case class LoadTable(
} else {
(dataFrame, dataFrame)
}
+
GlobalDictionaryUtil
.generateGlobalDictionary(
sparkSession.sqlContext,
@@ -794,9 +801,9 @@ private[sql] case class DeleteLoadByDate(
def run(sparkSession: SparkSession): Seq[Row] = {
Checker.validateTableExists(databaseNameOp, tableName, sparkSession)
- val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .getTableFromMetadata(getDB.getDatabaseName(databaseNameOp, sparkSession), tableName)
- .map(_.carbonTable).getOrElse(null)
+ val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore.
+ lookupRelation(databaseNameOp, tableName)(sparkSession).asInstanceOf[CarbonRelation].
+ tableMeta.carbonTable
CarbonStore.deleteLoadByDate(
loadDate,
getDB.getDatabaseName(databaseNameOp, sparkSession),
@@ -818,9 +825,7 @@ case class CleanFiles(
val catalog = CarbonEnv.getInstance(sparkSession).carbonMetastore
val relation = catalog
.lookupRelation(databaseNameOp, tableName)(sparkSession).asInstanceOf[CarbonRelation]
- val carbonTable = catalog
- .getTableFromMetadata(getDB.getDatabaseName(databaseNameOp, sparkSession), tableName)
- .map(_.carbonTable).getOrElse(null)
+ val carbonTable = relation.tableMeta.carbonTable
CarbonStore.cleanFiles(
getDB.getDatabaseName(databaseNameOp, sparkSession),
tableName,
@@ -839,9 +844,9 @@ case class ShowLoads(
def run(sparkSession: SparkSession): Seq[Row] = {
Checker.validateTableExists(databaseNameOp, tableName, sparkSession)
- val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore
- .getTableFromMetadata(getDB.getDatabaseName(databaseNameOp, sparkSession), tableName)
- .map(_.carbonTable).getOrElse(null)
+ val carbonTable = CarbonEnv.getInstance(sparkSession).carbonMetastore.
+ lookupRelation(databaseNameOp, tableName)(sparkSession).asInstanceOf[CarbonRelation].
+ tableMeta.carbonTable
CarbonStore.showSegments(
getDB.getDatabaseName(databaseNameOp, sparkSession),
tableName,
@@ -867,15 +872,11 @@ case class CarbonDropTableCommand(ifExistsSet: Boolean,
catalog.checkSchemasModifiedTimeAndReloadTables()
val carbonLocks: scala.collection.mutable.ListBuffer[ICarbonLock] = ListBuffer()
try {
- val carbonTable = catalog.getTableFromMetadata(dbName, tableName).map(_.carbonTable).orNull
- locksToBeAcquired foreach {
- lock => carbonLocks += CarbonLockUtil.getLockObject(carbonTable, lock)
+ locksToBeAcquired foreach {
+ lock => carbonLocks += CarbonLockUtil.getLockObject(carbonTableIdentifier, lock)
}
LOGGER.audit(s"Deleting table [$tableName] under database [$dbName]")
- if (null != carbonTable) {
- // clear driver B-tree and dictionary cache
- ManageDictionaryAndBTree.clearBTreeAndDictionaryLRUCache(carbonTable)
- }
+
CarbonEnv.getInstance(sparkSession).carbonMetastore
.dropTable(storePath, identifier)(sparkSession)
LOGGER.audit(s"Deleted table [$tableName] under database [$dbName]")
[07/15] carbondata git commit: [CARBONDATA-1284]Implement hive based
schema storage in carbon
Posted by ra...@apache.org.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
new file mode 100644
index 0000000..048681c
--- /dev/null
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
@@ -0,0 +1,531 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive
+
+import java.util.UUID
+import java.util.concurrent.atomic.AtomicLong
+
+import scala.collection.mutable.ArrayBuffer
+
+import org.apache.spark.sql.{RuntimeConfig, SparkSession}
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+
+import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.cache.dictionary.ManageDictionaryAndBTree
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.datastore.impl.FileFactory.FileType
+import org.apache.carbondata.core.fileoperations.FileWriteOperation
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata, CarbonTableIdentifier}
+import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.stats.{QueryStatistic, QueryStatisticsConstants}
+import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, CarbonUtil}
+import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.writer.ThriftWriter
+import org.apache.carbondata.format.{SchemaEvolutionEntry, TableInfo}
+import org.apache.carbondata.processing.merger.TableMeta
+import org.apache.carbondata.spark.util.CarbonSparkUtil
+
+case class MetaData(var tablesMeta: ArrayBuffer[TableMeta]) {
+ // clear the metadata
+ def clear(): Unit = {
+ tablesMeta.clear()
+ }
+}
+
+case class CarbonMetaData(dims: Seq[String],
+ msrs: Seq[String],
+ carbonTable: CarbonTable,
+ dictionaryMap: DictionaryMap)
+
+case class DictionaryMap(dictionaryMap: Map[String, Boolean]) {
+ def get(name: String): Option[Boolean] = {
+ dictionaryMap.get(name.toLowerCase)
+ }
+}
+
+class CarbonFileMetastore(conf: RuntimeConfig, val storePath: String) extends CarbonMetaStore {
+
+ @transient
+ val LOGGER = LogServiceFactory.getLogService("org.apache.spark.sql.CarbonMetastoreCatalog")
+
+ val tableModifiedTimeStore = new java.util.HashMap[String, Long]()
+ tableModifiedTimeStore
+ .put(CarbonCommonConstants.DATABASE_DEFAULT_NAME, System.currentTimeMillis())
+
+ private val nextId = new AtomicLong(0)
+
+ def nextQueryId: String = {
+ System.nanoTime() + ""
+ }
+
+ lazy val metadata = loadMetadata(storePath, nextQueryId)
+
+
+ /**
+ * Create spark session from paramters.
+ *
+ * @param parameters
+ * @param absIdentifier
+ * @param sparkSession
+ */
+ override def createCarbonRelation(parameters: Map[String, String],
+ absIdentifier: AbsoluteTableIdentifier,
+ sparkSession: SparkSession): CarbonRelation = {
+ lookupRelation(TableIdentifier(absIdentifier.getCarbonTableIdentifier.getTableName,
+ Some(absIdentifier.getCarbonTableIdentifier.getDatabaseName)))(sparkSession)
+ .asInstanceOf[CarbonRelation]
+ }
+
+ def lookupRelation(dbName: Option[String], tableName: String)
+ (sparkSession: SparkSession): LogicalPlan = {
+ lookupRelation(TableIdentifier(tableName, dbName))(sparkSession)
+ }
+
+ def lookupRelation(tableIdentifier: TableIdentifier)
+ (sparkSession: SparkSession): LogicalPlan = {
+ checkSchemasModifiedTimeAndReloadTables()
+ val database = tableIdentifier.database.getOrElse(
+ sparkSession.catalog.currentDatabase
+ )
+ val tables = getTableFromMetadata(database, tableIdentifier.table, true)
+ tables match {
+ case Some(t) =>
+ CarbonRelation(database, tableIdentifier.table,
+ CarbonSparkUtil.createSparkMeta(tables.head.carbonTable), tables.head)
+ case None =>
+ throw new NoSuchTableException(database, tableIdentifier.table)
+ }
+ }
+
+ /**
+ * This method will search for a table in the catalog metadata
+ *
+ * @param database
+ * @param tableName
+ * @return
+ */
+ def getTableFromMetadata(database: String,
+ tableName: String, readStore: Boolean = false): Option[TableMeta] = {
+ metadata.tablesMeta
+ .find(c => c.carbonTableIdentifier.getDatabaseName.equalsIgnoreCase(database) &&
+ c.carbonTableIdentifier.getTableName.equalsIgnoreCase(tableName))
+ }
+
+ def tableExists(
+ table: String,
+ databaseOp: Option[String] = None)(sparkSession: SparkSession): Boolean = {
+ tableExists(TableIdentifier(table, databaseOp))(sparkSession)
+ }
+
+ def tableExists(tableIdentifier: TableIdentifier)(sparkSession: SparkSession): Boolean = {
+ checkSchemasModifiedTimeAndReloadTables()
+ val database = tableIdentifier.database.getOrElse(sparkSession.catalog.currentDatabase)
+ val tables = metadata.tablesMeta.filter(
+ c => c.carbonTableIdentifier.getDatabaseName.equalsIgnoreCase(database) &&
+ c.carbonTableIdentifier.getTableName.equalsIgnoreCase(tableIdentifier.table))
+ tables.nonEmpty
+ }
+
+ def loadMetadata(metadataPath: String, queryId: String): MetaData = {
+ val recorder = CarbonTimeStatisticsFactory.createDriverRecorder()
+ val statistic = new QueryStatistic()
+ // creating zookeeper instance once.
+ // if zookeeper is configured as carbon lock type.
+ val zookeeperurl = conf.get(CarbonCommonConstants.ZOOKEEPER_URL, null)
+ if (null != zookeeperurl) {
+ CarbonProperties.getInstance
+ .addProperty(CarbonCommonConstants.ZOOKEEPER_URL, zookeeperurl)
+ }
+ if (metadataPath == null) {
+ return null
+ }
+ // if no locktype is configured and store type is HDFS set HDFS lock as default
+ if (null == CarbonProperties.getInstance
+ .getProperty(CarbonCommonConstants.LOCK_TYPE) &&
+ FileType.HDFS == FileFactory.getFileType(metadataPath)) {
+ CarbonProperties.getInstance
+ .addProperty(CarbonCommonConstants.LOCK_TYPE,
+ CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS
+ )
+ LOGGER.info("Default lock type HDFSLOCK is configured")
+ }
+ val fileType = FileFactory.getFileType(metadataPath)
+ val metaDataBuffer = new ArrayBuffer[TableMeta]
+ fillMetaData(metadataPath, fileType, metaDataBuffer)
+ updateSchemasUpdatedTime(readSchemaFileSystemTime("", ""))
+ statistic.addStatistics(QueryStatisticsConstants.LOAD_META,
+ System.currentTimeMillis())
+ recorder.recordStatisticsForDriver(statistic, queryId)
+ MetaData(metaDataBuffer)
+ }
+
+ private def fillMetaData(basePath: String, fileType: FileType,
+ metaDataBuffer: ArrayBuffer[TableMeta]): Unit = {
+ val databasePath = basePath // + "/schemas"
+ try {
+ if (FileFactory.isFileExist(databasePath, fileType)) {
+ val file = FileFactory.getCarbonFile(databasePath, fileType)
+ val databaseFolders = file.listFiles()
+
+ databaseFolders.foreach(databaseFolder => {
+ if (databaseFolder.isDirectory) {
+ val dbName = databaseFolder.getName
+ val tableFolders = databaseFolder.listFiles()
+
+ tableFolders.foreach(tableFolder => {
+ if (tableFolder.isDirectory) {
+ val carbonTableIdentifier = new CarbonTableIdentifier(databaseFolder.getName,
+ tableFolder.getName, UUID.randomUUID().toString)
+ val carbonTablePath = CarbonStorePath.getCarbonTablePath(basePath,
+ carbonTableIdentifier)
+ val tableMetadataFile = carbonTablePath.getSchemaFilePath
+
+ if (FileFactory.isFileExist(tableMetadataFile, fileType)) {
+ val tableName = tableFolder.getName
+ val tableUniqueName = databaseFolder.getName + "_" + tableFolder.getName
+ val tableInfo: TableInfo = CarbonUtil.readSchemaFile(tableMetadataFile)
+ val schemaConverter = new ThriftWrapperSchemaConverterImpl
+ val wrapperTableInfo = schemaConverter
+ .fromExternalToWrapperTableInfo(tableInfo, dbName, tableName, basePath)
+ val schemaFilePath = CarbonStorePath
+ .getCarbonTablePath(storePath, carbonTableIdentifier).getSchemaFilePath
+ wrapperTableInfo.setStorePath(storePath)
+ wrapperTableInfo
+ .setMetaDataFilepath(CarbonTablePath.getFolderContainingFile(schemaFilePath))
+ CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo)
+ val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
+ metaDataBuffer += new TableMeta(carbonTable.getCarbonTableIdentifier, storePath,
+ carbonTable)
+ }
+ }
+ })
+ }
+ })
+ } else {
+ // Create folders and files.
+ FileFactory.mkdirs(databasePath, fileType)
+ }
+ } catch {
+ case s: java.io.FileNotFoundException =>
+ s.printStackTrace()
+ // Create folders and files.
+ FileFactory.mkdirs(databasePath, fileType)
+ }
+ }
+
+ /**
+ * This method will overwrite the existing schema and update it with the given details
+ *
+ * @param newTableIdentifier
+ * @param thriftTableInfo
+ * @param schemaEvolutionEntry
+ * @param carbonStorePath
+ * @param sparkSession
+ */
+ def updateTableSchema(newTableIdentifier: CarbonTableIdentifier,
+ oldTableIdentifier: CarbonTableIdentifier,
+ thriftTableInfo: org.apache.carbondata.format.TableInfo,
+ schemaEvolutionEntry: SchemaEvolutionEntry,
+ carbonStorePath: String)
+ (sparkSession: SparkSession): String = {
+ val schemaConverter = new ThriftWrapperSchemaConverterImpl
+ if (schemaEvolutionEntry != null) {
+ thriftTableInfo.fact_table.schema_evolution.schema_evolution_history.add(schemaEvolutionEntry)
+ }
+ val wrapperTableInfo = schemaConverter
+ .fromExternalToWrapperTableInfo(thriftTableInfo,
+ newTableIdentifier.getDatabaseName,
+ newTableIdentifier.getTableName,
+ carbonStorePath)
+ createSchemaThriftFile(wrapperTableInfo,
+ thriftTableInfo,
+ newTableIdentifier.getDatabaseName,
+ newTableIdentifier.getTableName)(sparkSession)
+ }
+
+ /**
+ * This method will is used to remove the evolution entry in case of failure.
+ *
+ * @param carbonTableIdentifier
+ * @param thriftTableInfo
+ * @param carbonStorePath
+ * @param sparkSession
+ */
+ def revertTableSchema(carbonTableIdentifier: CarbonTableIdentifier,
+ thriftTableInfo: org.apache.carbondata.format.TableInfo,
+ carbonStorePath: String)
+ (sparkSession: SparkSession): String = {
+ val schemaConverter = new ThriftWrapperSchemaConverterImpl
+ val wrapperTableInfo = schemaConverter
+ .fromExternalToWrapperTableInfo(thriftTableInfo,
+ carbonTableIdentifier.getDatabaseName,
+ carbonTableIdentifier.getTableName,
+ carbonStorePath)
+ val evolutionEntries = thriftTableInfo.fact_table.schema_evolution.schema_evolution_history
+ evolutionEntries.remove(evolutionEntries.size() - 1)
+ createSchemaThriftFile(wrapperTableInfo,
+ thriftTableInfo,
+ carbonTableIdentifier.getDatabaseName,
+ carbonTableIdentifier.getTableName)(sparkSession)
+ }
+
+
+
+ /**
+ *
+ * Prepare Thrift Schema from wrapper TableInfo and write to Schema file.
+ * Load CarbonTable from wrapper tableInfo
+ *
+ */
+ def createTableFromThrift(
+ tableInfo: org.apache.carbondata.core.metadata.schema.table.TableInfo,
+ dbName: String, tableName: String)(sparkSession: SparkSession): (String, String) = {
+ if (tableExists(tableName, Some(dbName))(sparkSession)) {
+ sys.error(s"Table [$tableName] already exists under Database [$dbName]")
+ }
+ val schemaEvolutionEntry = new SchemaEvolutionEntry(tableInfo.getLastUpdatedTime)
+ val schemaConverter = new ThriftWrapperSchemaConverterImpl
+ val thriftTableInfo = schemaConverter
+ .fromWrapperToExternalTableInfo(tableInfo, dbName, tableName)
+ thriftTableInfo.getFact_table.getSchema_evolution.getSchema_evolution_history
+ .add(schemaEvolutionEntry)
+ val carbonTablePath = createSchemaThriftFile(tableInfo,
+ thriftTableInfo,
+ dbName,
+ tableName)(sparkSession)
+ LOGGER.info(s"Table $tableName for Database $dbName created successfully.")
+ (carbonTablePath, "")
+ }
+
+ /**
+ * This method will write the schema thrift file in carbon store and load table metadata
+ *
+ * @param tableInfo
+ * @param thriftTableInfo
+ * @param dbName
+ * @param tableName
+ * @param sparkSession
+ * @return
+ */
+ private def createSchemaThriftFile(
+ tableInfo: org.apache.carbondata.core.metadata.schema.table.TableInfo,
+ thriftTableInfo: org.apache.carbondata.format.TableInfo,
+ dbName: String, tableName: String)
+ (sparkSession: SparkSession): String = {
+ val carbonTableIdentifier = new CarbonTableIdentifier(dbName, tableName,
+ tableInfo.getFactTable.getTableId)
+ val carbonTablePath = CarbonStorePath.getCarbonTablePath(storePath, carbonTableIdentifier)
+ val schemaFilePath = carbonTablePath.getSchemaFilePath
+ val schemaMetadataPath = CarbonTablePath.getFolderContainingFile(schemaFilePath)
+ tableInfo.setMetaDataFilepath(schemaMetadataPath)
+ tableInfo.setStorePath(storePath)
+ val fileType = FileFactory.getFileType(schemaMetadataPath)
+ if (!FileFactory.isFileExist(schemaMetadataPath, fileType)) {
+ FileFactory.mkdirs(schemaMetadataPath, fileType)
+ }
+ val thriftWriter = new ThriftWriter(schemaFilePath, false)
+ thriftWriter.open(FileWriteOperation.OVERWRITE)
+ thriftWriter.write(thriftTableInfo)
+ thriftWriter.close()
+ removeTableFromMetadata(dbName, tableName)
+ CarbonMetadata.getInstance().loadTableMetadata(tableInfo)
+ val tableMeta = new TableMeta(carbonTableIdentifier, storePath,
+ CarbonMetadata.getInstance().getCarbonTable(dbName + '_' + tableName))
+ metadata.tablesMeta += tableMeta
+ updateSchemasUpdatedTime(touchSchemaFileSystemTime(dbName, tableName))
+ carbonTablePath.getPath
+ }
+
+ /**
+ * This method will remove the table meta from catalog metadata array
+ *
+ * @param dbName
+ * @param tableName
+ */
+ def removeTableFromMetadata(dbName: String, tableName: String): Unit = {
+ val metadataToBeRemoved: Option[TableMeta] = getTableFromMetadata(dbName, tableName)
+ metadataToBeRemoved match {
+ case Some(tableMeta) =>
+ metadata.tablesMeta -= tableMeta
+ CarbonMetadata.getInstance.removeTable(dbName + "_" + tableName)
+ case None =>
+ LOGGER.debug(s"No entry for table $tableName in database $dbName")
+ }
+ }
+
+ private def updateMetadataByWrapperTable(
+ wrapperTableInfo: org.apache.carbondata.core.metadata.schema.table.TableInfo): Unit = {
+
+ CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo)
+ val carbonTable = CarbonMetadata.getInstance().getCarbonTable(
+ wrapperTableInfo.getTableUniqueName)
+ for (i <- metadata.tablesMeta.indices) {
+ if (wrapperTableInfo.getTableUniqueName.equals(
+ metadata.tablesMeta(i).carbonTableIdentifier.getTableUniqueName)) {
+ metadata.tablesMeta(i).carbonTable = carbonTable
+ }
+ }
+ }
+
+ def updateMetadataByThriftTable(schemaFilePath: String,
+ tableInfo: TableInfo, dbName: String, tableName: String, storePath: String): Unit = {
+
+ tableInfo.getFact_table.getSchema_evolution.getSchema_evolution_history.get(0)
+ .setTime_stamp(System.currentTimeMillis())
+ val schemaConverter = new ThriftWrapperSchemaConverterImpl
+ val wrapperTableInfo = schemaConverter
+ .fromExternalToWrapperTableInfo(tableInfo, dbName, tableName, storePath)
+ wrapperTableInfo
+ .setMetaDataFilepath(CarbonTablePath.getFolderContainingFile(schemaFilePath))
+ wrapperTableInfo.setStorePath(storePath)
+ updateMetadataByWrapperTable(wrapperTableInfo)
+ }
+
+
+ def isTablePathExists(tableIdentifier: TableIdentifier)(sparkSession: SparkSession): Boolean = {
+ val dbName = tableIdentifier.database.getOrElse(sparkSession.catalog.currentDatabase)
+ val tableName = tableIdentifier.table.toLowerCase
+
+ val tablePath = CarbonStorePath.getCarbonTablePath(this.storePath,
+ new CarbonTableIdentifier(dbName, tableName, "")).getPath
+
+ val fileType = FileFactory.getFileType(tablePath)
+ FileFactory.isFileExist(tablePath, fileType)
+ }
+
+ def dropTable(tableStorePath: String, tableIdentifier: TableIdentifier)
+ (sparkSession: SparkSession) {
+ val dbName = tableIdentifier.database.get
+ val tableName = tableIdentifier.table
+
+ val metadataFilePath = CarbonStorePath.getCarbonTablePath(tableStorePath,
+ new CarbonTableIdentifier(dbName, tableName, "")).getMetadataDirectoryPath
+ val carbonTable = CarbonMetadata.getInstance.getCarbonTable(dbName + "_" + tableName)
+ if (null != carbonTable) {
+ // clear driver B-tree and dictionary cache
+ ManageDictionaryAndBTree.clearBTreeAndDictionaryLRUCache(carbonTable)
+ }
+ val fileType = FileFactory.getFileType(metadataFilePath)
+
+ if (FileFactory.isFileExist(metadataFilePath, fileType)) {
+ // while drop we should refresh the schema modified time so that if any thing has changed
+ // in the other beeline need to update.
+ checkSchemasModifiedTimeAndReloadTables
+ val file = FileFactory.getCarbonFile(metadataFilePath, fileType)
+ CarbonUtil.deleteFoldersAndFilesSilent(file.getParentFile)
+ val metadataToBeRemoved: Option[TableMeta] = getTableFromMetadata(dbName,
+ tableIdentifier.table)
+ metadataToBeRemoved match {
+ case Some(tableMeta) =>
+ metadata.tablesMeta -= tableMeta
+ CarbonMetadata.getInstance.removeTable(dbName + "_" + tableName)
+ updateSchemasUpdatedTime(touchSchemaFileSystemTime(dbName, tableName))
+ case None =>
+ LOGGER.info(s"Metadata does not contain entry for table $tableName in database $dbName")
+ }
+ CarbonHiveMetadataUtil.invalidateAndDropTable(dbName, tableName, sparkSession)
+ // discard cached table info in cachedDataSourceTables
+ sparkSession.sessionState.catalog.refreshTable(tableIdentifier)
+ }
+ }
+
+ private def getTimestampFileAndType(databaseName: String, tableName: String) = {
+ val timestampFile = storePath + "/" + CarbonCommonConstants.SCHEMAS_MODIFIED_TIME_FILE
+ val timestampFileType = FileFactory.getFileType(timestampFile)
+ (timestampFile, timestampFileType)
+ }
+
+ /**
+ * This method will put the updated timestamp of schema file in the table modified time store map
+ *
+ * @param timeStamp
+ */
+ private def updateSchemasUpdatedTime(timeStamp: Long) {
+ tableModifiedTimeStore.put("default", timeStamp)
+ }
+
+ def updateAndTouchSchemasUpdatedTime(databaseName: String, tableName: String) {
+ updateSchemasUpdatedTime(touchSchemaFileSystemTime(databaseName, tableName))
+ }
+
+ /**
+ * This method will read the timestamp of empty schema file
+ *
+ * @param databaseName
+ * @param tableName
+ * @return
+ */
+ private def readSchemaFileSystemTime(databaseName: String, tableName: String): Long = {
+ val (timestampFile, timestampFileType) = getTimestampFileAndType(databaseName, tableName)
+ if (FileFactory.isFileExist(timestampFile, timestampFileType)) {
+ FileFactory.getCarbonFile(timestampFile, timestampFileType).getLastModifiedTime
+ } else {
+ System.currentTimeMillis()
+ }
+ }
+
+ /**
+ * This method will check and create an empty schema timestamp file
+ *
+ * @param databaseName
+ * @param tableName
+ * @return
+ */
+ private def touchSchemaFileSystemTime(databaseName: String, tableName: String): Long = {
+ val (timestampFile, timestampFileType) = getTimestampFileAndType(databaseName, tableName)
+ if (!FileFactory.isFileExist(timestampFile, timestampFileType)) {
+ LOGGER.audit(s"Creating timestamp file for $databaseName.$tableName")
+ FileFactory.createNewFile(timestampFile, timestampFileType)
+ }
+ val systemTime = System.currentTimeMillis()
+ FileFactory.getCarbonFile(timestampFile, timestampFileType)
+ .setLastModifiedTime(systemTime)
+ systemTime
+ }
+
+ def checkSchemasModifiedTimeAndReloadTables() {
+ val (timestampFile, timestampFileType) = getTimestampFileAndType("", "")
+ if (FileFactory.isFileExist(timestampFile, timestampFileType)) {
+ if (!(FileFactory.getCarbonFile(timestampFile, timestampFileType).
+ getLastModifiedTime ==
+ tableModifiedTimeStore.get(CarbonCommonConstants.DATABASE_DEFAULT_NAME))) {
+ refreshCache()
+ }
+ }
+ }
+
+ private def refreshCache() {
+ metadata.tablesMeta = loadMetadata(storePath, nextQueryId).tablesMeta
+ }
+
+ override def isReadFromHiveMetaStore: Boolean = false
+
+ override def listAllTables(sparkSession: SparkSession): Seq[CarbonTable] =
+ metadata.tablesMeta.map(_.carbonTable)
+
+ override def getThriftTableInfo(tablePath: CarbonTablePath)
+ (sparkSession: SparkSession): TableInfo = {
+ val tableMetadataFile = tablePath.getSchemaFilePath
+ CarbonUtil.readSchemaFile(tableMetadataFile)
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
new file mode 100644
index 0000000..03d0bde
--- /dev/null
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetaStore.scala
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.hive
+
+import scala.collection.mutable.ArrayBuffer
+import scala.collection.JavaConverters._
+
+import org.apache.spark.sql.{CarbonDatasourceHadoopRelation, RuntimeConfig, SparkSession}
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias}
+import org.apache.spark.sql.execution.datasources.LogicalRelation
+
+import org.apache.carbondata.core.cache.dictionary.ManageDictionaryAndBTree
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata, CarbonTableIdentifier}
+import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
+import org.apache.carbondata.core.metadata.schema
+import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
+import org.apache.carbondata.core.util.CarbonUtil
+import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.format
+import org.apache.carbondata.format.SchemaEvolutionEntry
+import org.apache.carbondata.processing.merger.TableMeta
+import org.apache.carbondata.spark.util.CarbonSparkUtil
+
+/**
+ * Metastore to store carbonschema in hive
+ */
+class CarbonHiveMetaStore(conf: RuntimeConfig, storePath: String)
+ extends CarbonFileMetastore(conf, storePath) {
+
+ override def isReadFromHiveMetaStore: Boolean = true
+
+
+ /**
+ * Create spark session from paramters.
+ *
+ * @param parameters
+ * @param absIdentifier
+ * @param sparkSession
+ */
+ override def createCarbonRelation(parameters: Map[String, String],
+ absIdentifier: AbsoluteTableIdentifier,
+ sparkSession: SparkSession): CarbonRelation = {
+ val info = CarbonUtil.convertGsonToTableInfo(parameters.asJava)
+ if (info != null) {
+ val table = CarbonTable.buildFromTableInfo(info)
+ val meta = new TableMeta(table.getCarbonTableIdentifier,
+ table.getStorePath, table)
+ CarbonRelation(info.getDatabaseName, info.getFactTable.getTableName,
+ CarbonSparkUtil.createSparkMeta(table), meta)
+ } else {
+ super.createCarbonRelation(parameters, absIdentifier, sparkSession)
+ }
+ }
+
+ override def lookupRelation(tableIdentifier: TableIdentifier)
+ (sparkSession: SparkSession): LogicalPlan = {
+ val database = tableIdentifier.database.getOrElse(
+ sparkSession.catalog.currentDatabase)
+ val relation = sparkSession.sessionState.catalog.lookupRelation(tableIdentifier) match {
+ case SubqueryAlias(_,
+ LogicalRelation(carbonDatasourceHadoopRelation: CarbonDatasourceHadoopRelation, _, _),
+ _) =>
+ carbonDatasourceHadoopRelation.carbonRelation
+ case LogicalRelation(
+ carbonDatasourceHadoopRelation: CarbonDatasourceHadoopRelation, _, _) =>
+ carbonDatasourceHadoopRelation.carbonRelation
+ case _ => throw new NoSuchTableException(database, tableIdentifier.table)
+ }
+ relation
+ }
+
+ /**
+ * This method will search for a table in the catalog metadata
+ *
+ * @param database
+ * @param tableName
+ * @return
+ */
+ override def getTableFromMetadata(database: String,
+ tableName: String,
+ readStore: Boolean): Option[TableMeta] = {
+ if (!readStore) {
+ None
+ } else {
+ super.getTableFromMetadata(database, tableName, readStore)
+ }
+ }
+
+ override def tableExists(tableIdentifier: TableIdentifier)
+ (sparkSession: SparkSession): Boolean = {
+ try {
+ lookupRelation(tableIdentifier)(sparkSession)
+ } catch {
+ case e: Exception =>
+ return false
+ }
+ true
+ }
+
+ override def loadMetadata(metadataPath: String,
+ queryId: String): MetaData = {
+ MetaData(new ArrayBuffer[TableMeta])
+ }
+
+
+ /**
+ *
+ * Prepare Thrift Schema from wrapper TableInfo and write to Schema file.
+ * Load CarbonTable from wrapper tableInfo
+ *
+ */
+ override def createTableFromThrift(tableInfo: TableInfo, dbName: String,
+ tableName: String)(sparkSession: SparkSession): (String, String) = {
+ val carbonTableIdentifier = new CarbonTableIdentifier(dbName, tableName,
+ tableInfo.getFactTable.getTableId)
+ val carbonTablePath = CarbonStorePath.getCarbonTablePath(storePath, carbonTableIdentifier)
+ val schemaMetadataPath =
+ CarbonTablePath.getFolderContainingFile(carbonTablePath.getSchemaFilePath)
+ tableInfo.setMetaDataFilepath(schemaMetadataPath)
+ tableInfo.setStorePath(storePath)
+ val schemaEvolutionEntry = new schema.SchemaEvolutionEntry
+ schemaEvolutionEntry.setTimeStamp(tableInfo.getLastUpdatedTime)
+ tableInfo.getFactTable.getSchemaEvalution.getSchemaEvolutionEntryList.add(schemaEvolutionEntry)
+ removeTableFromMetadata(dbName, tableName)
+ CarbonMetadata.getInstance().loadTableMetadata(tableInfo)
+ (carbonTablePath.getPath, CarbonUtil.convertToMultiGsonStrings(tableInfo, " ", "", ","))
+ }
+
+ /**
+ * This method will remove the table meta from catalog metadata array
+ *
+ * @param dbName
+ * @param tableName
+ */
+ override def removeTableFromMetadata(dbName: String,
+ tableName: String): Unit = {
+ // do nothing
+ }
+
+ override def isTablePathExists(tableIdentifier: TableIdentifier)
+ (sparkSession: SparkSession): Boolean = {
+ tableExists(tableIdentifier)(sparkSession)
+ }
+
+ override def dropTable(tableStorePath: String, tableIdentifier: TableIdentifier)
+ (sparkSession: SparkSession): Unit = {
+ val dbName = tableIdentifier.database.get
+ val tableName = tableIdentifier.table
+
+ val metadataFilePath = CarbonStorePath.getCarbonTablePath(tableStorePath,
+ new CarbonTableIdentifier(dbName, tableName, "")).getMetadataDirectoryPath
+ val carbonTable = CarbonMetadata.getInstance.getCarbonTable(dbName + "_" + tableName)
+ if (null != carbonTable) {
+ // clear driver B-tree and dictionary cache
+ ManageDictionaryAndBTree.clearBTreeAndDictionaryLRUCache(carbonTable)
+ }
+ val fileType = FileFactory.getFileType(metadataFilePath)
+
+ if (FileFactory.isFileExist(metadataFilePath, fileType)) {
+ // while drop we should refresh the schema modified time so that if any thing has changed
+ // in the other beeline need to update.
+ val file = FileFactory.getCarbonFile(metadataFilePath, fileType)
+ CarbonUtil.deleteFoldersAndFilesSilent(file.getParentFile)
+ }
+ CarbonHiveMetadataUtil.invalidateAndDropTable(dbName, tableName, sparkSession)
+ // discard cached table info in cachedDataSourceTables
+ sparkSession.sessionState.catalog.refreshTable(tableIdentifier)
+ }
+
+ override def checkSchemasModifiedTimeAndReloadTables(): Unit = {
+ // do nothing now
+ }
+
+ override def listAllTables(sparkSession: SparkSession): Seq[CarbonTable] = {
+ // Todo
+ Seq()
+ }
+
+ override def getThriftTableInfo(tablePath: CarbonTablePath)
+ (sparkSession: SparkSession): format.TableInfo = {
+ val identifier = tablePath.getCarbonTableIdentifier
+ val relation = lookupRelation(TableIdentifier(identifier.getTableName,
+ Some(identifier.getDatabaseName)))(sparkSession).asInstanceOf[CarbonRelation]
+ val carbonTable = relation.metaData.carbonTable
+ val schemaConverter = new ThriftWrapperSchemaConverterImpl
+ schemaConverter.fromWrapperToExternalTableInfo(carbonTable.getTableInfo,
+ carbonTable.getDatabaseName,
+ carbonTable.getFactTableName)
+ }
+
+ /**
+ * This method will overwrite the existing schema and update it with the given details
+ *
+ * @param newTableIdentifier
+ * @param thriftTableInfo
+ * @param schemaEvolutionEntry
+ * @param carbonStorePath
+ * @param sparkSession
+ */
+ override def updateTableSchema(newTableIdentifier: CarbonTableIdentifier,
+ oldTableIdentifier: CarbonTableIdentifier,
+ thriftTableInfo: format.TableInfo,
+ schemaEvolutionEntry: SchemaEvolutionEntry,
+ carbonStorePath: String)
+ (sparkSession: SparkSession): String = {
+ val schemaConverter = new ThriftWrapperSchemaConverterImpl
+ if (schemaEvolutionEntry != null) {
+ thriftTableInfo.fact_table.schema_evolution.schema_evolution_history.add(schemaEvolutionEntry)
+ }
+ updateHiveMetaStore(newTableIdentifier,
+ oldTableIdentifier,
+ thriftTableInfo,
+ carbonStorePath,
+ sparkSession,
+ schemaConverter)
+ }
+
+ private def updateHiveMetaStore(newTableIdentifier: CarbonTableIdentifier,
+ oldTableIdentifier: CarbonTableIdentifier,
+ thriftTableInfo: format.TableInfo,
+ carbonStorePath: String,
+ sparkSession: SparkSession,
+ schemaConverter: ThriftWrapperSchemaConverterImpl) = {
+ val wrapperTableInfo = schemaConverter
+ .fromExternalToWrapperTableInfo(thriftTableInfo,
+ newTableIdentifier.getDatabaseName,
+ newTableIdentifier.getTableName,
+ carbonStorePath)
+ wrapperTableInfo.setStorePath(storePath)
+ val carbonTablePath = CarbonStorePath.getCarbonTablePath(storePath, newTableIdentifier)
+ val schemaMetadataPath =
+ CarbonTablePath.getFolderContainingFile(carbonTablePath.getSchemaFilePath)
+ wrapperTableInfo.setMetaDataFilepath(schemaMetadataPath)
+ val dbName = oldTableIdentifier.getDatabaseName
+ val tableName = oldTableIdentifier.getTableName
+ val carbonUpdatedIdentifier = new CarbonTableIdentifier(dbName, tableName,
+ wrapperTableInfo.getFactTable.getTableId)
+ val schemaParts = CarbonUtil.convertToMultiGsonStrings(wrapperTableInfo, "=", "'", "")
+ sparkSession.sessionState.asInstanceOf[CarbonSessionState].metadataHive.runSqlHive(
+ s"ALTER TABLE $dbName.$tableName SET SERDEPROPERTIES($schemaParts)")
+ sparkSession.catalog.refreshTable(TableIdentifier(tableName, Some(dbName)).quotedString)
+ removeTableFromMetadata(wrapperTableInfo.getDatabaseName,
+ wrapperTableInfo.getFactTable.getTableName)
+ CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo)
+ CarbonStorePath.getCarbonTablePath(storePath, carbonUpdatedIdentifier).getPath
+ }
+
+ /**
+ * This method will is used to remove the evolution entry in case of failure.
+ *
+ * @param carbonTableIdentifier
+ * @param thriftTableInfo
+ * @param carbonStorePath
+ * @param sparkSession
+ */
+ override def revertTableSchema(carbonTableIdentifier: CarbonTableIdentifier,
+ thriftTableInfo: format.TableInfo,
+ carbonStorePath: String)
+ (sparkSession: SparkSession): String = {
+ val schemaConverter = new ThriftWrapperSchemaConverterImpl
+ val evolutionEntries = thriftTableInfo.fact_table.schema_evolution.schema_evolution_history
+ evolutionEntries.remove(evolutionEntries.size() - 1)
+ updateHiveMetaStore(carbonTableIdentifier,
+ carbonTableIdentifier,
+ thriftTableInfo,
+ carbonStorePath,
+ sparkSession,
+ schemaConverter)
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
new file mode 100644
index 0000000..ab27b4f
--- /dev/null
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.hive
+
+import org.apache.spark.sql.{RuntimeConfig, SparkSession}
+import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
+import org.apache.carbondata.core.metadata.schema.table
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
+import org.apache.carbondata.format.{SchemaEvolutionEntry, TableInfo}
+import org.apache.carbondata.processing.merger.TableMeta
+
+/**
+ * Interface for Carbonmetastore
+ */
+trait CarbonMetaStore {
+
+ def lookupRelation(dbName: Option[String], tableName: String)
+ (sparkSession: SparkSession): LogicalPlan
+
+ def lookupRelation(tableIdentifier: TableIdentifier)
+ (sparkSession: SparkSession): LogicalPlan
+
+ /**
+ * Create spark session from paramters.
+ * @param parameters
+ * @param absIdentifier
+ * @param sparkSession
+ */
+ def createCarbonRelation(parameters: Map[String, String],
+ absIdentifier: AbsoluteTableIdentifier,
+ sparkSession: SparkSession): CarbonRelation
+
+ /**
+ * Get table meta
+ * TODO remove it if possible
+ * @param database
+ * @param tableName
+ * @param readStore
+ * @return
+ */
+ def getTableFromMetadata(database: String,
+ tableName: String,
+ readStore: Boolean = false): Option[TableMeta]
+
+ def tableExists(
+ table: String,
+ databaseOp: Option[String] = None)(sparkSession: SparkSession): Boolean
+
+ def tableExists(tableIdentifier: TableIdentifier)(sparkSession: SparkSession): Boolean
+
+ def loadMetadata(metadataPath: String, queryId: String): MetaData
+
+ /**
+ * This method will overwrite the existing schema and update it with the given details
+ *
+ * @param newTableIdentifier
+ * @param thriftTableInfo
+ * @param schemaEvolutionEntry
+ * @param carbonStorePath
+ * @param sparkSession
+ */
+ def updateTableSchema(newTableIdentifier: CarbonTableIdentifier,
+ oldTableIdentifier: CarbonTableIdentifier,
+ thriftTableInfo: org.apache.carbondata.format.TableInfo,
+ schemaEvolutionEntry: SchemaEvolutionEntry,
+ carbonStorePath: String)(sparkSession: SparkSession): String
+
+ /**
+ * This method will is used to remove the evolution entry in case of failure.
+ *
+ * @param carbonTableIdentifier
+ * @param thriftTableInfo
+ * @param carbonStorePath
+ * @param sparkSession
+ */
+ def revertTableSchema(carbonTableIdentifier: CarbonTableIdentifier,
+ thriftTableInfo: org.apache.carbondata.format.TableInfo,
+ carbonStorePath: String)
+ (sparkSession: SparkSession): String
+
+ /**
+ *
+ * Prepare Thrift Schema from wrapper TableInfo and write to Schema file.
+ * Load CarbonTable from wrapper tableInfo
+ *
+ */
+ def createTableFromThrift(tableInfo: table.TableInfo,
+ dbName: String,
+ tableName: String)(sparkSession: SparkSession): (String, String)
+
+ /**
+ * This method will remove the table meta from catalog metadata array
+ *
+ * @param dbName
+ * @param tableName
+ */
+ def removeTableFromMetadata(dbName: String, tableName: String): Unit
+
+ def updateMetadataByThriftTable(schemaFilePath: String,
+ tableInfo: TableInfo, dbName: String, tableName: String, storePath: String): Unit
+
+ def isTablePathExists(tableIdentifier: TableIdentifier)(sparkSession: SparkSession): Boolean
+
+ def dropTable(tableStorePath: String, tableIdentifier: TableIdentifier)
+ (sparkSession: SparkSession)
+
+ def updateAndTouchSchemasUpdatedTime(databaseName: String, tableName: String)
+
+ def checkSchemasModifiedTimeAndReloadTables()
+
+ def isReadFromHiveMetaStore : Boolean
+
+ def listAllTables(sparkSession: SparkSession): Seq[CarbonTable]
+
+ def storePath: String
+
+ def getThriftTableInfo(tablePath: CarbonTablePath)(sparkSession: SparkSession): TableInfo
+
+}
+
+/**
+ * Factory for Carbon metastore
+ */
+object CarbonMetaStoreFactory {
+
+ def createCarbonMetaStore(conf: RuntimeConfig, storePath: String): CarbonMetaStore = {
+ val readSchemaFromHiveMetaStore = readSchemaFromHive(conf)
+ if (readSchemaFromHiveMetaStore) {
+ new CarbonHiveMetaStore(conf, storePath)
+ } else {
+ new CarbonFileMetastore(conf, storePath)
+ }
+ }
+
+ def readSchemaFromHive(conf: RuntimeConfig): Boolean = {
+ val readSchemaFromHive = {
+ if (conf.contains(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE)) {
+ conf.get(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE)
+ } else if (System.getProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE) != null) {
+ System.getProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE)
+ } else {
+ CarbonProperties.getInstance().
+ getProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
+ CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT)
+ }
+ }
+ readSchemaFromHive.toBoolean
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2a9debfc/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
deleted file mode 100644
index 04a94ce..0000000
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
+++ /dev/null
@@ -1,960 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.hive
-
-import java.io._
-import java.util.{GregorianCalendar, LinkedHashSet, UUID}
-import java.util.concurrent.atomic.AtomicLong
-
-import scala.Array.canBuildFrom
-import scala.collection.JavaConverters._
-import scala.collection.mutable.ArrayBuffer
-import scala.util.parsing.combinator.RegexParsers
-
-import org.apache.spark.sql.{RuntimeConfig, SparkSession}
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.{MultiInstanceRelation, NoSuchTableException}
-import org.apache.spark.sql.catalyst.expressions.AttributeReference
-import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Statistics}
-import org.apache.spark.sql.execution.command.Partitioner
-import org.apache.spark.sql.types._
-
-import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastore.filesystem.CarbonFile
-import org.apache.carbondata.core.datastore.impl.FileFactory
-import org.apache.carbondata.core.datastore.impl.FileFactory.FileType
-import org.apache.carbondata.core.fileoperations.FileWriteOperation
-import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifier}
-import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
-import org.apache.carbondata.core.metadata.datatype.DataType.DECIMAL
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension}
-import org.apache.carbondata.core.reader.ThriftReader
-import org.apache.carbondata.core.stats.{QueryStatistic, QueryStatisticsConstants}
-import org.apache.carbondata.core.statusmanager.SegmentStatusManager
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, CarbonUtil}
-import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath}
-import org.apache.carbondata.core.writer.ThriftWriter
-import org.apache.carbondata.format.{SchemaEvolutionEntry, TableInfo}
-import org.apache.carbondata.processing.merger.TableMeta
-import org.apache.carbondata.spark.util.CarbonSparkUtil
-
-case class MetaData(var tablesMeta: ArrayBuffer[TableMeta]) {
- // clear the metadata
- def clear(): Unit = {
- tablesMeta.clear()
- }
-}
-
-case class CarbonMetaData(dims: Seq[String],
- msrs: Seq[String],
- carbonTable: CarbonTable,
- dictionaryMap: DictionaryMap)
-
-object CarbonMetastore {
-
- def readSchemaFileToThriftTable(schemaFilePath: String): TableInfo = {
- val createTBase = new ThriftReader.TBaseCreator() {
- override def create(): org.apache.thrift.TBase[TableInfo, TableInfo._Fields] = {
- new TableInfo()
- }
- }
- val thriftReader = new ThriftReader(schemaFilePath, createTBase)
- var tableInfo: TableInfo = null
- try {
- thriftReader.open()
- tableInfo = thriftReader.read().asInstanceOf[TableInfo]
- } finally {
- thriftReader.close()
- }
- tableInfo
- }
-
- def writeThriftTableToSchemaFile(schemaFilePath: String, tableInfo: TableInfo): Unit = {
- val thriftWriter = new ThriftWriter(schemaFilePath, false)
- try {
- thriftWriter.open()
- thriftWriter.write(tableInfo);
- } finally {
- thriftWriter.close()
- }
- }
-
-}
-
-case class DictionaryMap(dictionaryMap: Map[String, Boolean]) {
- def get(name: String): Option[Boolean] = {
- dictionaryMap.get(name.toLowerCase)
- }
-}
-
-class CarbonMetastore(conf: RuntimeConfig, val storePath: String) {
-
- @transient
- val LOGGER = LogServiceFactory.getLogService("org.apache.spark.sql.CarbonMetastoreCatalog")
-
- val tableModifiedTimeStore = new java.util.HashMap[String, Long]()
- tableModifiedTimeStore
- .put(CarbonCommonConstants.DATABASE_DEFAULT_NAME, System.currentTimeMillis())
-
- private val nextId = new AtomicLong(0)
-
- def nextQueryId: String = {
- System.nanoTime() + ""
- }
-
- val metadata = loadMetadata(storePath, nextQueryId)
-
- def getTableCreationTime(databaseName: String, tableName: String): Long = {
- val tableMeta = metadata.tablesMeta.filter(
- c => c.carbonTableIdentifier.getDatabaseName.equalsIgnoreCase(databaseName) &&
- c.carbonTableIdentifier.getTableName.equalsIgnoreCase(tableName))
- val tableCreationTime = tableMeta.head.carbonTable.getTableLastUpdatedTime
- tableCreationTime
- }
-
- def cleanStore(): Unit = {
- try {
- val fileType = FileFactory.getFileType(storePath)
- FileFactory.deleteFile(storePath, fileType)
- metadata.clear()
- } catch {
- case e: Throwable => LOGGER.error(e, "clean store failed")
- }
- }
-
- def lookupRelation(dbName: Option[String], tableName: String)
- (sparkSession: SparkSession): LogicalPlan = {
- lookupRelation(TableIdentifier(tableName, dbName))(sparkSession)
- }
-
- def lookupRelation(tableIdentifier: TableIdentifier, alias: Option[String] = None)
- (sparkSession: SparkSession): LogicalPlan = {
- checkSchemasModifiedTimeAndReloadTables()
- val database = tableIdentifier.database.getOrElse(
- sparkSession.catalog.currentDatabase
- )
- val tables = getTableFromMetadata(database, tableIdentifier.table)
- tables match {
- case Some(t) =>
- CarbonRelation(database, tableIdentifier.table,
- CarbonSparkUtil.createSparkMeta(tables.head.carbonTable), tables.head, alias)
- case None =>
- throw new NoSuchTableException(database, tableIdentifier.table)
- }
- }
-
- /**
- * This method will search for a table in the catalog metadata
- *
- * @param database
- * @param tableName
- * @return
- */
- def getTableFromMetadata(database: String,
- tableName: String): Option[TableMeta] = {
- metadata.tablesMeta
- .find(c => c.carbonTableIdentifier.getDatabaseName.equalsIgnoreCase(database) &&
- c.carbonTableIdentifier.getTableName.equalsIgnoreCase(tableName))
- }
-
- def tableExists(
- table: String,
- databaseOp: Option[String] = None)(sparkSession: SparkSession): Boolean = {
- checkSchemasModifiedTimeAndReloadTables()
- val database = databaseOp.getOrElse(sparkSession.catalog.currentDatabase)
- val tables = metadata.tablesMeta.filter(
- c => c.carbonTableIdentifier.getDatabaseName.equalsIgnoreCase(database) &&
- c.carbonTableIdentifier.getTableName.equalsIgnoreCase(table))
- tables.nonEmpty
- }
-
- def tableExists(tableIdentifier: TableIdentifier)(sparkSession: SparkSession): Boolean = {
- checkSchemasModifiedTimeAndReloadTables()
- val database = tableIdentifier.database.getOrElse(sparkSession.catalog.currentDatabase)
- val tables = metadata.tablesMeta.filter(
- c => c.carbonTableIdentifier.getDatabaseName.equalsIgnoreCase(database) &&
- c.carbonTableIdentifier.getTableName.equalsIgnoreCase(tableIdentifier.table))
- tables.nonEmpty
- }
-
- def loadMetadata(metadataPath: String, queryId: String): MetaData = {
- val recorder = CarbonTimeStatisticsFactory.createDriverRecorder()
- val statistic = new QueryStatistic()
- // creating zookeeper instance once.
- // if zookeeper is configured as carbon lock type.
- val zookeeperurl = conf.get(CarbonCommonConstants.ZOOKEEPER_URL, null)
- if (null != zookeeperurl) {
- CarbonProperties.getInstance
- .addProperty(CarbonCommonConstants.ZOOKEEPER_URL, zookeeperurl)
- }
- if (metadataPath == null) {
- return null
- }
- // if no locktype is configured and store type is HDFS set HDFS lock as default
- if (null == CarbonProperties.getInstance
- .getProperty(CarbonCommonConstants.LOCK_TYPE) &&
- FileType.HDFS == FileFactory.getFileType(metadataPath)) {
- CarbonProperties.getInstance
- .addProperty(CarbonCommonConstants.LOCK_TYPE,
- CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS
- )
- LOGGER.info("Default lock type HDFSLOCK is configured")
- }
- val fileType = FileFactory.getFileType(metadataPath)
- val metaDataBuffer = new ArrayBuffer[TableMeta]
- fillMetaData(metadataPath, fileType, metaDataBuffer)
- updateSchemasUpdatedTime(readSchemaFileSystemTime("", ""))
- statistic.addStatistics(QueryStatisticsConstants.LOAD_META,
- System.currentTimeMillis())
- recorder.recordStatisticsForDriver(statistic, queryId)
- MetaData(metaDataBuffer)
- }
-
- private def fillMetaData(basePath: String, fileType: FileType,
- metaDataBuffer: ArrayBuffer[TableMeta]): Unit = {
- val databasePath = basePath // + "/schemas"
- try {
- if (FileFactory.isFileExist(databasePath, fileType)) {
- val file = FileFactory.getCarbonFile(databasePath, fileType)
- val databaseFolders = file.listFiles()
-
- databaseFolders.foreach(databaseFolder => {
- if (databaseFolder.isDirectory) {
- val dbName = databaseFolder.getName
- val tableFolders = databaseFolder.listFiles()
-
- tableFolders.foreach(tableFolder => {
- if (tableFolder.isDirectory) {
- val carbonTableIdentifier = new CarbonTableIdentifier(databaseFolder.getName,
- tableFolder.getName, UUID.randomUUID().toString)
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(basePath,
- carbonTableIdentifier)
- val tableMetadataFile = carbonTablePath.getSchemaFilePath
-
- if (FileFactory.isFileExist(tableMetadataFile, fileType)) {
- val tableName = tableFolder.getName
- val tableUniqueName = databaseFolder.getName + "_" + tableFolder.getName
- val tableInfo: TableInfo = readSchemaFile(tableMetadataFile)
- val schemaConverter = new ThriftWrapperSchemaConverterImpl
- val wrapperTableInfo = schemaConverter
- .fromExternalToWrapperTableInfo(tableInfo, dbName, tableName, basePath)
- val schemaFilePath = CarbonStorePath
- .getCarbonTablePath(storePath, carbonTableIdentifier).getSchemaFilePath
- wrapperTableInfo.setStorePath(storePath)
- wrapperTableInfo
- .setMetaDataFilepath(CarbonTablePath.getFolderContainingFile(schemaFilePath))
- CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo)
- val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
- metaDataBuffer += new TableMeta(carbonTable.getCarbonTableIdentifier, storePath,
- carbonTable)
- }
- }
- })
- }
- })
- } else {
- // Create folders and files.
- FileFactory.mkdirs(databasePath, fileType)
- }
- } catch {
- case s: java.io.FileNotFoundException =>
- // Create folders and files.
- FileFactory.mkdirs(databasePath, fileType)
- }
- }
-
- /**
- * This method will read the schema file from a given path
- *
- * @param schemaFilePath
- * @return
- */
- def readSchemaFile(schemaFilePath: String): TableInfo = {
- val createTBase = new ThriftReader.TBaseCreator() {
- override def create(): org.apache.thrift.TBase[TableInfo, TableInfo._Fields] = {
- new TableInfo()
- }
- }
- val thriftReader = new ThriftReader(schemaFilePath, createTBase)
- thriftReader.open()
- val tableInfo: TableInfo = thriftReader.read().asInstanceOf[TableInfo]
- thriftReader.close()
- tableInfo
- }
-
- /**
- * This method will overwrite the existing schema and update it with the given details
- *
- * @param carbonTableIdentifier
- * @param thriftTableInfo
- * @param schemaEvolutionEntry
- * @param carbonStorePath
- * @param sparkSession
- */
- def updateTableSchema(carbonTableIdentifier: CarbonTableIdentifier,
- thriftTableInfo: org.apache.carbondata.format.TableInfo,
- schemaEvolutionEntry: SchemaEvolutionEntry,
- carbonStorePath: String)
- (sparkSession: SparkSession): String = {
- val schemaConverter = new ThriftWrapperSchemaConverterImpl
- thriftTableInfo.fact_table.schema_evolution.schema_evolution_history.add(schemaEvolutionEntry)
- val wrapperTableInfo = schemaConverter
- .fromExternalToWrapperTableInfo(thriftTableInfo,
- carbonTableIdentifier.getDatabaseName,
- carbonTableIdentifier.getTableName,
- carbonStorePath)
- createSchemaThriftFile(wrapperTableInfo,
- thriftTableInfo,
- carbonTableIdentifier.getDatabaseName,
- carbonTableIdentifier.getTableName)(sparkSession)
- }
-
- /**
- * This method will is used to remove the evolution entry in case of failure.
- *
- * @param carbonTableIdentifier
- * @param thriftTableInfo
- * @param carbonStorePath
- * @param sparkSession
- */
- def revertTableSchema(carbonTableIdentifier: CarbonTableIdentifier,
- thriftTableInfo: org.apache.carbondata.format.TableInfo,
- carbonStorePath: String)
- (sparkSession: SparkSession): String = {
- val schemaConverter = new ThriftWrapperSchemaConverterImpl
- val wrapperTableInfo = schemaConverter
- .fromExternalToWrapperTableInfo(thriftTableInfo,
- carbonTableIdentifier.getDatabaseName,
- carbonTableIdentifier.getTableName,
- carbonStorePath)
- val evolutionEntries = thriftTableInfo.fact_table.schema_evolution.schema_evolution_history
- evolutionEntries.remove(evolutionEntries.size() - 1)
- createSchemaThriftFile(wrapperTableInfo,
- thriftTableInfo,
- carbonTableIdentifier.getDatabaseName,
- carbonTableIdentifier.getTableName)(sparkSession)
- }
-
-
-
- /**
- *
- * Prepare Thrift Schema from wrapper TableInfo and write to Schema file.
- * Load CarbonTable from wrapper tableInfo
- *
- */
- def createTableFromThrift(
- tableInfo: org.apache.carbondata.core.metadata.schema.table.TableInfo,
- dbName: String, tableName: String)
- (sparkSession: SparkSession): String = {
- if (tableExists(tableName, Some(dbName))(sparkSession)) {
- sys.error(s"Table [$tableName] already exists under Database [$dbName]")
- }
- val schemaEvolutionEntry = new SchemaEvolutionEntry(tableInfo.getLastUpdatedTime)
- val schemaConverter = new ThriftWrapperSchemaConverterImpl
- val thriftTableInfo = schemaConverter
- .fromWrapperToExternalTableInfo(tableInfo, dbName, tableName)
- thriftTableInfo.getFact_table.getSchema_evolution.getSchema_evolution_history
- .add(schemaEvolutionEntry)
- val carbonTablePath = createSchemaThriftFile(tableInfo,
- thriftTableInfo,
- dbName,
- tableName)(sparkSession)
- LOGGER.info(s"Table $tableName for Database $dbName created successfully.")
- carbonTablePath
- }
-
- /**
- * This method will write the schema thrift file in carbon store and load table metadata
- *
- * @param tableInfo
- * @param thriftTableInfo
- * @param dbName
- * @param tableName
- * @param sparkSession
- * @return
- */
- private def createSchemaThriftFile(
- tableInfo: org.apache.carbondata.core.metadata.schema.table.TableInfo,
- thriftTableInfo: org.apache.carbondata.format.TableInfo,
- dbName: String, tableName: String)
- (sparkSession: SparkSession): String = {
- val carbonTableIdentifier = new CarbonTableIdentifier(dbName, tableName,
- tableInfo.getFactTable.getTableId)
- val carbonTablePath = CarbonStorePath.getCarbonTablePath(storePath, carbonTableIdentifier)
- val schemaFilePath = carbonTablePath.getSchemaFilePath
- val schemaMetadataPath = CarbonTablePath.getFolderContainingFile(schemaFilePath)
- tableInfo.setMetaDataFilepath(schemaMetadataPath)
- tableInfo.setStorePath(storePath)
- val fileType = FileFactory.getFileType(schemaMetadataPath)
- if (!FileFactory.isFileExist(schemaMetadataPath, fileType)) {
- FileFactory.mkdirs(schemaMetadataPath, fileType)
- }
- val thriftWriter = new ThriftWriter(schemaFilePath, false)
- thriftWriter.open(FileWriteOperation.OVERWRITE)
- thriftWriter.write(thriftTableInfo)
- thriftWriter.close()
- removeTableFromMetadata(dbName, tableName)
- CarbonMetadata.getInstance().loadTableMetadata(tableInfo)
- val tableMeta = new TableMeta(carbonTableIdentifier, storePath,
- CarbonMetadata.getInstance().getCarbonTable(dbName + '_' + tableName))
- metadata.tablesMeta += tableMeta
- updateSchemasUpdatedTime(touchSchemaFileSystemTime(dbName, tableName))
- carbonTablePath.getPath
- }
-
- /**
- * This method will remove the table meta from catalog metadata array
- *
- * @param dbName
- * @param tableName
- */
- def removeTableFromMetadata(dbName: String, tableName: String): Unit = {
- val metadataToBeRemoved: Option[TableMeta] = getTableFromMetadata(dbName, tableName)
- metadataToBeRemoved match {
- case Some(tableMeta) =>
- metadata.tablesMeta -= tableMeta
- CarbonMetadata.getInstance.removeTable(dbName + "_" + tableName)
- case None =>
- LOGGER.debug(s"No entry for table $tableName in database $dbName")
- }
- }
-
- private def updateMetadataByWrapperTable(
- wrapperTableInfo: org.apache.carbondata.core.metadata.schema.table.TableInfo): Unit = {
-
- CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo)
- val carbonTable = CarbonMetadata.getInstance().getCarbonTable(
- wrapperTableInfo.getTableUniqueName)
- for (i <- metadata.tablesMeta.indices) {
- if (wrapperTableInfo.getTableUniqueName.equals(
- metadata.tablesMeta(i).carbonTableIdentifier.getTableUniqueName)) {
- metadata.tablesMeta(i).carbonTable = carbonTable
- }
- }
- }
-
- def updateMetadataByThriftTable(schemaFilePath: String,
- tableInfo: TableInfo, dbName: String, tableName: String, storePath: String): Unit = {
-
- tableInfo.getFact_table.getSchema_evolution.getSchema_evolution_history.get(0)
- .setTime_stamp(System.currentTimeMillis())
- val schemaConverter = new ThriftWrapperSchemaConverterImpl
- val wrapperTableInfo = schemaConverter
- .fromExternalToWrapperTableInfo(tableInfo, dbName, tableName, storePath)
- wrapperTableInfo
- .setMetaDataFilepath(CarbonTablePath.getFolderContainingFile(schemaFilePath))
- wrapperTableInfo.setStorePath(storePath)
- updateMetadataByWrapperTable(wrapperTableInfo)
- }
-
- /**
- * Shows all schemas which has Database name like
- */
- def showDatabases(schemaLike: Option[String]): Seq[String] = {
- checkSchemasModifiedTimeAndReloadTables()
- metadata.tablesMeta.map { c =>
- schemaLike match {
- case Some(name) =>
- if (c.carbonTableIdentifier.getDatabaseName.contains(name)) {
- c.carbonTableIdentifier
- .getDatabaseName
- } else {
- null
- }
- case _ => c.carbonTableIdentifier.getDatabaseName
- }
- }.filter(f => f != null)
- }
-
- /**
- * Shows all tables in all schemas.
- */
- def getAllTables(): Seq[TableIdentifier] = {
- checkSchemasModifiedTimeAndReloadTables()
- metadata.tablesMeta.map { c =>
- TableIdentifier(c.carbonTableIdentifier.getTableName,
- Some(c.carbonTableIdentifier.getDatabaseName))
- }
- }
-
- def isTablePathExists(tableIdentifier: TableIdentifier)(sparkSession: SparkSession): Boolean = {
- val dbName = tableIdentifier.database.getOrElse(sparkSession.catalog.currentDatabase)
- val tableName = tableIdentifier.table.toLowerCase
-
- val tablePath = CarbonStorePath.getCarbonTablePath(this.storePath,
- new CarbonTableIdentifier(dbName, tableName, "")).getPath
-
- val fileType = FileFactory.getFileType(tablePath)
- FileFactory.isFileExist(tablePath, fileType)
- }
-
- def dropTable(tableStorePath: String, tableIdentifier: TableIdentifier)
- (sparkSession: SparkSession) {
- val dbName = tableIdentifier.database.get
- val tableName = tableIdentifier.table
-
- val metadataFilePath = CarbonStorePath.getCarbonTablePath(tableStorePath,
- new CarbonTableIdentifier(dbName, tableName, "")).getMetadataDirectoryPath
-
- val fileType = FileFactory.getFileType(metadataFilePath)
-
- if (FileFactory.isFileExist(metadataFilePath, fileType)) {
- // while drop we should refresh the schema modified time so that if any thing has changed
- // in the other beeline need to update.
- checkSchemasModifiedTimeAndReloadTables
- val file = FileFactory.getCarbonFile(metadataFilePath, fileType)
- CarbonUtil.deleteFoldersAndFilesSilent(file.getParentFile)
- val metadataToBeRemoved: Option[TableMeta] = getTableFromMetadata(dbName,
- tableIdentifier.table)
- metadataToBeRemoved match {
- case Some(tableMeta) =>
- metadata.tablesMeta -= tableMeta
- CarbonMetadata.getInstance.removeTable(dbName + "_" + tableName)
- updateSchemasUpdatedTime(touchSchemaFileSystemTime(dbName, tableName))
- case None =>
- LOGGER.info(s"Metadata does not contain entry for table $tableName in database $dbName")
- }
- CarbonHiveMetadataUtil.invalidateAndDropTable(dbName, tableName, sparkSession)
- // discard cached table info in cachedDataSourceTables
- sparkSession.sessionState.catalog.refreshTable(tableIdentifier)
- }
- }
-
- private def getTimestampFileAndType(databaseName: String, tableName: String) = {
- val timestampFile = storePath + "/" + CarbonCommonConstants.SCHEMAS_MODIFIED_TIME_FILE
- val timestampFileType = FileFactory.getFileType(timestampFile)
- (timestampFile, timestampFileType)
- }
-
- /**
- * This method will put the updated timestamp of schema file in the table modified time store map
- *
- * @param timeStamp
- */
- def updateSchemasUpdatedTime(timeStamp: Long) {
- tableModifiedTimeStore.put("default", timeStamp)
- }
-
- /**
- * This method will read the timestamp of empty schema file
- *
- * @param databaseName
- * @param tableName
- * @return
- */
- def readSchemaFileSystemTime(databaseName: String, tableName: String): Long = {
- val (timestampFile, timestampFileType) = getTimestampFileAndType(databaseName, tableName)
- if (FileFactory.isFileExist(timestampFile, timestampFileType)) {
- FileFactory.getCarbonFile(timestampFile, timestampFileType).getLastModifiedTime
- } else {
- System.currentTimeMillis()
- }
- }
-
- /**
- * This method will check and create an empty schema timestamp file
- *
- * @param databaseName
- * @param tableName
- * @return
- */
- def touchSchemaFileSystemTime(databaseName: String, tableName: String): Long = {
- val (timestampFile, timestampFileType) = getTimestampFileAndType(databaseName, tableName)
- if (!FileFactory.isFileExist(timestampFile, timestampFileType)) {
- LOGGER.audit(s"Creating timestamp file for $databaseName.$tableName")
- FileFactory.createNewFile(timestampFile, timestampFileType)
- }
- val systemTime = System.currentTimeMillis()
- FileFactory.getCarbonFile(timestampFile, timestampFileType)
- .setLastModifiedTime(systemTime)
- systemTime
- }
-
- def checkSchemasModifiedTimeAndReloadTables() {
- val (timestampFile, timestampFileType) = getTimestampFileAndType("", "")
- if (FileFactory.isFileExist(timestampFile, timestampFileType)) {
- if (!(FileFactory.getCarbonFile(timestampFile, timestampFileType).
- getLastModifiedTime ==
- tableModifiedTimeStore.get(CarbonCommonConstants.DATABASE_DEFAULT_NAME))) {
- refreshCache()
- }
- }
- }
-
- def refreshCache() {
- metadata.tablesMeta = loadMetadata(storePath, nextQueryId).tablesMeta
- }
-
- def getSchemaLastUpdatedTime(databaseName: String, tableName: String): Long = {
- var schemaLastUpdatedTime = System.currentTimeMillis
- val (timestampFile, timestampFileType) = getTimestampFileAndType(databaseName, tableName)
- if (FileFactory.isFileExist(timestampFile, timestampFileType)) {
- schemaLastUpdatedTime = FileFactory.getCarbonFile(timestampFile, timestampFileType)
- .getLastModifiedTime
- }
- schemaLastUpdatedTime
- }
-
- def readTableMetaDataFile(tableFolder: CarbonFile,
- fileType: FileFactory.FileType):
- (String, String, String, String, Partitioner, Long) = {
- val tableMetadataFile = tableFolder.getAbsolutePath + "/metadata"
-
- var schema: String = ""
- var databaseName: String = ""
- var tableName: String = ""
- var dataPath: String = ""
- var partitioner: Partitioner = null
- val cal = new GregorianCalendar(2011, 1, 1)
- var tableCreationTime = cal.getTime.getTime
-
- if (FileFactory.isFileExist(tableMetadataFile, fileType)) {
- // load metadata
- val in = FileFactory.getDataInputStream(tableMetadataFile, fileType)
- var len = 0
- try {
- len = in.readInt()
- } catch {
- case others: EOFException => len = 0
- }
-
- while (len > 0) {
- val databaseNameBytes = new Array[Byte](len)
- in.readFully(databaseNameBytes)
-
- databaseName = new String(databaseNameBytes, "UTF8")
- val tableNameLen = in.readInt()
- val tableNameBytes = new Array[Byte](tableNameLen)
- in.readFully(tableNameBytes)
- tableName = new String(tableNameBytes, "UTF8")
-
- val dataPathLen = in.readInt()
- val dataPathBytes = new Array[Byte](dataPathLen)
- in.readFully(dataPathBytes)
- dataPath = new String(dataPathBytes, "UTF8")
-
- val versionLength = in.readInt()
- val versionBytes = new Array[Byte](versionLength)
- in.readFully(versionBytes)
-
- val schemaLen = in.readInt()
- val schemaBytes = new Array[Byte](schemaLen)
- in.readFully(schemaBytes)
- schema = new String(schemaBytes, "UTF8")
-
- val partitionLength = in.readInt()
- val partitionBytes = new Array[Byte](partitionLength)
- in.readFully(partitionBytes)
- val inStream = new ByteArrayInputStream(partitionBytes)
- val objStream = new ObjectInputStream(inStream)
- partitioner = objStream.readObject().asInstanceOf[Partitioner]
- objStream.close()
-
- try {
- tableCreationTime = in.readLong()
- len = in.readInt()
- } catch {
- case others: EOFException => len = 0
- }
-
- }
- in.close()
- }
-
- (databaseName, tableName, dataPath, schema, partitioner, tableCreationTime)
- }
-
- def createDatabaseDirectory(dbName: String) {
- val databasePath = storePath + File.separator + dbName.toLowerCase
- val fileType = FileFactory.getFileType(databasePath)
- FileFactory.mkdirs(databasePath, fileType)
- }
-
- def dropDatabaseDirectory(dbName: String) {
- val databasePath = storePath + File.separator + dbName
- val fileType = FileFactory.getFileType(databasePath)
- if (FileFactory.isFileExist(databasePath, fileType)) {
- val dbPath = FileFactory.getCarbonFile(databasePath, fileType)
- CarbonUtil.deleteFoldersAndFiles(dbPath)
- }
- }
-
-}
-
-
-object CarbonMetastoreTypes extends RegexParsers {
- protected lazy val primitiveType: Parser[DataType] =
- "string" ^^^ StringType |
- "float" ^^^ FloatType |
- "int" ^^^ IntegerType |
- "tinyint" ^^^ ShortType |
- "short" ^^^ ShortType |
- "double" ^^^ DoubleType |
- "long" ^^^ LongType |
- "binary" ^^^ BinaryType |
- "boolean" ^^^ BooleanType |
- fixedDecimalType |
- "decimal" ^^^ "decimal" ^^^ DecimalType(10, 0) |
- "varchar\\((\\d+)\\)".r ^^^ StringType |
- "date" ^^^ DateType |
- "timestamp" ^^^ TimestampType
-
- protected lazy val fixedDecimalType: Parser[DataType] =
- "decimal" ~> "(" ~> "^[1-9]\\d*".r ~ ("," ~> "^[0-9]\\d*".r <~ ")") ^^ {
- case precision ~ scale =>
- DecimalType(precision.toInt, scale.toInt)
- }
-
- protected lazy val arrayType: Parser[DataType] =
- "array" ~> "<" ~> dataType <~ ">" ^^ {
- case tpe => ArrayType(tpe)
- }
-
- protected lazy val mapType: Parser[DataType] =
- "map" ~> "<" ~> dataType ~ "," ~ dataType <~ ">" ^^ {
- case t1 ~ _ ~ t2 => MapType(t1, t2)
- }
-
- protected lazy val structField: Parser[StructField] =
- "[a-zA-Z0-9_]*".r ~ ":" ~ dataType ^^ {
- case name ~ _ ~ tpe => StructField(name, tpe, nullable = true)
- }
-
- protected lazy val structType: Parser[DataType] =
- "struct" ~> "<" ~> repsep(structField, ",") <~ ">" ^^ {
- case fields => StructType(fields)
- }
-
- protected lazy val dataType: Parser[DataType] =
- arrayType |
- mapType |
- structType |
- primitiveType
-
- def toDataType(metastoreType: String): DataType = {
- parseAll(dataType, metastoreType) match {
- case Success(result, _) => result
- case failure: NoSuccess => sys.error(s"Unsupported dataType: $metastoreType")
- }
- }
-
- def toMetastoreType(dt: DataType): String = {
- dt match {
- case ArrayType(elementType, _) => s"array<${ toMetastoreType(elementType) }>"
- case StructType(fields) =>
- s"struct<${
- fields.map(f => s"${ f.name }:${ toMetastoreType(f.dataType) }")
- .mkString(",")
- }>"
- case StringType => "string"
- case FloatType => "float"
- case IntegerType => "int"
- case ShortType => "tinyint"
- case DoubleType => "double"
- case LongType => "bigint"
- case BinaryType => "binary"
- case BooleanType => "boolean"
- case DecimalType() => "decimal"
- case TimestampType => "timestamp"
- case DateType => "date"
- }
- }
-}
-
-
-/**
- * Represents logical plan for one carbon table
- */
-case class CarbonRelation(
- databaseName: String,
- tableName: String,
- var metaData: CarbonMetaData,
- tableMeta: TableMeta,
- alias: Option[String])
- extends LeafNode with MultiInstanceRelation {
-
- def recursiveMethod(dimName: String, childDim: CarbonDimension): String = {
- childDim.getDataType.toString.toLowerCase match {
- case "array" => s"${
- childDim.getColName.substring(dimName.length + 1)
- }:array<${ getArrayChildren(childDim.getColName) }>"
- case "struct" => s"${
- childDim.getColName.substring(dimName.length + 1)
- }:struct<${ getStructChildren(childDim.getColName) }>"
- case dType => s"${ childDim.getColName.substring(dimName.length + 1) }:${ dType }"
- }
- }
-
- def getArrayChildren(dimName: String): String = {
- metaData.carbonTable.getChildren(dimName).asScala.map(childDim => {
- childDim.getDataType.toString.toLowerCase match {
- case "array" => s"array<${ getArrayChildren(childDim.getColName) }>"
- case "struct" => s"struct<${ getStructChildren(childDim.getColName) }>"
- case dType => addDecimalScaleAndPrecision(childDim, dType)
- }
- }).mkString(",")
- }
-
- def getStructChildren(dimName: String): String = {
- metaData.carbonTable.getChildren(dimName).asScala.map(childDim => {
- childDim.getDataType.toString.toLowerCase match {
- case "array" => s"${
- childDim.getColName.substring(dimName.length + 1)
- }:array<${ getArrayChildren(childDim.getColName) }>"
- case "struct" => s"${
- childDim.getColName.substring(dimName.length + 1)
- }:struct<${ metaData.carbonTable.getChildren(childDim.getColName)
- .asScala.map(f => s"${ recursiveMethod(childDim.getColName, f) }").mkString(",")
- }>"
- case dType => s"${ childDim.getColName
- .substring(dimName.length() + 1) }:${ addDecimalScaleAndPrecision(childDim, dType) }"
- }
- }).mkString(",")
- }
-
- override def newInstance(): LogicalPlan = {
- CarbonRelation(databaseName, tableName, metaData, tableMeta, alias)
- .asInstanceOf[this.type]
- }
-
- val dimensionsAttr = {
- val sett = new LinkedHashSet(
- tableMeta.carbonTable.getDimensionByTableName(tableMeta.carbonTableIdentifier.getTableName)
- .asScala.asJava)
- sett.asScala.toSeq.map(dim => {
- val dimval = metaData.carbonTable
- .getDimensionByName(metaData.carbonTable.getFactTableName, dim.getColName)
- val output: DataType = dimval.getDataType
- .toString.toLowerCase match {
- case "array" =>
- CarbonMetastoreTypes.toDataType(s"array<${ getArrayChildren(dim.getColName) }>")
- case "struct" =>
- CarbonMetastoreTypes.toDataType(s"struct<${ getStructChildren(dim.getColName) }>")
- case dType =>
- val dataType = addDecimalScaleAndPrecision(dimval, dType)
- CarbonMetastoreTypes.toDataType(dataType)
- }
-
- AttributeReference(
- dim.getColName,
- output,
- nullable = true)()
- })
- }
-
- val measureAttr = {
- val factTable = tableMeta.carbonTable.getFactTableName
- new LinkedHashSet(
- tableMeta.carbonTable.
- getMeasureByTableName(tableMeta.carbonTable.getFactTableName).
- asScala.asJava).asScala.toSeq
- .map(x => AttributeReference(x.getColName, CarbonMetastoreTypes.toDataType(
- metaData.carbonTable.getMeasureByName(factTable, x.getColName).getDataType.toString
- .toLowerCase match {
- case "decimal" => "decimal(" + x.getPrecision + "," + x.getScale + ")"
- case others => others
- }),
- nullable = true)())
- }
-
- override val output = {
- val columns = tableMeta.carbonTable.getCreateOrderColumn(tableMeta.carbonTable.getFactTableName)
- .asScala
- // convert each column to Attribute
- columns.filter(!_.isInvisible).map { column =>
- if (column.isDimension()) {
- val output: DataType = column.getDataType.toString.toLowerCase match {
- case "array" =>
- CarbonMetastoreTypes.toDataType(s"array<${getArrayChildren(column.getColName)}>")
- case "struct" =>
- CarbonMetastoreTypes.toDataType(s"struct<${getStructChildren(column.getColName)}>")
- case dType =>
- val dataType = addDecimalScaleAndPrecision(column, dType)
- CarbonMetastoreTypes.toDataType(dataType)
- }
- AttributeReference(column.getColName, output, nullable = true )(
- qualifier = Option(tableName + "." + column.getColName))
- } else {
- val output = CarbonMetastoreTypes.toDataType {
- column.getDataType.toString
- .toLowerCase match {
- case "decimal" => "decimal(" + column.getColumnSchema.getPrecision + "," + column
- .getColumnSchema.getScale + ")"
- case others => others
- }
- }
- AttributeReference(column.getColName, output, nullable = true)(
- qualifier = Option(tableName + "." + column.getColName))
- }
- }
- }
-
- def addDecimalScaleAndPrecision(dimval: CarbonColumn, dataType: String): String = {
- var dType = dataType
- if (dimval.getDataType == DECIMAL) {
- dType +=
- "(" + dimval.getColumnSchema.getPrecision + "," + dimval.getColumnSchema.getScale + ")"
- }
- dType
- }
-
- // TODO: Use data from the footers.
- override lazy val statistics = Statistics(sizeInBytes = this.sizeInBytes)
-
- override def equals(other: Any): Boolean = {
- other match {
- case p: CarbonRelation =>
- p.databaseName == databaseName && p.output == output && p.tableName == tableName
- case _ => false
- }
- }
-
- def addDecimalScaleAndPrecision(dimval: CarbonDimension, dataType: String): String = {
- var dType = dataType
- if (dimval.getDataType == DECIMAL) {
- dType +=
- "(" + dimval.getColumnSchema.getPrecision + "," + dimval.getColumnSchema.getScale + ")"
- }
- dType
- }
-
- private var tableStatusLastUpdateTime = 0L
-
- private var sizeInBytesLocalValue = 0L
-
- def sizeInBytes: Long = {
- val tableStatusNewLastUpdatedTime = SegmentStatusManager.getTableStatusLastModifiedTime(
- tableMeta.carbonTable.getAbsoluteTableIdentifier)
-
- if (tableStatusLastUpdateTime != tableStatusNewLastUpdatedTime) {
- val tablePath = CarbonStorePath.getCarbonTablePath(
- tableMeta.storePath,
- tableMeta.carbonTableIdentifier).getPath
- val fileType = FileFactory.getFileType(tablePath)
- if(FileFactory.isFileExist(tablePath, fileType)) {
- tableStatusLastUpdateTime = tableStatusNewLastUpdatedTime
- sizeInBytesLocalValue = FileFactory.getDirectorySize(tablePath)
- }
- }
- sizeInBytesLocalValue
- }
-
-}
[10/15] carbondata git commit: [CARBONDATA-1306] Fixed carbondata
crash after using short-int datatype
Posted by ra...@apache.org.
[CARBONDATA-1306] Fixed carbondata crash after using short-int datatype
This closes #1171
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5b49afcf
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5b49afcf
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5b49afcf
Branch: refs/heads/datamap
Commit: 5b49afcfa67a1b3c822a2182e0fdbed813c5c2d0
Parents: c7f0b10
Author: Ravindra Pesala <ra...@gmail.com>
Authored: Thu Jul 13 16:42:08 2017 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Thu Jul 13 19:34:43 2017 +0800
----------------------------------------------------------------------
.../carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java | 2 +-
.../org/apache/carbondata/core/metadata/datatype/DataType.java | 3 +++
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5b49afcf/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
index 2382599..3df91a3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
@@ -367,7 +367,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
public byte[] compress(Compressor compressor) throws MemoryException, IOException {
if (UnsafeMemoryManager.isOffHeap()) {
// use raw compression and copy to byte[]
- int inputSize = pageSize << dataType.getSizeBits();
+ int inputSize = pageSize * dataType.getSizeInBytes();
int compressedMaxSize = compressor.maxCompressedLength(inputSize);
MemoryBlock compressed = UnsafeMemoryManager.allocateMemoryWithRetry(compressedMaxSize);
long outSize = compressor.rawCompress(baseOffset, inputSize, compressed.getBaseOffset());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5b49afcf/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
index 3a7f75d..e97cce0 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
@@ -69,6 +69,9 @@ public enum DataType {
}
public int getSizeBits() {
+ if (this == SHORT_INT) {
+ throw new UnsupportedOperationException("Should not call this from datatype " + SHORT_INT);
+ }
return (int) (Math.log(getSizeInBytes()) / Math.log(2));
}