You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/11/08 04:15:46 UTC
[12/24] carbondata git commit: [CARBONDATA-1594] Add precision and
scale to DecimalType
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
index 7a80c88..87e7dea 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -113,9 +113,13 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
DataType msrType = getMeasureDataType(msrColumnEvaluatorInfo);
for (int i = 0; i < ColumnPages.length; i++) {
BitSet bitSet =
- getFilteredIndexesForMeasure(measureRawColumnChunk.convertToColumnPage(i),
- measureRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
- blockChunkHolder.getBitSetGroup(), i, msrType);;
+ getFilteredIndexesForMeasure(
+ measureRawColumnChunk.convertToColumnPage(i),
+ measureRawColumnChunk.getRowCount()[i],
+ useBitsetPipeLine,
+ blockChunkHolder.getBitSetGroup(),
+ i,
+ msrType);
bitSetGroup.setBitSet(bitSet, i);
}
return bitSetGroup;
@@ -132,15 +136,14 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
return DataTypes.INT;
} else if (msrColumnEvaluatorInfo.getType() == DataTypes.LONG) {
return DataTypes.LONG;
- } else if (msrColumnEvaluatorInfo.getType() == DataTypes.DECIMAL) {
- return DataTypes.DECIMAL;
+ } else if (DataTypes.isDecimal(msrColumnEvaluatorInfo.getType())) {
+ return DataTypes.createDefaultDecimalType();
} else {
return DataTypes.DOUBLE;
}
}
- protected BitSet getFilteredIndexes(ColumnPage columnPage,
- int numerOfRows, DataType msrType) {
+ private BitSet getFilteredIndexes(ColumnPage columnPage, int numerOfRows, DataType msrType) {
// Here the algorithm is
// Get the measure values from the chunk. compare sequentially with the
// the filter values. The one that matches sets it Bitset.
@@ -311,6 +314,7 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
}
return bitSet;
}
+
private BitSet setFilterdIndexToBitSetWithColumnIndex(
DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) {
BitSet bitSet = new BitSet(numerOfRows);
@@ -383,14 +387,14 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
}
@Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
- if (isDimensionPresentInCurrentBlock == true) {
+ if (isDimensionPresentInCurrentBlock) {
int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
.get(dimColEvaluatorInfo.getColumnIndex());
if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
.getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
}
- } else if (isMeasurePresentInCurrentBlock == true) {
+ } else if (isMeasurePresentInCurrentBlock) {
int blockIndex = segmentProperties.getMeasuresOrdinalToBlockMapping()
.get(msrColumnEvaluatorInfo.getColumnIndex());
if (null == blockChunkHolder.getMeasureRawDataChunk()[blockIndex]) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
index 843da71..0022a72 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -156,8 +156,8 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
return DataTypes.INT;
} else if (msrColumnEvaluatorInfo.getType() == DataTypes.LONG) {
return DataTypes.LONG;
- } else if (msrColumnEvaluatorInfo.getType() == DataTypes.DECIMAL) {
- return DataTypes.DECIMAL;
+ } else if (DataTypes.isDecimal(msrColumnEvaluatorInfo.getType())) {
+ return DataTypes.createDefaultDecimalType();
} else {
return DataTypes.DOUBLE;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index dbf8d4c..10664c8 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -359,8 +359,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
msrType = DataTypes.INT;
} else if (dataType == DataTypes.LONG) {
msrType = DataTypes.LONG;
- } else if (dataType == DataTypes.DECIMAL) {
- msrType = DataTypes.DECIMAL;
+ } else if (DataTypes.isDecimal(dataType)) {
+ msrType = DataTypes.createDefaultDecimalType();
} else {
msrType = DataTypes.DOUBLE;
}
@@ -386,7 +386,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
msrValue = (int) columnPage.getLong(index);
} else if (msrType == DataTypes.LONG) {
msrValue = columnPage.getLong(index);
- } else if (msrType == DataTypes.DECIMAL) {
+ } else if (DataTypes.isDecimal(msrType)) {
BigDecimal bigDecimalValue = columnPage.getDecimal(index);
if (null != bigDecimalValue
&& msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema().getScale()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/main/java/org/apache/carbondata/core/scan/partition/PartitionUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/partition/PartitionUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/partition/PartitionUtil.java
index 85e6cef..676cf48 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/partition/PartitionUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/partition/PartitionUtil.java
@@ -66,7 +66,7 @@ public class PartitionUtil {
return dateFormatter.parse(data).getTime();
} else if (actualDataType == DataTypes.TIMESTAMP) {
return timestampFormatter.parse(data).getTime();
- } else if (actualDataType == DataTypes.DECIMAL) {
+ } else if (DataTypes.isDecimal(actualDataType)) {
return new BigDecimal(data);
} else {
return data;
@@ -102,7 +102,7 @@ public class PartitionUtil {
return Long.parseLong(data);
} else if (actualDataType == DataTypes.DATE || actualDataType == DataTypes.TIMESTAMP) {
return Long.parseLong(data) / 1000;
- } else if (actualDataType == DataTypes.DECIMAL) {
+ } else if (DataTypes.isDecimal(actualDataType)) {
return new BigDecimal(data);
} else {
return data;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
index cf6c88b..5916fd3 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
@@ -372,7 +372,7 @@ public class MeasureDataVectorProcessor {
return new IntegralMeasureVectorFiller();
} else if (dataType == DataTypes.LONG) {
return new LongMeasureVectorFiller();
- } else if (dataType == DataTypes.DECIMAL) {
+ } else if (DataTypes.isDecimal(dataType)) {
return new DecimalMeasureVectorFiller();
} else {
return new DefaultMeasureVectorFiller();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index cba9931..aca2fd6 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -37,6 +37,7 @@ import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.datatype.DecimalType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.reader.CarbonIndexFileReader;
@@ -262,8 +263,13 @@ public abstract class AbstractDataFileFooterConverter {
wrapperColumnSchema.setColumnUniqueId(externalColumnSchema.getColumn_id());
wrapperColumnSchema.setColumnName(externalColumnSchema.getColumn_name());
wrapperColumnSchema.setColumnar(externalColumnSchema.isColumnar());
- wrapperColumnSchema
- .setDataType(thriftDataTyopeToWrapperDataType(externalColumnSchema.data_type));
+ DataType dataType = thriftDataTyopeToWrapperDataType(externalColumnSchema.data_type);
+ if (DataTypes.isDecimal(dataType)) {
+ DecimalType decimalType = (DecimalType) dataType;
+ decimalType.setPrecision(externalColumnSchema.getPrecision());
+ decimalType.setScale(externalColumnSchema.getScale());
+ }
+ wrapperColumnSchema.setDataType(dataType);
wrapperColumnSchema.setDimensionColumn(externalColumnSchema.isDimension());
List<Encoding> encoders = new ArrayList<Encoding>();
for (org.apache.carbondata.format.Encoding encoder : externalColumnSchema.getEncoders()) {
@@ -369,7 +375,7 @@ public abstract class AbstractDataFileFooterConverter {
case DOUBLE:
return DataTypes.DOUBLE;
case DECIMAL:
- return DataTypes.DECIMAL;
+ return DataTypes.createDefaultDecimalType();
case DATE:
return DataTypes.DATE;
case TIMESTAMP:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index d232d61..6579a6f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -363,7 +363,7 @@ public class CarbonMetadataUtil {
firstBuffer.flip();
secondBuffer.flip();
return (int) (firstBuffer.getLong() - secondBuffer.getLong());
- } else if (dataType == DataTypes.DECIMAL) {
+ } else if (DataTypes.isDecimal(dataType)) {
return DataTypeUtil.byteToBigDecimal(first).compareTo(DataTypeUtil.byteToBigDecimal(second));
} else {
throw new IllegalArgumentException("Invalid data type");
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 77789c5..17a4b5f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -1848,6 +1848,7 @@ public final class CarbonUtil {
return map;
}
+ // TODO: move this to carbon store API as it is related to TableInfo creation
public static TableInfo convertGsonToTableInfo(Map<String, String> properties) {
Gson gson = new Gson();
String partsNo = properties.get("carbonSchemaPartsNo");
@@ -1864,9 +1865,35 @@ public final class CarbonUtil {
builder.append(part);
}
TableInfo tableInfo = gson.fromJson(builder.toString(), TableInfo.class);
+
+ // The tableInfo is deserialized from GSON string, need to update the scale and
+ // precision if there are any decimal field, because DecimalType is added in Carbon 1.3,
+ // If it is not updated, read compactibility will be break for table generated before Carbon 1.3
+ updateDecimalType(tableInfo);
return tableInfo;
}
+ // Update decimal type inside `tableInfo` to set scale and precision, if there are any decimal
+ private static void updateDecimalType(TableInfo tableInfo) {
+ List<ColumnSchema> deserializedColumns = tableInfo.getFactTable().getListOfColumns();
+ for (ColumnSchema column : deserializedColumns) {
+ DataType dataType = column.getDataType();
+ if (DataTypes.isDecimal(dataType)) {
+ column.setDataType(DataTypes.createDecimalType(column.getPrecision(), column.getScale()));
+ }
+ }
+ if (tableInfo.getFactTable().getPartitionInfo() != null) {
+ List<ColumnSchema> partitionColumns =
+ tableInfo.getFactTable().getPartitionInfo().getColumnSchemaList();
+ for (ColumnSchema column : partitionColumns) {
+ DataType dataType = column.getDataType();
+ if (DataTypes.isDecimal(dataType)) {
+ column.setDataType(DataTypes.createDecimalType(column.getPrecision(), column.getScale()));
+ }
+ }
+ }
+ }
+
/**
* Removes schema from properties
* @param properties
@@ -1964,7 +1991,7 @@ public final class CarbonUtil {
b.putDouble((double) value);
b.flip();
return b.array();
- } else if (dataType == DataTypes.DECIMAL) {
+ } else if (DataTypes.isDecimal(dataType)) {
return DataTypeUtil.bigDecimalToByte((BigDecimal) value);
} else if (dataType == DataTypes.BYTE_ARRAY) {
return (byte[]) value;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 01e34a7..0961a63 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -27,8 +27,6 @@ import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
-import java.util.HashMap;
-import java.util.Map;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
@@ -51,7 +49,6 @@ public final class DataTypeUtil {
*/
private static final LogService LOGGER =
LogServiceFactory.getLogService(DataTypeUtil.class.getName());
- private static final Map<String, String> dataTypeDisplayNames;
private static final ThreadLocal<DateFormat> timeStampformatter = new ThreadLocal<DateFormat>() {
@Override protected DateFormat initialValue() {
@@ -71,23 +68,6 @@ public final class DataTypeUtil {
}
};
- static {
- dataTypeDisplayNames = new HashMap<String, String>(16);
- dataTypeDisplayNames.put(DataTypes.DATE.toString(), DataTypes.DATE.getName());
- dataTypeDisplayNames.put(DataTypes.LONG.toString(), DataTypes.LONG.getName());
- dataTypeDisplayNames.put(DataTypes.INT.toString(), DataTypes.INT.getName());
- dataTypeDisplayNames.put(DataTypes.FLOAT.toString(), DataTypes.FLOAT.getName());
- dataTypeDisplayNames.put(DataTypes.BOOLEAN.toString(), DataTypes.BOOLEAN.getName());
- dataTypeDisplayNames.put(DataTypes.NULL.toString(), DataTypes.NULL.getName());
- dataTypeDisplayNames.put(DataTypes.DECIMAL.toString(), DataTypes.DECIMAL.getName());
- dataTypeDisplayNames.put(DataTypes.ARRAY.toString(), DataTypes.ARRAY.getName());
- dataTypeDisplayNames.put(DataTypes.STRUCT.toString(), DataTypes.STRUCT.getName());
- dataTypeDisplayNames.put(DataTypes.TIMESTAMP.toString(), DataTypes.TIMESTAMP.getName());
- dataTypeDisplayNames.put(DataTypes.DATE.toString(), DataTypes.DATE.getName());
- dataTypeDisplayNames.put(DataTypes.SHORT.toString(), DataTypes.SHORT.getName());
- dataTypeDisplayNames.put(DataTypes.STRING.toString(), DataTypes.STRING.getName());
- }
-
/**
* DataType converter for different computing engines
*/
@@ -105,7 +85,7 @@ public final class DataTypeUtil {
CarbonMeasure carbonMeasure) {
if (dataType == DataTypes.BOOLEAN) {
return BooleanConvert.parseBoolean(msrValue);
- } else if (dataType == DataTypes.DECIMAL) {
+ } else if (DataTypes.isDecimal(dataType)) {
BigDecimal bigDecimal =
new BigDecimal(msrValue).setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP);
return normalizeDecimalValue(bigDecimal, carbonMeasure.getPrecision());
@@ -137,7 +117,7 @@ public final class DataTypeUtil {
return (int) bb.getLong();
} else if (dataType == DataTypes.LONG) {
return bb.getLong();
- } else if (dataType == DataTypes.DECIMAL) {
+ } else if (DataTypes.isDecimal(dataType)) {
return byteToBigDecimal(data);
} else {
return bb.getDouble();
@@ -154,7 +134,7 @@ public final class DataTypeUtil {
return (int) measurePage.getLong(index);
} else if (dataType == DataTypes.LONG) {
return measurePage.getLong(index);
- } else if (dataType == DataTypes.DECIMAL) {
+ } else if (DataTypes.isDecimal(dataType)) {
BigDecimal bigDecimalMsrValue = measurePage.getDecimal(index);
if (null != bigDecimalMsrValue && carbonMeasure.getScale() > bigDecimalMsrValue.scale()) {
bigDecimalMsrValue =
@@ -163,22 +143,13 @@ public final class DataTypeUtil {
if (null != bigDecimalMsrValue) {
return normalizeDecimalValue(bigDecimalMsrValue, carbonMeasure.getPrecision());
} else {
- return bigDecimalMsrValue;
+ return null;
}
} else {
return measurePage.getDouble(index);
}
}
-
- /**
- * @param dataType
- * @return
- */
- public static String getColumnDataTypeDisplayName(String dataType) {
- return dataTypeDisplayNames.get(dataType);
- }
-
/**
* This method will check the digits before dot with the max precision allowed
*
@@ -239,55 +210,6 @@ public final class DataTypeUtil {
}
/**
- * returns the SqlStatement.Type of corresponding string value
- *
- * @param dataTypeStr
- * @return return the SqlStatement.Type
- */
- public static DataType getDataType(String dataTypeStr) {
- DataType dataType = null;
- switch (dataTypeStr) {
- case "BOOLEAN":
- dataType = DataTypes.BOOLEAN;
- break;
- case "DATE":
- dataType = DataTypes.DATE;
- break;
- case "TIMESTAMP":
- dataType = DataTypes.TIMESTAMP;
- break;
- case "STRING":
- dataType = DataTypes.STRING;
- break;
- case "INT":
- dataType = DataTypes.INT;
- break;
- case "SMALLINT":
- dataType = DataTypes.SHORT;
- break;
- case "LONG":
- dataType = DataTypes.LONG;
- break;
- case "DOUBLE":
- dataType = DataTypes.DOUBLE;
- break;
- case "DECIMAL":
- dataType = DataTypes.DECIMAL;
- break;
- case "ARRAY":
- dataType = DataTypes.ARRAY;
- break;
- case "STRUCT":
- dataType = DataTypes.STRUCT;
- break;
- case "MAP":
- default:
- dataType = DataTypes.STRING;
- }
- return dataType;
- }
-
- /**
* Below method will be used to convert the data passed to its actual data
* type
*
@@ -365,7 +287,7 @@ public final class DataTypeUtil {
LOGGER.error("Cannot convert value to Time/Long type value" + e.getMessage());
return null;
}
- } else if (actualDataType == DataTypes.DECIMAL) {
+ } else if (DataTypes.isDecimal(actualDataType)) {
if (data.isEmpty()) {
return null;
}
@@ -518,7 +440,7 @@ public final class DataTypeUtil {
LOGGER.error("Cannot convert value to Time/Long type value" + e.getMessage());
return null;
}
- } else if (dataType == DataTypes.DECIMAL) {
+ } else if (DataTypes.isDecimal(dataType)) {
String data7 = new String(dataInBytes, CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
if (data7.isEmpty()) {
return null;
@@ -585,7 +507,7 @@ public final class DataTypeUtil {
// validation will not be done for timestamp datatype as for timestamp direct dictionary
// is generated. No dictionary file is created for timestamp datatype column
DataType dataType = dimension.getDataType();
- if (dataType == DataTypes.DECIMAL) {
+ if (DataTypes.isDecimal(dataType)) {
return parseStringToBigDecimal(value, dimension);
} else if (dataType == DataTypes.SHORT || dataType == DataTypes.INT ||
dataType == DataTypes.LONG) {
@@ -618,7 +540,7 @@ public final class DataTypeUtil {
}
try {
DataType dataType = dimension.getDataType();
- if (dataType == DataTypes.DECIMAL) {
+ if (DataTypes.isDecimal(dataType)) {
return parseStringToBigDecimal(value, dimension);
} else if (dataType == DataTypes.INT) {
Integer.parseInt(value);
@@ -721,7 +643,7 @@ public final class DataTypeUtil {
return null;
}
}
- } else if (dataType == DataTypes.DECIMAL) {
+ } else if (DataTypes.isDecimal(dataType)) {
String parsedValue = parseStringToBigDecimal(data, columnSchema);
if (null == parsedValue) {
return null;
@@ -749,7 +671,7 @@ public final class DataTypeUtil {
try {
Object parsedValue = null;
DataType dataType = columnSchema.getDataType();
- if (dataType == DataTypes.DECIMAL) {
+ if (DataTypes.isDecimal(dataType)) {
return parseStringToBigDecimal(value, columnSchema);
} else if (dataType == DataTypes.SHORT || dataType == DataTypes.INT ||
dataType == DataTypes.LONG) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java b/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java
index d1beb16..c162396 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java
@@ -37,7 +37,7 @@ public final class Comparator {
} else if (dataType == DataTypes.LONG || dataType == DataTypes.DATE
|| dataType == DataTypes.TIMESTAMP) {
return new LongSerializableComparator();
- } else if (dataType == DataTypes.DECIMAL) {
+ } else if (DataTypes.isDecimal(dataType)) {
return new BigDecimalSerializableComparator();
} else {
return new ByteArraySerializableComparator();
@@ -61,7 +61,7 @@ public final class Comparator {
return new LongSerializableComparator();
} else if (dataType == DataTypes.DOUBLE) {
return new DoubleSerializableComparator();
- } else if (dataType == DataTypes.DECIMAL) {
+ } else if (DataTypes.isDecimal(dataType)) {
return new BigDecimalSerializableComparator();
} else {
throw new IllegalArgumentException("Unsupported data type");
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java
index 4396d6a..5756c0d 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java
@@ -82,7 +82,7 @@ public class CarbonDictionarySortModel implements Comparable<CarbonDictionarySor
return -1;
}
return d1.compareTo(d2);
- } else if (dataType == DataTypes.DECIMAL) {
+ } else if (DataTypes.isDecimal(dataType)) {
java.math.BigDecimal val1 = null;
java.math.BigDecimal val2 = null;
try {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfoTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfoTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfoTest.java
index 33121e2..cb0892f 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfoTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfoTest.java
@@ -25,6 +25,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.datatype.DecimalType;
import org.apache.carbondata.core.util.CarbonUtil;
import mockit.Mock;
@@ -244,7 +245,7 @@ public class ColumnDictionaryInfoTest {
}
@Test public void testGetIncrementalSurrogateKeyFromDictionaryWithDecimalType() {
- columnDictionaryInfo = new ColumnDictionaryInfo(DataTypes.DECIMAL);
+ columnDictionaryInfo = new ColumnDictionaryInfo(DataTypes.createDefaultDecimalType());
List<String> evaluateResultList = Arrays.asList("150011.550");
List<byte[]> byteValuesOfFilterMembers = convertListElementsIntoByteArray(evaluateResultList);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/RLECodecSuite.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/RLECodecSuite.java b/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/RLECodecSuite.java
index be898c2..8be5f9d 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/RLECodecSuite.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/page/encoding/RLECodecSuite.java
@@ -46,9 +46,9 @@ public class RLECodecSuite {
TestData(byte[] inputByteData, byte[] expectedEncodedByteData) throws IOException, MemoryException {
this.inputByteData = inputByteData;
inputBytePage = ColumnPage.newPage(
- new TableSpec.ColumnSpec("test", DataTypes.BYTE, ColumnType.MEASURE),
+ TableSpec.ColumnSpec.newInstance("test", DataTypes.BYTE, ColumnType.MEASURE),
DataTypes.BYTE, inputByteData.length);
- inputBytePage.setStatsCollector(PrimitivePageStatsCollector.newInstance(DataTypes.BYTE, 0, 0));
+ inputBytePage.setStatsCollector(PrimitivePageStatsCollector.newInstance(DataTypes.BYTE));
for (int i = 0; i < inputByteData.length; i++) {
inputBytePage.putData(i, inputByteData[i]);
}
@@ -131,7 +131,7 @@ public class RLECodecSuite {
private void testBytePageDecode(byte[] inputBytes, byte[] expectedDecodedBytes) throws IOException, MemoryException {
RLECodec codec = new RLECodec();
RLEEncoderMeta meta = new RLEEncoderMeta(
- new TableSpec.ColumnSpec("test", DataTypes.BYTE, ColumnType.MEASURE),
+ TableSpec.ColumnSpec.newInstance("test", DataTypes.BYTE, ColumnType.MEASURE),
DataTypes.BYTE, expectedDecodedBytes.length, null);
ColumnPageDecoder decoder = codec.createDecoder(meta);
ColumnPage page = decoder.decode(inputBytes, 0, inputBytes.length);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
index 4a3ef32..42c0ad6 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
@@ -23,6 +23,8 @@ import java.util.Map;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.datatype.DecimalType;
+import org.apache.carbondata.core.metadata.datatype.IntType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.SchemaEvolution;
import org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry;
@@ -874,7 +876,7 @@ public class ThriftWrapperSchemaConverterImplTest {
}
@Mock public DataType getDataType() {
- return DataTypes.DECIMAL;
+ return DataTypes.createDefaultDecimalType();
}
@Mock public String getColumnName() {
@@ -1158,7 +1160,12 @@ public class ThriftWrapperSchemaConverterImplTest {
@Test public void testFromWrapperToExternalColumnSchemaForDatatypeNullCase() {
org.apache.carbondata.format.ColumnSchema thriftColumnSchema =
- new org.apache.carbondata.format.ColumnSchema(null, "columnName", "1", true, encoders,
+ new org.apache.carbondata.format.ColumnSchema(
+ org.apache.carbondata.format.DataType.INT,
+ "columnName",
+ "1",
+ true,
+ encoders,
true);
thriftColumnSchema.setSchemaOrdinal(1);
@@ -1172,7 +1179,7 @@ public class ThriftWrapperSchemaConverterImplTest {
}
@Mock public DataType getDataType() {
- return null;
+ return DataTypes.INT;
}
@Mock public String getColumnName() {
@@ -1548,9 +1555,7 @@ long time =1112745600000L;
wrapperColumnSchema.setDimensionColumn(true);
wrapperColumnSchema.setEncodingList(encodings);
wrapperColumnSchema.setNumberOfChild(1);
- wrapperColumnSchema.setPrecision(1);
wrapperColumnSchema.setColumnGroup(1);
- wrapperColumnSchema.setScale(1);
wrapperColumnSchema.setDefaultValue(new byte[] { 1, 2 });
wrapperColumnSchema.setColumnProperties(columnPropertyMap);
wrapperColumnSchema.setInvisible(true);
@@ -1686,8 +1691,9 @@ long time =1112745600000L;
@Test public void testFromExternalToWrapperColumnSchemaForDatatypeNullCase() {
org.apache.carbondata.format.ColumnSchema thriftColumnSchema =
- new org.apache.carbondata.format.ColumnSchema(null, "columnName", "1", true, encoders,
- true);
+ new org.apache.carbondata.format.ColumnSchema(
+ org.apache.carbondata.format.DataType.STRING,
+ "columnName", "1", true, encoders, true);
ColumnSchema actualResult =
thriftWrapperSchemaConverter.fromExternalToWrapperColumnSchema(thriftColumnSchema);
Boolean expectedResult = true;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
index 048b194..172a53e 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
@@ -45,22 +45,27 @@ public class RestructureUtilTest {
encodingList.add(Encoding.DICTIONARY);
ColumnSchema columnSchema1 = new ColumnSchema();
columnSchema1.setColumnName("Id");
+ columnSchema1.setDataType(DataTypes.STRING);
columnSchema1.setColumnUniqueId(UUID.randomUUID().toString());
columnSchema1.setEncodingList(encodingList);
ColumnSchema columnSchema2 = new ColumnSchema();
columnSchema2.setColumnName("Name");
+ columnSchema2.setDataType(DataTypes.STRING);
columnSchema2.setColumnUniqueId(UUID.randomUUID().toString());
columnSchema2.setEncodingList(encodingList);
ColumnSchema columnSchema3 = new ColumnSchema();
columnSchema3.setColumnName("Age");
+ columnSchema3.setDataType(DataTypes.INT);
columnSchema3.setColumnUniqueId(UUID.randomUUID().toString());
columnSchema3.setEncodingList(encodingList);
ColumnSchema columnSchema4 = new ColumnSchema();
columnSchema4.setColumnName("Salary");
+ columnSchema4.setDataType(DataTypes.INT);
columnSchema4.setColumnUniqueId(UUID.randomUUID().toString());
columnSchema4.setEncodingList(encodingList);
ColumnSchema columnSchema5 = new ColumnSchema();
columnSchema5.setColumnName("Address");
+ columnSchema5.setDataType(DataTypes.STRING);
columnSchema5.setColumnUniqueId(UUID.randomUUID().toString());
columnSchema5.setEncodingList(encodingList);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/scan/expression/ExpressionResultTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/ExpressionResultTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/ExpressionResultTest.java
index cdc7fb9..1be4c2e 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/ExpressionResultTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/ExpressionResultTest.java
@@ -26,6 +26,7 @@ import java.util.List;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.datatype.DecimalType;
import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
import org.junit.BeforeClass;
@@ -255,7 +256,7 @@ public class ExpressionResultTest {
}
@Test public void testGetDecimalForDecimal() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForDecimal = new ExpressionResult(DataTypes.DECIMAL, 5);
+ ExpressionResult expressionResultForDecimal = new ExpressionResult(DataTypes.createDefaultDecimalType(), 5);
BigDecimal actualValue = expressionResultForDecimal.getDecimal();
BigDecimal expectedValue = new BigDecimal(5);
assertEquals(expectedValue, actualValue);
@@ -403,8 +404,8 @@ public class ExpressionResultTest {
}
@Test public void testEqualsForDecimal() {
- ExpressionResult expressionResult = new ExpressionResult(DataTypes.DECIMAL, 5);
- ExpressionResult objToCompare = new ExpressionResult(DataTypes.DECIMAL, 6);
+ ExpressionResult expressionResult = new ExpressionResult(DataTypes.createDefaultDecimalType(), 5);
+ ExpressionResult objToCompare = new ExpressionResult(DataTypes.createDefaultDecimalType(), 6);
boolean result = expressionResult.equals(objToCompare);
assertFalse(result);
}
@@ -443,8 +444,8 @@ public class ExpressionResultTest {
}
@Test public void compareToForDecimal() {
- ExpressionResult obj = new ExpressionResult(DataTypes.DECIMAL, 5);
- ExpressionResult expressionResult = new ExpressionResult(DataTypes.DECIMAL, 6);
+ ExpressionResult obj = new ExpressionResult(DataTypes.createDefaultDecimalType(), 5);
+ ExpressionResult expressionResult = new ExpressionResult(DataTypes.createDefaultDecimalType(), 6);
int actualValue = expressionResult.compareTo(obj);
int expectedValue = 1;
assertEquals(expectedValue, actualValue);
@@ -452,7 +453,7 @@ public class ExpressionResultTest {
@Test public void compareToForException() {
ExpressionResult obj = new ExpressionResult(DataTypes.INT, 5);
- ExpressionResult expressionResult = new ExpressionResult(DataTypes.DECIMAL, 6);
+ ExpressionResult expressionResult = new ExpressionResult(DataTypes.createDefaultDecimalType(), 6);
int actualValue = expressionResult.compareTo(obj);
int expectedValue = -1;
assertEquals(expectedValue, actualValue);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java
index 51bb0fe..cef1953 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java
@@ -25,6 +25,7 @@ import java.util.Date;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.datatype.DecimalType;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.ExpressionResult;
import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
@@ -294,7 +295,7 @@ public class EqualToExpressionUnitTest {
@Test public void testEvaluateForEqualToExpressionWithDecimalDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("contact", DataTypes.DECIMAL);
+ ColumnExpression right = new ColumnExpression("contact", DataTypes.createDefaultDecimalType());
right.setColIndex(0);
equalToExpression = new EqualToExpression(right, right);
RowImpl value = new RowImpl();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java
index 0375a6d..de45721 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java
@@ -25,6 +25,7 @@ import java.util.Date;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.datatype.DecimalType;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.ExpressionResult;
import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
@@ -205,9 +206,9 @@ public class GreaterThanEqualToExpressionUnitTest {
@Test public void testEvaluateForGreaterThanEqualToExpressionWithDecimalDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("contact", DataTypes.DECIMAL);
+ ColumnExpression right = new ColumnExpression("contact", DataTypes.createDefaultDecimalType());
right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("contact", DataTypes.DECIMAL);
+ ColumnExpression left = new ColumnExpression("contact", DataTypes.createDefaultDecimalType());
left.setColIndex(1);
greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
RowImpl value = new RowImpl();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
index 1940069..ef32f69 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
@@ -25,6 +25,7 @@ import java.util.Date;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.datatype.DecimalType;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.ExpressionResult;
import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
@@ -251,9 +252,9 @@ public class GreaterThanExpressionUnitTest {
@Test public void testEvaluateForGreaterThanExpressionWithDecimalDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("contact", DataTypes.DECIMAL);
+ ColumnExpression right = new ColumnExpression("contact", DataTypes.createDefaultDecimalType());
right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("contact", DataTypes.DECIMAL);
+ ColumnExpression left = new ColumnExpression("contact", DataTypes.createDefaultDecimalType());
left.setColIndex(1);
greaterThanExpression = new GreaterThanExpression(left, right);
RowImpl value = new RowImpl();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java
index d31ec31..aa3c208 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java
@@ -25,6 +25,7 @@ import java.util.Date;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.datatype.DecimalType;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.ExpressionResult;
import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
@@ -203,9 +204,9 @@ public class InExpressionUnitTest {
@Test public void testEvaluateForInExpressionWithDecimalDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("left_contact", DataTypes.DECIMAL);
+ ColumnExpression left = new ColumnExpression("left_contact", DataTypes.createDefaultDecimalType());
left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_contact", DataTypes.DECIMAL);
+ ColumnExpression right = new ColumnExpression("right_contact", DataTypes.createDefaultDecimalType());
right.setColIndex(1);
inExpression = new InExpression(left, right);
RowImpl value = new RowImpl();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java
index a2a5d2a..9b0aa66 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java
@@ -25,6 +25,7 @@ import java.util.Date;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.datatype.DecimalType;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.ExpressionResult;
import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
@@ -254,9 +255,9 @@ public class LessThanEqualToExpressionUnitTest {
@Test public void testEvaluateForLessThanEqualToExpressionWithDecimalDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("right_contact", DataTypes.DECIMAL);
+ ColumnExpression right = new ColumnExpression("right_contact", DataTypes.createDefaultDecimalType());
right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("left_contact", DataTypes.DECIMAL);
+ ColumnExpression left = new ColumnExpression("left_contact", DataTypes.createDefaultDecimalType());
left.setColIndex(1);
lessThanEqualToExpression = new LessThanEqualToExpression(left, right);
RowImpl value = new RowImpl();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java
index d385dc1..4bd02af 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java
@@ -25,6 +25,7 @@ import java.util.Date;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.datatype.DecimalType;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.ExpressionResult;
import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
@@ -251,9 +252,9 @@ public class LessThanExpressionUnitTest {
@Test public void testEvaluateForLessThanExpressionWithDecimalDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("contact", DataTypes.DECIMAL);
+ ColumnExpression right = new ColumnExpression("contact", DataTypes.createDefaultDecimalType());
right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("contact", DataTypes.DECIMAL);
+ ColumnExpression left = new ColumnExpression("contact", DataTypes.createDefaultDecimalType());
left.setColIndex(1);
lessThanExpression = new LessThanExpression(left, right);
RowImpl value = new RowImpl();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java
index e7fa544..5ee6a15 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java
@@ -25,6 +25,7 @@ import java.util.Date;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.datatype.DecimalType;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.ExpressionResult;
import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
@@ -261,9 +262,9 @@ public class NotEqualsExpressionUnitTest {
@Test public void testEvaluateWithForNotEqualsExpressionDecimalDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("contact", DataTypes.DECIMAL);
+ ColumnExpression right = new ColumnExpression("contact", DataTypes.createDefaultDecimalType());
right.setColIndex(1);
- ColumnExpression left = new ColumnExpression("contact", DataTypes.DECIMAL);
+ ColumnExpression left = new ColumnExpression("contact", DataTypes.createDefaultDecimalType());
left.setColIndex(0);
notEqualsExpression = new NotEqualsExpression(left, right);
RowImpl value = new RowImpl();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java
index 5758625..c797ce9 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java
@@ -25,6 +25,7 @@ import java.util.Date;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.datatype.DecimalType;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.ExpressionResult;
import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
@@ -202,9 +203,9 @@ public class NotInExpressionUnitTest {
@Test public void testEvaluateForNotInExpressionWithDecimalDataType()
throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("left_contact", DataTypes.DECIMAL);
+ ColumnExpression left = new ColumnExpression("left_contact", DataTypes.createDefaultDecimalType());
left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_contact", DataTypes.DECIMAL);
+ ColumnExpression right = new ColumnExpression("right_contact", DataTypes.createDefaultDecimalType());
right.setColIndex(1);
notInExpression = new NotInExpression(left, right);
RowImpl value = new RowImpl();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
index a6f483b..f1f05f0 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
@@ -27,6 +27,7 @@ import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.datatype.DecimalType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
@@ -149,7 +150,7 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest {
String dictionaryVal = "1111111";
String memberVal = "1111";
int actualResult =
- FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataTypes.DECIMAL);
+ FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataTypes.createDefaultDecimalType());
int expectedResult = 1;
assertEquals(expectedResult, actualResult);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
index ca78e66..4cf3d11 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
@@ -189,7 +189,7 @@ public class CarbonMetadataUtilTest {
final EncodedColumnPage measure = new EncodedColumnPage(new DataChunk2(), new byte[]{0,1},
PrimitivePageStatsCollector.newInstance(
- org.apache.carbondata.core.metadata.datatype.DataTypes.BYTE, 0, 0));
+ org.apache.carbondata.core.metadata.datatype.DataTypes.BYTE));
new MockUp<EncodedTablePage>() {
@SuppressWarnings("unused") @Mock
public EncodedColumnPage getMeasure(int measureIndex) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index 658dda9..56c59e1 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -40,6 +40,7 @@ import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.datatype.DecimalType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
@@ -508,7 +509,7 @@ public class CarbonUtilTest {
}
@Test public void testForHasDataTypes() {
- DataType[] dataTypes = { DataTypes.DECIMAL, DataTypes.BOOLEAN, DataTypes.INT };
+ DataType[] dataTypes = {DataTypes.createDefaultDecimalType(), DataTypes.BOOLEAN, DataTypes.INT };
assertTrue(CarbonUtil.hasDataType(DataTypes.BOOLEAN, dataTypes));
assertTrue(!CarbonUtil.hasDataType(DataTypes.DATE, dataTypes));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
index b6ea765..c944d18 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/DataTypeUtilTest.java
@@ -21,6 +21,7 @@ import java.math.BigDecimal;
import java.math.BigInteger;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.datatype.DecimalType;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
@@ -30,9 +31,7 @@ import static junit.framework.TestCase.assertEquals;
import static junit.framework.TestCase.assertTrue;
import static org.apache.carbondata.core.util.DataTypeUtil.bigDecimalToByte;
import static org.apache.carbondata.core.util.DataTypeUtil.byteToBigDecimal;
-import static org.apache.carbondata.core.util.DataTypeUtil.getColumnDataTypeDisplayName;
import static org.apache.carbondata.core.util.DataTypeUtil.getDataBasedOnDataType;
-import static org.apache.carbondata.core.util.DataTypeUtil.getDataType;
import static org.apache.carbondata.core.util.DataTypeUtil.getMeasureValueBasedOnDataType;
import static org.apache.carbondata.core.util.DataTypeUtil.normalizeIntAndLongValues;
@@ -40,7 +39,7 @@ public class DataTypeUtilTest {
@Test public void testGetColumnDataTypeDisplayName() {
String expected = DataTypes.INT.getName();
- String result = getColumnDataTypeDisplayName("INT");
+ String result = "INT";
assertEquals(expected, result);
}
@@ -60,22 +59,6 @@ public class DataTypeUtilTest {
assertTrue(result == result);
}
- @Test public void testGetDataType() {
- assertEquals(DataTypes.TIMESTAMP, getDataType("TIMESTAMP"));
- assertEquals(DataTypes.DATE, getDataType("DATE"));
- assertEquals(DataTypes.STRING, getDataType("STRING"));
- assertEquals(DataTypes.INT, getDataType("INT"));
- assertEquals(DataTypes.SHORT, getDataType("SMALLINT"));
- assertEquals(DataTypes.LONG, getDataType("LONG"));
- assertEquals(DataTypes.DOUBLE, getDataType("DOUBLE"));
- assertEquals(DataTypes.DECIMAL, getDataType("DECIMAL"));
- assertEquals(DataTypes.ARRAY, getDataType("ARRAY"));
- assertEquals(DataTypes.STRUCT, getDataType("STRUCT"));
- assertEquals(DataTypes.STRING, getDataType("MAP"));
- assertEquals(DataTypes.STRING, getDataType("default"));
-
- }
-
@Test public void testGetDataBasedOnDataType() throws NumberFormatException {
String data = " ";
if (data.isEmpty()) {
@@ -87,7 +70,7 @@ public class DataTypeUtilTest {
assertEquals(getDataBasedOnDataType("0", DataTypes.LONG), 0L);
java.math.BigDecimal javaDecVal = new java.math.BigDecimal(1);
scala.math.BigDecimal scalaDecVal = new scala.math.BigDecimal(javaDecVal);
- assertEquals(getDataBasedOnDataType("1", DataTypes.DECIMAL),
+ assertEquals(getDataBasedOnDataType("1", DataTypes.createDefaultDecimalType()),
DataTypeUtil.getDataTypeConverter().convertToDecimal(scalaDecVal));
assertEquals(getDataBasedOnDataType("default", DataTypes.NULL),
DataTypeUtil.getDataTypeConverter().convertFromStringToUTF8String("default"));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModelTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModelTest.java b/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModelTest.java
index a42b214..f89128d 100644
--- a/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModelTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModelTest.java
@@ -18,6 +18,7 @@ package org.apache.carbondata.core.writer.sortindex;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.datatype.DecimalType;
import org.junit.Test;
@@ -69,27 +70,27 @@ public class CarbonDictionarySortModelTest {
}
@Test public void testCompareToForDataTypeDecimalCase() {
- carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.DECIMAL, "72.34");
+ carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), "72.34");
CarbonDictionarySortModel testCarbonDictionarySortModel =
- new CarbonDictionarySortModel(2, DataTypes.DECIMAL, "56.78");
+ new CarbonDictionarySortModel(2, DataTypes.createDefaultDecimalType(), "56.78");
int expectedResult = 1;
int actualResult = carbonDictionarySortModel.compareTo(testCarbonDictionarySortModel);
assertEquals(actualResult, expectedResult);
}
@Test public void testCompareToForDataTypeDecimalExceptionCase() {
- carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.DECIMAL, "decimal");
+ carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), "decimal");
CarbonDictionarySortModel testCarbonDictionarySortModel =
- new CarbonDictionarySortModel(2, DataTypes.DECIMAL, "@NU#LL$!");
+ new CarbonDictionarySortModel(2, DataTypes.createDefaultDecimalType(), "@NU#LL$!");
int expectedResult = -1;
int actualResult = carbonDictionarySortModel.compareTo(testCarbonDictionarySortModel);
assertEquals(actualResult, expectedResult);
}
@Test public void testCompareToForDataTypeDecimalExceptionCaseForOtherObject() {
- carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.DECIMAL, "15.24");
+ carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), "15.24");
CarbonDictionarySortModel testCarbonDictionarySortModel =
- new CarbonDictionarySortModel(2, DataTypes.DECIMAL, "@NU#LL$!");
+ new CarbonDictionarySortModel(2, DataTypes.createDefaultDecimalType(), "@NU#LL$!");
int expectedResult = -1;
int actualResult = carbonDictionarySortModel.compareTo(testCarbonDictionarySortModel);
assertEquals(actualResult, expectedResult);
@@ -126,28 +127,28 @@ public class CarbonDictionarySortModelTest {
}
@Test public void testHashCode() {
- carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.DECIMAL, "15.24");
+ carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), "15.24");
int actualResult = carbonDictionarySortModel.hashCode();
int expectedResult = 46877260;
assertTrue(actualResult == expectedResult);
}
@Test public void testHashCodeNullCaseForMemberValue() {
- carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.DECIMAL, null);
+ carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), null);
int actualResult = carbonDictionarySortModel.hashCode();
int expectedResult = 0;
assertEquals(actualResult, expectedResult);
}
@Test public void testEquals() {
- carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.DECIMAL, "15.24");
+ carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), "15.24");
CarbonDictionarySortModel testCarbonDictionarySortModel = carbonDictionarySortModel;
boolean result = carbonDictionarySortModel.equals(testCarbonDictionarySortModel);
assertTrue(result);
}
@Test public void testEqualsMemberValueNullCase() {
- carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.DECIMAL, null);
+ carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), null);
CarbonDictionarySortModel testCarbonDictionarySortModel =
new CarbonDictionarySortModel(1, DataTypes.BOOLEAN, "false");
boolean result = carbonDictionarySortModel.equals(testCarbonDictionarySortModel);
@@ -155,7 +156,7 @@ public class CarbonDictionarySortModelTest {
}
@Test public void testEqualsWhenMemberValueDiffers() {
- carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.DECIMAL, "12.45");
+ carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), "12.45");
CarbonDictionarySortModel testCarbonDictionarySortModel =
new CarbonDictionarySortModel(1, DataTypes.BOOLEAN, "false");
boolean result = carbonDictionarySortModel.equals(testCarbonDictionarySortModel);
@@ -163,15 +164,15 @@ public class CarbonDictionarySortModelTest {
}
@Test public void testEqualsWhenMemberValueIsSame() {
- carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.DECIMAL, "12.45");
+ carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), "12.45");
CarbonDictionarySortModel testCarbonDictionarySortModel =
- new CarbonDictionarySortModel(1, DataTypes.DECIMAL, "12.45");
+ new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), "12.45");
boolean result = carbonDictionarySortModel.equals(testCarbonDictionarySortModel);
assertTrue(result);
}
@Test public void testEqualsForDifferentObjects() {
- carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.DECIMAL, "12.45");
+ carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), "12.45");
Object testCarbonDictionarySortModel = new Object();
boolean result = carbonDictionarySortModel.equals(testCarbonDictionarySortModel);
assertFalse(result);
@@ -187,9 +188,9 @@ public class CarbonDictionarySortModelTest {
}
@Test public void testCompareToForDataTypeDecimalExceptionCaseForDifferentObject() {
- carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.DECIMAL, "12.il");
+ carbonDictionarySortModel = new CarbonDictionarySortModel(1, DataTypes.createDefaultDecimalType(), "12.il");
CarbonDictionarySortModel testCarbonDictionarySortModel =
- new CarbonDictionarySortModel(2, DataTypes.DECIMAL, "12.89");
+ new CarbonDictionarySortModel(2, DataTypes.createDefaultDecimalType(), "12.89");
int expectedResult = 1;
int actualResult = carbonDictionarySortModel.compareTo(testCarbonDictionarySortModel);
assertEquals(actualResult, expectedResult);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
----------------------------------------------------------------------
diff --git a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
index 36ae65c..05ad93a 100644
--- a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
+++ b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
@@ -239,7 +239,7 @@ public class CarbonDictionaryDecodeReadSupport<T> implements CarbonReadSupport<T
return new TimestampWritable(new Timestamp((long) obj / 1000));
} else if (dataType == DataTypes.STRING) {
return new Text(obj.toString());
- } else if (dataType == DataTypes.DECIMAL) {
+ } else if (DataTypes.isDecimal(dataType)) {
return new HiveDecimalWritable(HiveDecimal.create(new java.math.BigDecimal(obj.toString())));
} else {
throw new IOException("unsupported data type:" + dataType);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonTypeUtil.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonTypeUtil.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonTypeUtil.java
index e2c9c68..ddc8d9e 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonTypeUtil.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonTypeUtil.java
@@ -20,7 +20,7 @@ public class CarbonTypeUtil {
return DataTypes.DoubleType;
} else if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.BOOLEAN) {
return DataTypes.BooleanType;
- } else if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.DECIMAL) {
+ } else if (org.apache.carbondata.core.metadata.datatype.DataTypes.isDecimal(carbonDataType)) {
return DataTypes.createDecimalType();
} else if (carbonDataType == org.apache.carbondata.core.metadata.datatype.DataTypes.TIMESTAMP) {
return DataTypes.TimestampType;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java
index 2db2d23..f129474 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbonVectorizedRecordReader.java
@@ -205,7 +205,7 @@ class CarbonVectorizedRecordReader extends AbstractRecordReader<Object> {
fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(),
CarbonTypeUtil.convertCarbonToSparkDataType(msr.getMeasure().getDataType()), true,
null);
- } else if (dataType == DataTypes.DECIMAL) {
+ } else if (DataTypes.isDecimal(dataType)) {
fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(),
new DecimalType(msr.getMeasure().getPrecision(), msr.getMeasure().getScale()), true,
null);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java
index 4a9b7ed..fb9a0ba 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataColumnHandle.java
@@ -76,8 +76,10 @@ public class CarbondataColumnHandle implements ColumnHandle {
return columnUniqueId;
}
- @JsonCreator public CarbondataColumnHandle(@JsonProperty("connectorId") String connectorId,
- @JsonProperty("columnName") String columnName, @JsonProperty("columnType") Type columnType,
+ @JsonCreator public CarbondataColumnHandle(
+ @JsonProperty("connectorId") String connectorId,
+ @JsonProperty("columnName") String columnName,
+ @JsonProperty("columnType") Type columnType,
@JsonProperty("ordinalPosition") int ordinalPosition,
@JsonProperty("keyOrdinal") int keyOrdinal,
@JsonProperty("columnGroupOrdinal") int columnGroupOrdinal,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
index 99109d1..f106a08 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/CarbondataMetadata.java
@@ -232,7 +232,7 @@ public class CarbondataMetadata implements ConnectorMetadata {
return BigintType.BIGINT;
} else if (colType == DataTypes.FLOAT || colType == DataTypes.DOUBLE) {
return DoubleType.DOUBLE;
- } else if (colType == DataTypes.DECIMAL) {
+ } else if (DataTypes.isDecimal(colType)) {
if (columnSchema.getPrecision() > 0) {
return DecimalType.createDecimalType(columnSchema.getPrecision(), columnSchema.getScale());
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java
----------------------------------------------------------------------
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java
index c9fb177..0982254 100644
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/PrestoFilterUtil.java
@@ -79,7 +79,10 @@ public class PrestoFilterUtil {
else if (colType == DateType.DATE) return DataTypes.DATE;
else if (colType == TimestampType.TIMESTAMP) return DataTypes.TIMESTAMP;
else if (colType.equals(DecimalType.createDecimalType(carbondataColumnHandle.getPrecision(),
- carbondataColumnHandle.getScale()))) return DataTypes.DECIMAL;
+ carbondataColumnHandle.getScale())))
+ return org.apache.carbondata.core.metadata.datatype.DataTypes.createDecimalType(
+ carbondataColumnHandle.getPrecision(),
+ carbondataColumnHandle.getScale());
else return DataTypes.STRING;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
----------------------------------------------------------------------
diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
index b8031a2..acee71b 100644
--- a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
+++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala
@@ -40,7 +40,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.fileoperations.{AtomicFileOperations, AtomicFileOperationsImpl, FileWriteOperation}
import org.apache.carbondata.core.metadata.converter.{SchemaConverter, ThriftWrapperSchemaConverterImpl}
-import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes}
+import org.apache.carbondata.core.metadata.datatype.DataTypes
import org.apache.carbondata.core.metadata.encoder.Encoding
import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension, CarbonMeasure, ColumnSchema}
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo, TableSchema}
@@ -236,7 +236,7 @@ object CarbonDataStoreCreator {
val bonus: ColumnSchema = new ColumnSchema()
bonus.setColumnName("bonus")
bonus.setColumnar(true)
- bonus.setDataType(DataTypes.DECIMAL)
+ bonus.setDataType(DataTypes.createDecimalType(10, 4))
bonus.setPrecision(10)
bonus.setScale(4)
bonus.setEncodingList(encodings)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index cf37a18..2043ecf 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -25,12 +25,11 @@ import org.apache.spark.sql.execution.command.DataTypeInfo
import org.apache.spark.sql.types._
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.datatype.{DataType => CarbonDataType, DataTypes => CarbonDataTypes}
+import org.apache.carbondata.core.metadata.datatype.{DataType => CarbonDataType, DataTypes => CarbonDataTypes, DecimalType => CarbonDecimalType}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn
object CarbonScalaUtil {
- def convertSparkToCarbonDataType(
- dataType: org.apache.spark.sql.types.DataType): CarbonDataType = {
+ def convertSparkToCarbonDataType(dataType: DataType): CarbonDataType = {
dataType match {
case StringType => CarbonDataTypes.STRING
case ShortType => CarbonDataTypes.SHORT
@@ -44,7 +43,9 @@ object CarbonScalaUtil {
case ArrayType(_, _) => CarbonDataTypes.ARRAY
case StructType(_) => CarbonDataTypes.STRUCT
case NullType => CarbonDataTypes.NULL
- case _ => CarbonDataTypes.DECIMAL
+ case decimal: DecimalType =>
+ CarbonDataTypes.createDecimalType(decimal.precision, decimal.scale)
+ case _ => throw new UnsupportedOperationException("getting " + dataType + " from spark")
}
}
@@ -66,27 +67,19 @@ object CarbonScalaUtil {
}
def convertCarbonToSparkDataType(dataType: CarbonDataType): types.DataType = {
- dataType match {
- case CarbonDataTypes.STRING => StringType
- case CarbonDataTypes.SHORT => ShortType
- case CarbonDataTypes.INT => IntegerType
- case CarbonDataTypes.LONG => LongType
- case CarbonDataTypes.DOUBLE => DoubleType
- case CarbonDataTypes.BOOLEAN => BooleanType
- case CarbonDataTypes.DECIMAL => DecimalType.SYSTEM_DEFAULT
- case CarbonDataTypes.TIMESTAMP => TimestampType
- case CarbonDataTypes.DATE => DateType
- }
- }
-
- def updateDataType(
- currentDataType: org.apache.spark.sql.types.DataType): org.apache.spark.sql.types.DataType = {
- currentDataType match {
- case decimal: DecimalType =>
- val scale = currentDataType.asInstanceOf[DecimalType].scale
- DecimalType(DecimalType.MAX_PRECISION, scale)
- case _ =>
- currentDataType
+ if (CarbonDataTypes.isDecimal(dataType)) {
+ DecimalType.SYSTEM_DEFAULT
+ } else {
+ dataType match {
+ case CarbonDataTypes.STRING => StringType
+ case CarbonDataTypes.SHORT => ShortType
+ case CarbonDataTypes.INT => IntegerType
+ case CarbonDataTypes.LONG => LongType
+ case CarbonDataTypes.DOUBLE => DoubleType
+ case CarbonDataTypes.BOOLEAN => BooleanType
+ case CarbonDataTypes.TIMESTAMP => TimestampType
+ case CarbonDataTypes.DATE => DateType
+ }
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
index 89a3ac3..0460e30 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
@@ -17,7 +17,7 @@
package org.apache.carbondata.spark.util
-import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes}
+import org.apache.carbondata.core.metadata.datatype.{DataType, DataTypes, DecimalType}
import org.apache.carbondata.format.{DataType => ThriftDataType}
object DataTypeConverterUtil {
@@ -37,8 +37,8 @@ object DataTypeConverterUtil {
case "numeric" => DataTypes.DOUBLE
case "double" => DataTypes.DOUBLE
case "float" => DataTypes.DOUBLE
- case "decimal" => DataTypes.DECIMAL
- case FIXED_DECIMAL(_, _) => DataTypes.DECIMAL
+ case "decimal" => DataTypes.createDefaultDecimalType
+ case FIXED_DECIMAL(_, _) => DataTypes.createDefaultDecimalType
case "timestamp" => DataTypes.TIMESTAMP
case "date" => DataTypes.DATE
case "array" => DataTypes.ARRAY
@@ -60,8 +60,8 @@ object DataTypeConverterUtil {
case "numerictype" => DataTypes.DOUBLE
case "doubletype" => DataTypes.DOUBLE
case "floattype" => DataTypes.DOUBLE
- case "decimaltype" => DataTypes.DECIMAL
- case FIXED_DECIMALTYPE(_, _) => DataTypes.DECIMAL
+ case "decimaltype" => DataTypes.createDefaultDecimalType
+ case FIXED_DECIMALTYPE(_, _) => DataTypes.createDefaultDecimalType
case "timestamptype" => DataTypes.TIMESTAMP
case "datetype" => DataTypes.DATE
case others =>
@@ -80,19 +80,22 @@ object DataTypeConverterUtil {
}
def convertToString(dataType: DataType): String = {
- dataType match {
- case DataTypes.BOOLEAN => "boolean"
+ if (DataTypes.isDecimal(dataType)) {
+ "decimal"
+ } else {
+ dataType match {
+ case DataTypes.BOOLEAN => "boolean"
case DataTypes.STRING => "string"
- case DataTypes.SHORT => "smallint"
- case DataTypes.INT => "int"
- case DataTypes.LONG => "bigint"
- case DataTypes.DOUBLE => "double"
- case DataTypes.FLOAT => "double"
- case DataTypes.DECIMAL => "decimal"
- case DataTypes.TIMESTAMP => "timestamp"
- case DataTypes.DATE => "date"
- case DataTypes.ARRAY => "array"
- case DataTypes.STRUCT => "struct"
+ case DataTypes.SHORT => "smallint"
+ case DataTypes.INT => "int"
+ case DataTypes.LONG => "bigint"
+ case DataTypes.DOUBLE => "double"
+ case DataTypes.FLOAT => "double"
+ case DataTypes.TIMESTAMP => "timestamp"
+ case DataTypes.DATE => "date"
+ case DataTypes.ARRAY => "array"
+ case DataTypes.STRUCT => "struct"
+ }
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f209e8ee/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 1163b3f..edb471e 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -622,7 +622,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
dimFields += field
} else if (dictIncludeCols.exists(x => x.equalsIgnoreCase(field.column))) {
dimFields += field
- } else if (DataTypeUtil.getDataType(field.dataType.get.toUpperCase) == DataTypes.TIMESTAMP &&
+ } else if (field.dataType.get.toUpperCase.equals("TIMESTAMP") &&
!dictIncludeCols.exists(x => x.equalsIgnoreCase(field.column))) {
noDictionaryDims :+= field.column
dimFields += field