You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/11/10 14:09:07 UTC
[2/2] carbondata git commit: [CARBONDATA-1662] Make ArrayType and
StructType contain child DataType
[CARBONDATA-1662] Make ArrayType and StructType contain child DataType
StructType and ArrayType should be class that have nested children.
This closes #1429
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/933e30cc
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/933e30cc
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/933e30cc
Branch: refs/heads/master
Commit: 933e30ccc74d499a6323f328349c9a71ba0c44e3
Parents: a91c6e4
Author: Jacky Li <ja...@qq.com>
Authored: Mon Nov 6 11:13:40 2017 +0800
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Nov 10 19:38:49 2017 +0530
----------------------------------------------------------------------
.../core/datastore/block/SegmentProperties.java | 3 +-
.../blockletindex/BlockletDataMap.java | 3 +-
.../core/indexstore/row/DataMapRowImpl.java | 2 +-
.../ThriftWrapperSchemaConverterImpl.java | 8 +-
.../core/metadata/datatype/ArrayType.java | 12 +-
.../core/metadata/datatype/BooleanType.java | 4 +-
.../core/metadata/datatype/ByteArrayType.java | 4 +-
.../core/metadata/datatype/ByteType.java | 4 +-
.../core/metadata/datatype/DataTypes.java | 110 ++++++++++++++-----
.../core/metadata/datatype/DateType.java | 4 +-
.../core/metadata/datatype/DoubleType.java | 4 +-
.../core/metadata/datatype/FloatType.java | 5 +-
.../core/metadata/datatype/IntType.java | 4 +-
.../core/metadata/datatype/LongType.java | 4 +-
.../core/metadata/datatype/MapType.java | 14 +--
.../core/metadata/datatype/NullType.java | 4 +-
.../core/metadata/datatype/ShortIntType.java | 4 +-
.../core/metadata/datatype/ShortType.java | 4 +-
.../core/metadata/datatype/StringType.java | 4 +-
.../core/metadata/datatype/StructField.java | 42 +++++++
.../core/metadata/datatype/StructType.java | 18 +--
.../core/metadata/datatype/TimestampType.java | 4 +-
.../executor/impl/AbstractQueryExecutor.java | 4 +-
.../core/scan/executor/util/QueryUtil.java | 8 +-
.../expression/RangeExpressionEvaluator.java | 28 +----
.../scan/filter/FilterExpressionProcessor.java | 29 ++---
.../executer/RowLevelFilterExecuterImpl.java | 6 +-
.../resolver/ConditionalFilterResolverImpl.java | 7 +-
.../util/AbstractDataFileFooterConverter.java | 4 +-
.../DictionaryColumnUniqueIdentifierTest.java | 8 +-
.../datastore/block/SegmentPropertiesTest.java | 2 +-
.../block/SegmentPropertiesTestUtil.java | 2 +-
...ctDictionaryKeyGeneratorFactoryUnitTest.java | 2 +-
.../ThriftWrapperSchemaConverterImplTest.java | 6 +-
.../table/CarbonTableWithComplexTypesTest.java | 4 +-
.../carbondata/core/util/CarbonUtilTest.java | 4 +-
.../CarbonDictionarySortInfoPreparatorTest.java | 4 +-
.../streaming/CarbonStreamInputFormat.java | 8 +-
.../streaming/CarbonStreamRecordReader.java | 32 +++---
.../streaming/CarbonStreamRecordWriter.java | 21 ++--
.../hive/CarbonDictionaryDecodeReadSupport.java | 4 +-
.../carbondata/spark/util/CarbonScalaUtil.scala | 16 ++-
.../spark/util/DataTypeConverterUtil.scala | 14 ++-
.../spark/util/GlobalDictionaryUtil.scala | 37 +++----
.../spark/sql/CarbonDictionaryDecoder.scala | 12 +-
.../converter/impl/FieldEncoderFactory.java | 6 +-
.../loading/parser/CarbonParserFactory.java | 6 +-
.../util/CarbonDataProcessorUtil.java | 6 +-
48 files changed, 305 insertions(+), 240 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
index c93b771..5f82e87 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
@@ -378,8 +378,7 @@ public class SegmentProperties {
}
// as complex type will be stored at last so once complex type started all the dimension
// will be added to complex type
- else if (isComplexDimensionStarted || CarbonUtil.hasDataType(columnSchema.getDataType(),
- new DataType[] { DataTypes.ARRAY, DataTypes.STRUCT })) {
+ else if (isComplexDimensionStarted || columnSchema.getDataType().isComplexType()) {
cardinalityIndexForComplexDimensionColumn.add(tableOrdinal);
carbonDimension =
new CarbonDimension(columnSchema, dimensonOrdinal++, -1, -1, ++complexTypeOrdinal);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
index d5bd695..7829034 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -284,7 +284,8 @@ public class BlockletDataMap implements DataMap, Cacheable {
mapSchemas[i] = new DataMapSchema.FixedDataMapSchema(DataTypes.BYTE_ARRAY, minMaxLen[i]);
}
}
- DataMapSchema mapSchema = new DataMapSchema.StructDataMapSchema(DataTypes.STRUCT, mapSchemas);
+ DataMapSchema mapSchema = new DataMapSchema.StructDataMapSchema(
+ DataTypes.createDefaultStructType(), mapSchemas);
indexSchemas.add(mapSchema);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRowImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRowImpl.java b/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRowImpl.java
index bc55e74..032b29e 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRowImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRowImpl.java
@@ -99,7 +99,7 @@ public class DataMapRowImpl extends DataMapRow {
}
@Override public void setRow(DataMapRow row, int ordinal) {
- assert (schemas[ordinal].getDataType() == DataTypes.STRUCT);
+ assert (DataTypes.isStructType(schemas[ordinal].getDataType()));
data[ordinal] = row;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index 70a6e63..adcac7d 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -145,9 +145,9 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
return org.apache.carbondata.format.DataType.DATE;
} else if (dataType.getId() == DataTypes.TIMESTAMP.getId()) {
return org.apache.carbondata.format.DataType.TIMESTAMP;
- } else if (dataType.getId() == DataTypes.ARRAY.getId()) {
+ } else if (DataTypes.isArrayType(dataType)) {
return org.apache.carbondata.format.DataType.ARRAY;
- } else if (dataType.getId() == DataTypes.STRUCT.getId()) {
+ } else if (DataTypes.isStructType(dataType)) {
return org.apache.carbondata.format.DataType.STRUCT;
} else {
return org.apache.carbondata.format.DataType.STRING;
@@ -392,9 +392,9 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
case DATE:
return DataTypes.DATE;
case ARRAY:
- return DataTypes.ARRAY;
+ return DataTypes.createDefaultArrayType();
case STRUCT:
- return DataTypes.STRUCT;
+ return DataTypes.createDefaultStructType();
default:
return DataTypes.STRING;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
index 241d0f6..c30e21c 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
@@ -19,15 +19,11 @@ package org.apache.carbondata.core.metadata.datatype;
public class ArrayType extends DataType {
- public static final DataType ARRAY = new ArrayType(DataTypes.ARRAY_TYPE_ID, 9, "ARRAY", -1);
+ private DataType elementType;
- private ArrayType(int id, int precedenceOrder, String name, int sizeInBytes) {
- super(id, precedenceOrder, name, sizeInBytes);
- }
-
- // this function is needed to ensure singleton pattern while supporting java serialization
- private Object readResolve() {
- return DataTypes.ARRAY;
+ ArrayType(DataType elementType) {
+ super(DataTypes.ARRAY_TYPE_ID, 9, "ARRAY", -1);
+ this.elementType = elementType;
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/BooleanType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/BooleanType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/BooleanType.java
index 36cb84f..0ce0412 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/BooleanType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/BooleanType.java
@@ -17,9 +17,9 @@
package org.apache.carbondata.core.metadata.datatype;
-public class BooleanType extends DataType {
+class BooleanType extends DataType {
- public static final DataType BOOLEAN =
+ static final DataType BOOLEAN =
new BooleanType(DataTypes.BOOLEAN_TYPE_ID, 1, "BOOLEAN", 1);
private BooleanType(int id, int precedenceOrder, String name, int sizeInBytes) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ByteArrayType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ByteArrayType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ByteArrayType.java
index 1f2872d..9cb33ed 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ByteArrayType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ByteArrayType.java
@@ -17,9 +17,9 @@
package org.apache.carbondata.core.metadata.datatype;
-public class ByteArrayType extends DataType {
+class ByteArrayType extends DataType {
- public static final DataType BYTE_ARRAY =
+ static final DataType BYTE_ARRAY =
new ByteArrayType(DataTypes.BYTE_ARRAY_TYPE_ID, 13, "BYTE_ARRAY", -1);
private ByteArrayType(int id, int precedenceOrder, String name, int sizeInBytes) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ByteType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ByteType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ByteType.java
index 11138d2..e1b39a3 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ByteType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ByteType.java
@@ -17,9 +17,9 @@
package org.apache.carbondata.core.metadata.datatype;
-public class ByteType extends DataType {
+class ByteType extends DataType {
- public static final DataType BYTE = new ByteType(DataTypes.BYTE_TYPE_ID, 12, "BYTE", 1);
+ static final DataType BYTE = new ByteType(DataTypes.BYTE_TYPE_ID, 12, "BYTE", 1);
private ByteType(int id, int precedenceOrder, String name, int sizeInBytes) {
super(id, precedenceOrder, name, sizeInBytes);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
index 8686583..e96917f 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
@@ -17,6 +17,9 @@
package org.apache.carbondata.core.metadata.datatype;
+import java.util.ArrayList;
+import java.util.List;
+
/**
* Holds all singleton object for all data type used in carbon
*/
@@ -44,28 +47,25 @@ public class DataTypes {
// Only for internal use for backward compatability. It is only used for V1 version
public static final DataType LEGACY_LONG = LegacyLongType.LEGACY_LONG;
- public static final DataType ARRAY = ArrayType.ARRAY;
- public static final DataType STRUCT = StructType.STRUCT;
- public static final DataType MAP = MapType.MAP;
-
- public static final int STRING_TYPE_ID = 0;
- public static final int DATE_TYPE_ID = 1;
- public static final int TIMESTAMP_TYPE_ID = 2;
- public static final int BOOLEAN_TYPE_ID = 3;
- public static final int SHORT_TYPE_ID = 4;
- public static final int INT_TYPE_ID = 5;
- public static final int FLOAT_TYPE_ID = 6;
- public static final int LONG_TYPE_ID = 7;
- public static final int DOUBLE_TYPE_ID = 8;
- public static final int NULL_TYPE_ID = 9;
- public static final int BYTE_TYPE_ID = 10;
- public static final int BYTE_ARRAY_TYPE_ID = 11;
- public static final int SHORT_INT_TYPE_ID = 12;
- public static final int LEGACY_LONG_TYPE_ID = 13;
- public static final int DECIMAL_TYPE_ID = 20;
- public static final int ARRAY_TYPE_ID = 21;
- public static final int STRUCT_TYPE_ID = 22;
- public static final int MAP_TYPE_ID = 23;
+ // these IDs are used within this package only
+ static final int STRING_TYPE_ID = 0;
+ static final int DATE_TYPE_ID = 1;
+ static final int TIMESTAMP_TYPE_ID = 2;
+ static final int BOOLEAN_TYPE_ID = 3;
+ static final int SHORT_TYPE_ID = 4;
+ static final int INT_TYPE_ID = 5;
+ static final int FLOAT_TYPE_ID = 6;
+ static final int LONG_TYPE_ID = 7;
+ static final int DOUBLE_TYPE_ID = 8;
+ static final int NULL_TYPE_ID = 9;
+ static final int BYTE_TYPE_ID = 10;
+ static final int BYTE_ARRAY_TYPE_ID = 11;
+ static final int SHORT_INT_TYPE_ID = 12;
+ static final int LEGACY_LONG_TYPE_ID = 13;
+ static final int DECIMAL_TYPE_ID = 20;
+ static final int ARRAY_TYPE_ID = 21;
+ static final int STRUCT_TYPE_ID = 22;
+ static final int MAP_TYPE_ID = 23;
/**
* create a DataType instance from uniqueId of the DataType
@@ -99,12 +99,12 @@ public class DataTypes {
return NULL;
} else if (id == DECIMAL_TYPE_ID) {
return createDefaultDecimalType();
- } else if (id == ARRAY.getId()) {
- return ARRAY;
- } else if (id == STRUCT.getId()) {
- return STRUCT;
- } else if (id == MAP.getId()) {
- return MAP;
+ } else if (id == ARRAY_TYPE_ID) {
+ return createDefaultArrayType();
+ } else if (id == STRUCT_TYPE_ID) {
+ return createDefaultStructType();
+ } else if (id == MAP_TYPE_ID) {
+ return createDefaultMapType();
} else if (id == BYTE_ARRAY.getId()) {
return BYTE_ARRAY;
} else {
@@ -130,4 +130,58 @@ public class DataTypes {
return dataType.getId() == DECIMAL_TYPE_ID;
}
+ /**
+ * create array type with specified element type
+ */
+ public static ArrayType createArrayType(DataType elementType) {
+ return new ArrayType(elementType);
+ }
+
+ /**
+ * create a array type object with no child
+ */
+ public static ArrayType createDefaultArrayType() {
+ return new ArrayType(STRING);
+ }
+
+ public static boolean isArrayType(DataType dataType) {
+ return dataType.getId() == ARRAY_TYPE_ID;
+ }
+
+ /**
+ * create struct type with specified fields
+ */
+ public static StructType createStructType(List<StructField> fields) {
+ return new StructType(fields);
+ }
+
+ /**
+ * create a struct type object with no field
+ */
+ public static StructType createDefaultStructType() {
+ return new StructType(new ArrayList<StructField>());
+ }
+
+ public static boolean isStructType(DataType dataType) {
+ return dataType.getId() == STRUCT_TYPE_ID;
+ }
+
+ /**
+ * create map type with specified key type and value type
+ */
+ public static MapType createMapType(DataType keyType, DataType valueType) {
+ return new MapType(keyType, valueType);
+ }
+
+ /**
+ * create a map type object with no child
+ */
+ public static MapType createDefaultMapType() {
+ return new MapType(STRING, STRING);
+ }
+
+ public static boolean isMapType(DataType dataType) {
+ return dataType.getId() == MAP_TYPE_ID;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DateType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DateType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DateType.java
index 2b930c3..efc9799 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DateType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DateType.java
@@ -17,9 +17,9 @@
package org.apache.carbondata.core.metadata.datatype;
-public class DateType extends DataType {
+class DateType extends DataType {
- public static final DataType DATE = new DateType(DataTypes.DATE_TYPE_ID, 1, "DATE", -1);
+ static final DataType DATE = new DateType(DataTypes.DATE_TYPE_ID, 1, "DATE", -1);
private DateType(int id, int precedenceOrder, String name, int sizeInBytes) {
super(id, precedenceOrder, name, sizeInBytes);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DoubleType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DoubleType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DoubleType.java
index 59a4cc2..5e032a0 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DoubleType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DoubleType.java
@@ -17,9 +17,9 @@
package org.apache.carbondata.core.metadata.datatype;
-public class DoubleType extends DataType {
+class DoubleType extends DataType {
- public static final DataType DOUBLE = new DoubleType(DataTypes.DOUBLE_TYPE_ID, 6, "DOUBLE", 8);
+ static final DataType DOUBLE = new DoubleType(DataTypes.DOUBLE_TYPE_ID, 6, "DOUBLE", 8);
private DoubleType(int id, int precedenceOrder, String name, int sizeInBytes) {
super(id, precedenceOrder, name, sizeInBytes);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/FloatType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/FloatType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/FloatType.java
index 1c0bfb6..9f600f9 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/FloatType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/FloatType.java
@@ -17,8 +17,9 @@
package org.apache.carbondata.core.metadata.datatype;
-public class FloatType extends DataType {
- public static final DataType FLOAT = new FloatType(DataTypes.FLOAT_TYPE_ID, 4, "FLOAT", 4);
+class FloatType extends DataType {
+
+ static final DataType FLOAT = new FloatType(DataTypes.FLOAT_TYPE_ID, 4, "FLOAT", 4);
private FloatType(int id, int precedenceOrder, String name, int sizeInBytes) {
super(id, precedenceOrder, name, sizeInBytes);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/IntType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/IntType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/IntType.java
index 4324d9e..308a53b 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/IntType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/IntType.java
@@ -17,9 +17,9 @@
package org.apache.carbondata.core.metadata.datatype;
-public class IntType extends DataType {
+class IntType extends DataType {
- public static final DataType INT = new IntType(DataTypes.INT_TYPE_ID, 3, "INT", 4);
+ static final DataType INT = new IntType(DataTypes.INT_TYPE_ID, 3, "INT", 4);
private IntType(int id, int precedenceOrder, String name, int sizeInBytes) {
super(id, precedenceOrder, name, sizeInBytes);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/LongType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/LongType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/LongType.java
index 3d05fd9..3bc1a4b 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/LongType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/LongType.java
@@ -17,9 +17,9 @@
package org.apache.carbondata.core.metadata.datatype;
-public class LongType extends DataType {
+class LongType extends DataType {
- public static final DataType LONG = new LongType(DataTypes.LONG_TYPE_ID, 5, "LONG", 8);
+ static final DataType LONG = new LongType(DataTypes.LONG_TYPE_ID, 5, "LONG", 8);
private LongType(int id, int precedenceOrder, String name, int sizeInBytes) {
super(id, precedenceOrder, name, sizeInBytes);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/MapType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/MapType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/MapType.java
index a360243..69d49b8 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/MapType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/MapType.java
@@ -19,15 +19,13 @@ package org.apache.carbondata.core.metadata.datatype;
public class MapType extends DataType {
- public static final DataType MAP = new MapType(DataTypes.MAP_TYPE_ID, 11, "MAP", -1);
+ private DataType keyType;
+ private DataType valueType;
- private MapType(int id, int precedenceOrder, String name, int sizeInBytes) {
- super(id, precedenceOrder, name, sizeInBytes);
- }
-
- // this function is needed to ensure singleton pattern while supporting java serialization
- private Object readResolve() {
- return DataTypes.MAP;
+ MapType(DataType keyType, DataType valueType) {
+ super(DataTypes.MAP_TYPE_ID, 11, "MAP", -1);
+ this.keyType = keyType;
+ this.valueType = valueType;
}
@Override
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/NullType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/NullType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/NullType.java
index 43ffe41..81248c5 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/NullType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/NullType.java
@@ -17,9 +17,9 @@
package org.apache.carbondata.core.metadata.datatype;
-public class NullType extends DataType {
+class NullType extends DataType {
- public static final DataType NULL = new NullType(DataTypes.NULL_TYPE_ID, 7, "NULL", 1);
+ static final DataType NULL = new NullType(DataTypes.NULL_TYPE_ID, 7, "NULL", 1);
private NullType(int id, int precedenceOrder, String name, int sizeInBytes) {
super(id, precedenceOrder, name, sizeInBytes);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ShortIntType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ShortIntType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ShortIntType.java
index 5d57748..24c90d5 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ShortIntType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ShortIntType.java
@@ -17,9 +17,9 @@
package org.apache.carbondata.core.metadata.datatype;
-public class ShortIntType extends DataType {
+class ShortIntType extends DataType {
- public static final DataType SHORT_INT =
+ static final DataType SHORT_INT =
new ShortIntType(DataTypes.SHORT_INT_TYPE_ID, 14, "SHORT_INT", 3);
private ShortIntType(int id, int precedenceOrder, String name, int sizeInBytes) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ShortType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ShortType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ShortType.java
index 0085227..4bf5563 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ShortType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ShortType.java
@@ -17,9 +17,9 @@
package org.apache.carbondata.core.metadata.datatype;
-public class ShortType extends DataType {
+class ShortType extends DataType {
- public static final DataType SHORT = new ShortType(DataTypes.SHORT_TYPE_ID, 2, "SHORT", 2);
+ static final DataType SHORT = new ShortType(DataTypes.SHORT_TYPE_ID, 2, "SHORT", 2);
private ShortType(int id, int precedenceOrder, String name, int sizeInBytes) {
super(id, precedenceOrder, name, sizeInBytes);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StringType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StringType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StringType.java
index 5c95480..18c505a 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StringType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StringType.java
@@ -17,8 +17,8 @@
package org.apache.carbondata.core.metadata.datatype;
-public class StringType extends DataType {
- public static final DataType STRING = new StringType(DataTypes.STRING_TYPE_ID, 0, "STRING", -1);
+class StringType extends DataType {
+ static final DataType STRING = new StringType(DataTypes.STRING_TYPE_ID, 0, "STRING", -1);
private StringType(int id, int precedenceOrder, String name, int sizeInBytes) {
super(id, precedenceOrder, name, sizeInBytes);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructField.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructField.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructField.java
new file mode 100644
index 0000000..efdc8e2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructField.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.metadata.datatype;
+
+import java.io.Serializable;
+
+public class StructField implements Serializable {
+
+ private static final long serialVersionUID = 3271726L;
+
+ private String fieldName;
+
+ private DataType dataType;
+
+ public StructField(String fieldName, DataType dataType) {
+ this.fieldName = fieldName;
+ this.dataType = dataType;
+ }
+
+ public DataType getDataType() {
+ return dataType;
+ }
+
+ public String getFieldName() {
+ return fieldName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java
index 081f952..6417f37 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java
@@ -17,21 +17,23 @@
package org.apache.carbondata.core.metadata.datatype;
-public class StructType extends DataType {
+import java.util.List;
- public static final DataType STRUCT = new StructType(DataTypes.STRUCT_TYPE_ID, 10, "STRUCT", -1);
+class StructType extends DataType {
- private StructType(int id, int precedenceOrder, String name, int sizeInBytes) {
- super(id, precedenceOrder, name, sizeInBytes);
- }
+ private List<StructField> fields;
- // this function is needed to ensure singleton pattern while supporting java serialization
- private Object readResolve() {
- return DataTypes.STRUCT;
+ StructType(List<StructField> fields) {
+ super(DataTypes.STRUCT_TYPE_ID, 10, "STRUCT", -1);
+ this.fields = fields;
}
@Override
public boolean isComplexType() {
return true;
}
+
+ public List<StructField> getFields() {
+ return fields;
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/TimestampType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/TimestampType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/TimestampType.java
index 287c457..42a6855 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/TimestampType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/TimestampType.java
@@ -17,8 +17,8 @@
package org.apache.carbondata.core.metadata.datatype;
-public class TimestampType extends DataType {
- public static final TimestampType TIMESTAMP =
+class TimestampType extends DataType {
+ static final TimestampType TIMESTAMP =
new TimestampType(DataTypes.TIMESTAMP_TYPE_ID, 2, "TIMESTAMP", -1);
private TimestampType(int id, int precedenceOrder, String name, int sizeInBytes) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index 56ed575..251a06b 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -49,7 +49,6 @@ import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.memory.UnsafeMemoryManager;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
@@ -478,8 +477,7 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
private int[] getComplexDimensionParentBlockIndexes(List<QueryDimension> queryDimensions) {
List<Integer> parentBlockIndexList = new ArrayList<Integer>();
for (QueryDimension queryDimension : queryDimensions) {
- if (CarbonUtil.hasDataType(queryDimension.getDimension().getDataType(),
- new DataType[] { DataTypes.ARRAY, DataTypes.STRUCT, DataTypes.MAP })) {
+ if (queryDimension.getDimension().getDataType().isComplexType()) {
parentBlockIndexList.add(queryDimension.getDimension().getOrdinal());
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
index 6aab8a0..b090e59 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
@@ -763,10 +763,10 @@ public class QueryUtil {
int[] eachComplexColumnValueSize, Map<String, Dictionary> columnIdToDictionaryMap) {
int parentBlockIndex = dimensionToBlockIndexMap.get(dimension.getOrdinal());
GenericQueryType parentQueryType;
- if (dimension.getDataType() == DataTypes.ARRAY) {
+ if (DataTypes.isArrayType(dimension.getDataType())) {
parentQueryType =
new ArrayQueryType(dimension.getColName(), dimension.getColName(), parentBlockIndex);
- } else if (dimension.getDataType() == DataTypes.STRUCT) {
+ } else if (DataTypes.isStructType(dimension.getDataType())) {
parentQueryType =
new StructQueryType(dimension.getColName(), dimension.getColName(),
dimensionToBlockIndexMap.get(dimension.getOrdinal()));
@@ -784,11 +784,11 @@ public class QueryUtil {
CarbonDimension dimension, GenericQueryType parentQueryType) {
for (int i = 0; i < dimension.getNumberOfChild(); i++) {
DataType dataType = dimension.getListOfChildDimensions().get(i).getDataType();
- if (dataType == DataTypes.ARRAY) {
+ if (DataTypes.isArrayType(dataType)) {
parentQueryType.addChildren(
new ArrayQueryType(dimension.getListOfChildDimensions().get(i).getColName(),
dimension.getColName(), ++parentBlockIndex));
- } else if (dataType == DataTypes.STRUCT) {
+ } else if (DataTypes.isStructType(dataType)) {
parentQueryType.addChildren(
new StructQueryType(dimension.getListOfChildDimensions().get(i).getColName(),
dimension.getColName(), ++parentBlockIndex));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
index 9d418ee..e7cc1c3 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
@@ -27,7 +27,6 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.scan.expression.conditional.GreaterThanEqualToExpression;
import org.apache.carbondata.core.scan.expression.conditional.GreaterThanExpression;
import org.apache.carbondata.core.scan.expression.conditional.LessThanEqualToExpression;
@@ -259,13 +258,9 @@ public class RangeExpressionEvaluator {
* @return
*/
private boolean isLessThanGreaterThanExp(Expression expr) {
- if ((expr instanceof LessThanEqualToExpression) || (expr instanceof LessThanExpression)
+ return (expr instanceof LessThanEqualToExpression) || (expr instanceof LessThanExpression)
|| (expr instanceof GreaterThanEqualToExpression)
- || (expr instanceof GreaterThanExpression)) {
- return true;
- } else {
- return false;
- }
+ || (expr instanceof GreaterThanExpression);
}
/**
@@ -277,15 +272,8 @@ public class RangeExpressionEvaluator {
private boolean eligibleForRangeExpConv(Expression expChild) {
for (Expression exp : expChild.getChildren()) {
if (exp instanceof ColumnExpression) {
- if (((ColumnExpression) exp).isDimension() == false) {
- return false;
- }
- if ((((ColumnExpression) exp).getDimension().getDataType() == DataTypes.ARRAY) || (
- ((ColumnExpression) exp).getDimension().getDataType() == DataTypes.STRUCT)) {
- return false;
- } else {
- return true;
- }
+ return ((ColumnExpression) exp).isDimension() &&
+ ! (((ColumnExpression) exp).getDimension().getDataType().isComplexType());
}
}
return false;
@@ -384,13 +372,9 @@ public class RangeExpressionEvaluator {
* @return
*/
private boolean matchExpType(ExpressionType src, ExpressionType tar) {
- if ((((src == LESSTHAN) || (src == LESSTHAN_EQUALTO)) && ((tar == GREATERTHAN) || (tar
+ return (((src == LESSTHAN) || (src == LESSTHAN_EQUALTO)) && ((tar == GREATERTHAN) || (tar
== GREATERTHAN_EQUALTO))) || (((src == GREATERTHAN) || (src == GREATERTHAN_EQUALTO)) && (
- (tar == LESSTHAN) || (tar == LESSTHAN_EQUALTO)))) {
- return true;
- } else {
- return false;
- }
+ (tar == LESSTHAN) || (tar == LESSTHAN_EQUALTO)));
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index 68787e3..e77d58e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -30,9 +30,9 @@ import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.impl.btree.BTreeDataRefNodeFinder;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.PartitionInfo;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.scan.expression.BinaryExpression;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.Expression;
@@ -402,13 +402,9 @@ public class FilterExpressionProcessor implements FilterProcessor {
return new TrueConditionalResolverImpl(expression, false, false, tableIdentifier);
case EQUALS:
currentCondExpression = (BinaryConditionalExpression) expression;
- if (currentCondExpression.isSingleColumn()
- && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
- != DataTypes.ARRAY
- && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
- != DataTypes.STRUCT) {
-
- if (currentCondExpression.getColumnList().get(0).getCarbonColumn().isMeasure()) {
+ CarbonColumn column = currentCondExpression.getColumnList().get(0).getCarbonColumn();
+ if (currentCondExpression.isSingleColumn() && ! column.getDataType().isComplexType()) {
+ if (column.isMeasure()) {
if (FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getLeft())
&& FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getRight()) || (
FilterUtil.checkIfRightExpressionRequireEvaluation(currentCondExpression.getRight())
@@ -463,13 +459,9 @@ public class FilterExpressionProcessor implements FilterProcessor {
tableIdentifier, false);
case NOT_EQUALS:
currentCondExpression = (BinaryConditionalExpression) expression;
- if (currentCondExpression.isSingleColumn()
- && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
- != DataTypes.ARRAY
- && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
- != DataTypes.STRUCT) {
-
- if (currentCondExpression.getColumnList().get(0).getCarbonColumn().isMeasure()) {
+ column = currentCondExpression.getColumnList().get(0).getCarbonColumn();
+ if (currentCondExpression.isSingleColumn() && ! column.getDataType().isComplexType()) {
+ if (column.isMeasure()) {
if (FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getLeft())
&& FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getRight()) || (
FilterUtil.checkIfRightExpressionRequireEvaluation(currentCondExpression.getRight())
@@ -522,11 +514,8 @@ public class FilterExpressionProcessor implements FilterProcessor {
default:
if (expression instanceof ConditionalExpression) {
condExpression = (ConditionalExpression) expression;
- if (condExpression.isSingleColumn()
- && condExpression.getColumnList().get(0).getCarbonColumn().getDataType()
- != DataTypes.ARRAY
- && condExpression.getColumnList().get(0).getCarbonColumn().getDataType()
- != DataTypes.STRUCT) {
+ column = condExpression.getColumnList().get(0).getCarbonColumn();
+ if (condExpression.isSingleColumn() && ! column.isComplex()) {
condExpression = (ConditionalExpression) expression;
if ((condExpression.getColumnList().get(0).getCarbonColumn()
.hasEncoding(Encoding.DICTIONARY) && !condExpression.getColumnList().get(0)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 777f564..224a69f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -297,8 +297,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
getDimensionDefaultValue(dimColumnEvaluatorInfo);
continue;
}
- if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataTypes.ARRAY
- && dimColumnEvaluatorInfo.getDimension().getDataType() != DataTypes.STRUCT) {
+ if (!dimColumnEvaluatorInfo.getDimension().getDataType().isComplexType()) {
if (!dimColumnEvaluatorInfo.isDimensionExistsInCurrentSilce()) {
record[dimColumnEvaluatorInfo.getRowIndex()] =
dimColumnEvaluatorInfo.getDimension().getDefaultValue();
@@ -534,8 +533,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
@Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
- if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataTypes.ARRAY
- && dimColumnEvaluatorInfo.getDimension().getDataType() != DataTypes.STRUCT) {
+ if (!dimColumnEvaluatorInfo.getDimension().getDataType().isComplexType()) {
if (null == blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]]) {
blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]] =
blockChunkHolder.getDataBlock()
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
index bd78e08..d966802 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
@@ -167,11 +167,8 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
metadata);
} else if ((null != columnList.get(0).getDimension()) && (
- columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY) && !(
- columnList.get(0).getDimension().getDataType()
- == org.apache.carbondata.core.metadata.datatype.DataTypes.STRUCT
- || columnList.get(0).getDimension().getDataType()
- == org.apache.carbondata.core.metadata.datatype.DataTypes.ARRAY))) {
+ columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY) &&
+ ! columnList.get(0).getDimension().getDataType().isComplexType())) {
dimColResolvedFilterInfo.setFilterValues(FilterUtil
.getFilterListForAllValues(absoluteTableIdentifier, exp, columnList.get(0),
isIncludeFilter, tableProvider));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index aca2fd6..ea8bcb2 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -381,9 +381,9 @@ public abstract class AbstractDataFileFooterConverter {
case TIMESTAMP:
return DataTypes.TIMESTAMP;
case ARRAY:
- return DataTypes.ARRAY;
+ return DataTypes.createDefaultArrayType();
case STRUCT:
- return DataTypes.STRUCT;
+ return DataTypes.createDefaultStructType();
default:
return DataTypes.STRING;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifierTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifierTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifierTest.java
index ec9a19a..bd8b6bc 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifierTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifierTest.java
@@ -49,17 +49,17 @@ public class DictionaryColumnUniqueIdentifierTest {
ColumnIdentifier columnIdentifier2 = new ColumnIdentifier("1", properties, DataTypes.INT);
dictionaryColumnUniqueIdentifier1 =
new DictionaryColumnUniqueIdentifier(carbonTableIdentifier1, columnIdentifier,
- DataTypes.MAP, null);
+ DataTypes.STRING, null);
dictionaryColumnUniqueIdentifier2 =
new DictionaryColumnUniqueIdentifier(carbonTableIdentifier2, columnIdentifier2,
- DataTypes.MAP, null);
+ DataTypes.STRING, null);
dictionaryColumnUniqueIdentifier3 =
new DictionaryColumnUniqueIdentifier(carbonTableIdentifier2, columnIdentifier,
- DataTypes.MAP, null);
+ DataTypes.STRING, null);
}
@Test public void testToGetDataType() {
- assertEquals(dictionaryColumnUniqueIdentifier1.getDataType(), DataTypes.MAP);
+ assertEquals(dictionaryColumnUniqueIdentifier1.getDataType(), DataTypes.STRING);
}
@Test public void testForEqualsWithDifferentObjectsWithDifferentColumnIdentifier() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java
index 2020dd8..9c43553 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java
@@ -296,7 +296,7 @@ public class SegmentPropertiesTest extends TestCase {
dimColumn.setColumnar(true);
dimColumn.setColumnName("IMEI5");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
- dimColumn.setDataType(DataTypes.ARRAY);
+ dimColumn.setDataType(DataTypes.createDefaultArrayType());
dimColumn.setDimensionColumn(true);
List<Encoding> encodeList =
new ArrayList<Encoding>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
index 1f9c712..9572627 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
@@ -179,7 +179,7 @@ public class SegmentPropertiesTestUtil {
dimColumn.setColumnar(true);
dimColumn.setColumnName("IMEI5");
dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
- dimColumn.setDataType(DataTypes.ARRAY);
+ dimColumn.setDataType(DataTypes.createDefaultArrayType());
dimColumn.setDimensionColumn(true);
List<Encoding> encodeList =
new ArrayList<Encoding>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactoryUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactoryUnitTest.java b/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactoryUnitTest.java
index c077eeb..2b73b86 100644
--- a/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactoryUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactoryUnitTest.java
@@ -37,7 +37,7 @@ public class DirectDictionaryKeyGeneratorFactoryUnitTest {
@Test public void testGetDirectDictionaryGeneratorReturnNull() throws Exception {
DirectDictionaryGenerator result =
- DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(DataTypes.ARRAY);
+ DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(DataTypes.STRING);
Assert.assertNull(result);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
index 42c0ad6..b8e4eea 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
@@ -23,8 +23,6 @@ import java.util.Map;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.datatype.DecimalType;
-import org.apache.carbondata.core.metadata.datatype.IntType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.SchemaEvolution;
import org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry;
@@ -1025,7 +1023,7 @@ public class ThriftWrapperSchemaConverterImplTest {
}
@Mock public DataType getDataType() {
- return DataTypes.ARRAY;
+ return DataTypes.createDefaultArrayType();
}
@Mock public String getColumnName() {
@@ -1099,7 +1097,7 @@ public class ThriftWrapperSchemaConverterImplTest {
}
@Mock public DataType getDataType() {
- return DataTypes.STRUCT;
+ return DataTypes.createDefaultStructType();
}
@Mock public String getColumnName() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java
index 4979038..0aed719 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java
@@ -83,7 +83,7 @@ public class CarbonTableWithComplexTypesTest extends TestCase {
structColumn.setColumnar(true);
structColumn.setColumnName("mobile");
structColumn.setColumnUniqueId(UUID.randomUUID().toString());
- structColumn.setDataType(DataTypes.STRUCT);
+ structColumn.setDataType(DataTypes.createDefaultStructType());
structColumn.setDimensionColumn(true);
structColumn.setEncodingList(encodeList);
structColumn.setNumberOfChild(2);
@@ -103,7 +103,7 @@ public class CarbonTableWithComplexTypesTest extends TestCase {
arrayColumn.setColumnar(true);
arrayColumn.setColumnName("mobile.val");
arrayColumn.setColumnUniqueId(UUID.randomUUID().toString());
- arrayColumn.setDataType(DataTypes.ARRAY);
+ arrayColumn.setDataType(DataTypes.createDefaultArrayType());
arrayColumn.setDimensionColumn(true);
arrayColumn.setEncodingList(encodeList);
arrayColumn.setNumberOfChild(1);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index 56c59e1..f9b5ec8 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -515,7 +515,7 @@ public class CarbonUtilTest {
}
@Test public void testForHasComplexDataTypes() {
- assertTrue(DataTypes.ARRAY.isComplexType());
+ assertTrue(DataTypes.createDefaultArrayType().isComplexType());
assertTrue(!DataTypes.DATE.isComplexType());
}
@@ -581,7 +581,7 @@ public class CarbonUtilTest {
column1.setDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
column2Schema.setColumnName("Column2");
- column2Schema.setDataType(DataTypes.ARRAY);
+ column2Schema.setDataType(DataTypes.createDefaultArrayType());
column2.setDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
QueryDimension[] queryDimensions = { column1, column2 };
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparatorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparatorTest.java b/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparatorTest.java
index cff1291..0775a3d 100644
--- a/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparatorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparatorTest.java
@@ -68,7 +68,7 @@ public class CarbonDictionarySortInfoPreparatorTest {
};
CarbonDictionarySortInfo carbonDictionarySortInfo = carbonDictionarySortInfoPreparator
- .getDictionarySortInfo(newDistinctValues, dictionary, DataTypes.ARRAY);
+ .getDictionarySortInfo(newDistinctValues, dictionary, DataTypes.STRING);
int expectedGetSortIndexValue = 1;
int expectedGetSortInvertedIndexLength = 3;
int actualGetSortIndexValue = carbonDictionarySortInfo.getSortIndex().get(0);
@@ -87,7 +87,7 @@ public class CarbonDictionarySortInfoPreparatorTest {
newDistinctValues.add("xyz");
Dictionary dictionary = null;
CarbonDictionarySortInfo carbonDictionarySortInfo = carbonDictionarySortInfoPreparator
- .getDictionarySortInfo(newDistinctValues, dictionary, DataTypes.ARRAY);
+ .getDictionarySortInfo(newDistinctValues, dictionary, DataTypes.createDefaultArrayType());
int expectedGetSortIndexValue = 1;
int expectedGetSortInvertedIndexLength = 2;
int actualGetSortIndexValue = carbonDictionarySortInfo.getSortIndex().get(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormat.java
index b10bc8b..eb07f7e 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormat.java
@@ -59,10 +59,10 @@ public class CarbonStreamInputFormat extends FileInputFormat<Void, Object> {
GenericQueryType[] queryTypes = new GenericQueryType[carbonColumns.length];
for (int i = 0; i < carbonColumns.length; i++) {
if (carbonColumns[i].isComplex()) {
- if (carbonColumns[i].getDataType() == DataTypes.ARRAY) {
+ if (DataTypes.isArrayType(carbonColumns[i].getDataType())) {
queryTypes[i] = new ArrayQueryType(carbonColumns[i].getColName(),
carbonColumns[i].getColName(), i);
- } else if (carbonColumns[i].getDataType() == DataTypes.STRUCT) {
+ } else if (DataTypes.isStructType(carbonColumns[i].getDataType())) {
queryTypes[i] = new StructQueryType(carbonColumns[i].getColName(),
carbonColumns[i].getColName(), i);
} else {
@@ -84,11 +84,11 @@ public class CarbonStreamInputFormat extends FileInputFormat<Void, Object> {
CarbonDimension child = dimension.getListOfChildDimensions().get(i);
DataType dataType = child.getDataType();
GenericQueryType queryType = null;
- if (dataType == DataTypes.ARRAY) {
+ if (DataTypes.isArrayType(dataType)) {
queryType =
new ArrayQueryType(child.getColName(), dimension.getColName(), ++parentBlockIndex);
- } else if (dataType == DataTypes.STRUCT) {
+ } else if (DataTypes.isStructType(dataType)) {
queryType =
new StructQueryType(child.getColName(), dimension.getColName(), ++parentBlockIndex);
parentQueryType.addChildren(queryType);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
index 1ff0fa7..4e3e6cf 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
@@ -35,6 +35,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
@@ -75,12 +76,9 @@ import org.apache.spark.sql.catalyst.expressions.GenericInternalRow;
import org.apache.spark.sql.execution.vectorized.ColumnVector;
import org.apache.spark.sql.execution.vectorized.ColumnarBatch;
import org.apache.spark.sql.types.CalendarIntervalType;
-import org.apache.spark.sql.types.DataType;
-import org.apache.spark.sql.types.DateType;
import org.apache.spark.sql.types.Decimal;
import org.apache.spark.sql.types.DecimalType;
import org.apache.spark.sql.types.StructType;
-import org.apache.spark.sql.types.TimestampType;
import org.apache.spark.unsafe.types.CalendarInterval;
import org.apache.spark.unsafe.types.UTF8String;
@@ -95,7 +93,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
private CarbonTable carbonTable;
private CarbonColumn[] storageColumns;
private boolean[] isRequired;
- private int[] measureDataTypes;
+ private DataType[] measureDataTypes;
private int dimensionCount;
private int measureCount;
@@ -171,9 +169,9 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
.getDirectDictionaryGenerator(storageColumns[i].getDataType());
}
}
- measureDataTypes = new int[measureCount];
+ measureDataTypes = new DataType[measureCount];
for (int i = 0; i < measureCount; i++) {
- measureDataTypes[i] = storageColumns[dimensionCount + i].getDataType().getId();
+ measureDataTypes[i] = storageColumns[dimensionCount + i].getDataType();
}
// decode data
@@ -518,7 +516,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
}
}
// measure
- int dataType;
+ DataType dataType;
for (int msrCount = 0; msrCount < measureCount; msrCount++, colCount++) {
if (nullBitSet.get(colCount)) {
if (isFilterRequired[colCount]) {
@@ -529,7 +527,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
}
} else {
dataType = measureDataTypes[msrCount];
- if (dataType == DataTypes.BOOLEAN_TYPE_ID) {
+ if (dataType == DataTypes.BOOLEAN) {
if (isRequired[colCount]) {
boolean v = input.readBoolean();
if (isFilterRequired[colCount]) {
@@ -541,7 +539,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
} else {
input.skipBytes(1);
}
- } else if (dataType == DataTypes.SHORT_TYPE_ID) {
+ } else if (dataType == DataTypes.SHORT) {
if (isRequired[colCount]) {
short v = input.readShort();
if (isFilterRequired[colCount]) {
@@ -553,7 +551,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
} else {
input.skipBytes(2);
}
- } else if (dataType == DataTypes.INT_TYPE_ID) {
+ } else if (dataType == DataTypes.INT) {
if (isRequired[colCount]) {
int v = input.readInt();
if (isFilterRequired[colCount]) {
@@ -565,7 +563,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
} else {
input.skipBytes(4);
}
- } else if (dataType == DataTypes.LONG_TYPE_ID) {
+ } else if (dataType == DataTypes.LONG) {
if (isRequired[colCount]) {
long v = input.readLong();
if (isFilterRequired[colCount]) {
@@ -577,7 +575,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
} else {
input.skipBytes(8);
}
- } else if (dataType == DataTypes.DOUBLE_TYPE_ID) {
+ } else if (dataType == DataTypes.DOUBLE) {
if (isRequired[colCount]) {
double v = input.readDouble();
if (isFilterRequired[colCount]) {
@@ -589,7 +587,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
} else {
input.skipBytes(8);
}
- } else if (dataType == DataTypes.DECIMAL_TYPE_ID) {
+ } else if (DataTypes.isDecimal(dataType)) {
int len = input.readShort();
if (isRequired[colCount]) {
BigDecimal v = DataTypeUtil.byteToBigDecimal(input.readBytes(len));
@@ -611,7 +609,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
for (int i = 0; i < projection.length; i++) {
Object value = outputValues[i];
ColumnVector col = columnarBatch.column(i);
- DataType t = col.dataType();
+ org.apache.spark.sql.types.DataType t = col.dataType();
if (null == value) {
col.putNull(rowId);
} else {
@@ -632,7 +630,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
} else if (t == org.apache.spark.sql.types.DataTypes.StringType) {
UTF8String v = (UTF8String) value;
col.putByteArray(rowId, v.getBytes());
- } else if (t instanceof DecimalType) {
+ } else if (t instanceof org.apache.spark.sql.types.DecimalType) {
DecimalType dt = (DecimalType)t;
Decimal d = (Decimal) value;
if (dt.precision() <= Decimal.MAX_INT_DIGITS()) {
@@ -648,9 +646,9 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
CalendarInterval c = (CalendarInterval) value;
col.getChildColumn(0).putInt(rowId, c.months);
col.getChildColumn(1).putLong(rowId, c.microseconds);
- } else if (t instanceof DateType) {
+ } else if (t instanceof org.apache.spark.sql.types.DateType) {
col.putInt(rowId, (int) value);
- } else if (t instanceof TimestampType) {
+ } else if (t instanceof org.apache.spark.sql.types.TimestampType) {
col.putLong(rowId, (long) value);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
index 8d7a2e3..3c4b55c 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
@@ -30,6 +30,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.datastore.row.CarbonRow;
+import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
@@ -83,7 +84,7 @@ public class CarbonStreamRecordWriter extends RecordWriter<Void, Object> {
private boolean[] isNoDictionaryDimensionColumn;
private int dimensionWithComplexCount;
private int measureCount;
- private int[] measureDataTypes;
+ private DataType[] measureDataTypes;
private StreamBlockletWriter output = null;
// data write
@@ -129,10 +130,10 @@ public class CarbonStreamRecordWriter extends RecordWriter<Void, Object> {
dimensionWithComplexCount = configuration.getDimensionCount();
measureCount = configuration.getMeasureCount();
dataFields = configuration.getDataFields();
- measureDataTypes = new int[measureCount];
+ measureDataTypes = new DataType[measureCount];
for (int i = 0; i < measureCount; i++) {
measureDataTypes[i] =
- dataFields[dimensionWithComplexCount + i].getColumn().getDataType().getId();
+ dataFields[dimensionWithComplexCount + i].getColumn().getDataType();
}
// initialize parser and converter
@@ -211,22 +212,22 @@ public class CarbonStreamRecordWriter extends RecordWriter<Void, Object> {
}
}
// measure
- int dataType;
+ DataType dataType;
for (int msrCount = 0; msrCount < measureCount; msrCount++) {
columnValue = currentRow.getObject(dimCount + msrCount);
if (null != columnValue) {
dataType = measureDataTypes[msrCount];
- if (dataType == DataTypes.BOOLEAN_TYPE_ID) {
+ if (dataType == DataTypes.BOOLEAN) {
output.writeBoolean((boolean) columnValue);
- } else if (dataType == DataTypes.SHORT_TYPE_ID) {
+ } else if (dataType == DataTypes.SHORT) {
output.writeShort((short) columnValue);
- } else if (dataType == DataTypes.INT_TYPE_ID) {
+ } else if (dataType == DataTypes.INT) {
output.writeInt((int) columnValue);
- } else if (dataType == DataTypes.LONG_TYPE_ID) {
+ } else if (dataType == DataTypes.LONG) {
output.writeLong((long) columnValue);
- } else if (dataType == DataTypes.DOUBLE_TYPE_ID) {
+ } else if (dataType == DataTypes.DOUBLE) {
output.writeDouble((double) columnValue);
- } else if (dataType == DataTypes.DECIMAL_TYPE_ID) {
+ } else if (DataTypes.isDecimal(dataType)) {
BigDecimal val = (BigDecimal) columnValue;
byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(val);
output.writeShort(bigDecimalInBytes.length);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
----------------------------------------------------------------------
diff --git a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
index 05ad93a..2a19271 100644
--- a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
+++ b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
@@ -137,9 +137,9 @@ public class CarbonDictionaryDecodeReadSupport<T> implements CarbonReadSupport<T
*/
private Writable createWritableObject(Object obj, CarbonColumn carbonColumn) throws IOException {
DataType dataType = carbonColumn.getDataType();
- if (dataType == DataTypes.STRUCT) {
+ if (DataTypes.isStructType(dataType)) {
return createStruct(obj, carbonColumn);
- } else if (dataType == DataTypes.ARRAY) {
+ } else if (DataTypes.isArrayType(dataType)) {
return createArray(obj, carbonColumn);
} else {
return createWritablePrimitive(obj, carbonColumn);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index 2043ecf..7abb737 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -19,13 +19,14 @@ package org.apache.carbondata.spark.util
import java.nio.charset.Charset
import java.text.SimpleDateFormat
+import java.util
import org.apache.spark.sql._
import org.apache.spark.sql.execution.command.DataTypeInfo
import org.apache.spark.sql.types._
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.datatype.{DataType => CarbonDataType, DataTypes => CarbonDataTypes, DecimalType => CarbonDecimalType}
+import org.apache.carbondata.core.metadata.datatype.{DataType => CarbonDataType, DataTypes => CarbonDataTypes, DecimalType => CarbonDecimalType, StructField => CarbonStructField}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn
object CarbonScalaUtil {
@@ -40,8 +41,17 @@ object CarbonScalaUtil {
case DateType => CarbonDataTypes.DATE
case BooleanType => CarbonDataTypes.BOOLEAN
case TimestampType => CarbonDataTypes.TIMESTAMP
- case ArrayType(_, _) => CarbonDataTypes.ARRAY
- case StructType(_) => CarbonDataTypes.STRUCT
+ case ArrayType(elementType, _) =>
+ CarbonDataTypes.createArrayType(CarbonScalaUtil.convertSparkToCarbonDataType(elementType))
+ case StructType(fields) =>
+ val carbonFields = new util.ArrayList[CarbonStructField]
+ fields.map { field =>
+ carbonFields.add(
+ new CarbonStructField(
+ field.name,
+ CarbonScalaUtil.convertSparkToCarbonDataType(field.dataType)))
+ }
+ CarbonDataTypes.createStructType(carbonFields)
case NullType => CarbonDataTypes.NULL
case decimal: DecimalType =>
CarbonDataTypes.createDecimalType(decimal.precision, decimal.scale)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
index 0460e30..6cf7298 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
@@ -41,8 +41,8 @@ object DataTypeConverterUtil {
case FIXED_DECIMAL(_, _) => DataTypes.createDefaultDecimalType
case "timestamp" => DataTypes.TIMESTAMP
case "date" => DataTypes.DATE
- case "array" => DataTypes.ARRAY
- case "struct" => DataTypes.STRUCT
+ case "array" => DataTypes.createDefaultArrayType
+ case "struct" => DataTypes.createDefaultStructType
case _ => convertToCarbonTypeForSpark2(dataType)
}
}
@@ -66,9 +66,9 @@ object DataTypeConverterUtil {
case "datetype" => DataTypes.DATE
case others =>
if (others != null && others.startsWith("arraytype")) {
- DataTypes.ARRAY
+ DataTypes.createDefaultArrayType()
} else if (others != null && others.startsWith("structtype")) {
- DataTypes.STRUCT
+ DataTypes.createDefaultStructType()
} else if (others != null && others.startsWith("char")) {
DataTypes.STRING
} else if (others != null && others.startsWith("varchar")) {
@@ -82,6 +82,10 @@ object DataTypeConverterUtil {
def convertToString(dataType: DataType): String = {
if (DataTypes.isDecimal(dataType)) {
"decimal"
+ } else if (DataTypes.isArrayType(dataType)) {
+ "array"
+ } else if (DataTypes.isStructType(dataType)) {
+ "struct"
} else {
dataType match {
case DataTypes.BOOLEAN => "boolean"
@@ -93,8 +97,6 @@ object DataTypeConverterUtil {
case DataTypes.FLOAT => "double"
case DataTypes.TIMESTAMP => "timestamp"
case DataTypes.DATE => "date"
- case DataTypes.ARRAY => "array"
- case DataTypes.STRUCT => "struct"
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index ddc4763..840e8ae 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -259,17 +259,16 @@ object GlobalDictionaryUtil {
case None =>
None
case Some(dim) =>
- dim.getDataType match {
- case DataTypes.ARRAY =>
- val arrDim = ArrayParser(dim, format)
- generateParserForChildrenDimension(dim, format, mapColumnValuesWithId, arrDim)
- Some(arrDim)
- case DataTypes.STRUCT =>
- val stuDim = StructParser(dim, format)
- generateParserForChildrenDimension(dim, format, mapColumnValuesWithId, stuDim)
- Some(stuDim)
- case _ =>
- Some(PrimitiveParser(dim, mapColumnValuesWithId.get(dim.getColumnId)))
+ if (DataTypes.isArrayType(dim.getDataType)) {
+ val arrDim = ArrayParser(dim, format)
+ generateParserForChildrenDimension(dim, format, mapColumnValuesWithId, arrDim)
+ Some(arrDim)
+ } else if (DataTypes.isStructType(dim.getDataType)) {
+ val stuDim = StructParser(dim, format)
+ generateParserForChildrenDimension(dim, format, mapColumnValuesWithId, stuDim)
+ Some(stuDim)
+ } else {
+ Some(PrimitiveParser(dim, mapColumnValuesWithId.get(dim.getColumnId)))
}
}
}
@@ -477,14 +476,14 @@ object GlobalDictionaryUtil {
val children = preDictDimension.getListOfChildDimensions.asScala.toArray
// for Array, user set ArrayFiled: path, while ArrayField has a child Array.val
val currentColName = {
- preDictDimension.getDataType match {
- case DataTypes.ARRAY =>
- if (children(0).isComplex) {
- "val." + colName.substring(middleDimName.length + 1)
- } else {
- "val"
- }
- case _ => colName.substring(middleDimName.length + 1)
+ if (DataTypes.isArrayType(preDictDimension.getDataType)) {
+ if (children(0).isComplex) {
+ "val." + colName.substring(middleDimName.length + 1)
+ } else {
+ "val"
+ }
+ } else {
+ colName.substring(middleDimName.length + 1)
}
}
setPredefineDict(carbonLoadModel, children, table, currentColName,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
index 91c07de..18f76d1 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
@@ -392,6 +392,12 @@ object CarbonDictionaryDecoder {
} else {
DecimalType(precision, scale)
}
+ } else if (CarbonDataTypes.isArrayType(carbonDimension.getDataType)) {
+ CarbonMetastoreTypes
+ .toDataType(s"array<${ relation.getArrayChildren(carbonDimension.getColName) }>")
+ } else if (CarbonDataTypes.isStructType(carbonDimension.getDataType)) {
+ CarbonMetastoreTypes
+ .toDataType(s"struct<${ relation.getStructChildren(carbonDimension.getColName) }>")
} else {
carbonDimension.getDataType match {
case CarbonDataTypes.STRING => StringType
@@ -402,12 +408,6 @@ object CarbonDictionaryDecoder {
case CarbonDataTypes.BOOLEAN => BooleanType
case CarbonDataTypes.TIMESTAMP => TimestampType
case CarbonDataTypes.DATE => DateType
- case CarbonDataTypes.STRUCT =>
- CarbonMetastoreTypes
- .toDataType(s"struct<${ relation.getStructChildren(carbonDimension.getColName) }>")
- case CarbonDataTypes.ARRAY =>
- CarbonMetastoreTypes
- .toDataType(s"array<${ relation.getArrayChildren(carbonDimension.getColName) }>")
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
index 193d3e6..5dbd177 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
@@ -111,7 +111,7 @@ public class FieldEncoderFactory {
CarbonTableIdentifier carbonTableIdentifier, DictionaryClient client, Boolean useOnePass,
String storePath, Map<Object, Integer> localCache) {
DataType dataType = carbonColumn.getDataType();
- if (dataType == DataTypes.ARRAY) {
+ if (DataTypes.isArrayType(dataType)) {
List<CarbonDimension> listOfChildDimensions =
((CarbonDimension) carbonColumn).getListOfChildDimensions();
// Create array parser with complex delimiter
@@ -123,7 +123,7 @@ public class FieldEncoderFactory {
client, useOnePass, storePath, localCache));
}
return arrayDataType;
- } else if (dataType == DataTypes.STRUCT) {
+ } else if (DataTypes.isStructType(dataType)) {
List<CarbonDimension> dimensions =
((CarbonDimension) carbonColumn).getListOfChildDimensions();
// Create struct parser with complex delimiter
@@ -135,7 +135,7 @@ public class FieldEncoderFactory {
client, useOnePass, storePath, localCache));
}
return structDataType;
- } else if (dataType == DataTypes.MAP) {
+ } else if (DataTypes.isMapType(dataType)) {
throw new UnsupportedOperationException("Complex type Map is not supported yet");
} else {
return new PrimitiveDataType(carbonColumn.getColName(), parentName,