You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/06/28 21:23:45 UTC
[1/2] incubator-carbondata git commit: Supported tinyint datatype in
carbondata
Repository: incubator-carbondata
Updated Branches:
refs/heads/master 02fcb1969 -> 634bdeefc
Supported tinyint datatype in carbondata
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/a7437571
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/a7437571
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/a7437571
Branch: refs/heads/master
Commit: a74375719a5c09249bb742bf7faef33b02bcf252
Parents: 63d3284
Author: ravipesala <ra...@gmail.com>
Authored: Tue Jun 28 21:53:58 2016 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Tue Jun 28 21:53:58 2016 +0530
----------------------------------------------------------------------
.../cache/dictionary/ColumnDictionaryInfo.java | 2 +
.../ThriftWrapperSchemaConverterImpl.java | 4 ++
.../core/carbon/metadata/datatype/DataType.java | 1 +
.../core/constants/CarbonCommonConstants.java | 1 +
.../core/util/DataFileFooterConverter.java | 2 +
.../org/carbondata/core/util/DataTypeUtil.java | 4 ++
.../sortindex/CarbonDictionarySortModel.java | 1 +
.../carbondata/scan/expression/DataType.java | 3 +-
.../scan/expression/ExpressionResult.java | 59 +++++++++++++++++++-
.../expression/arithmetic/AddExpression.java | 3 +
.../expression/arithmetic/DivideExpression.java | 3 +
.../arithmetic/MultiplyExpression.java | 3 +
.../arithmetic/SubstractExpression.java | 3 +
.../conditional/EqualToExpression.java | 3 +
.../GreaterThanEqualToExpression.java | 3 +
.../conditional/GreaterThanExpression.java | 3 +
.../expression/conditional/InExpression.java | 3 +
.../conditional/LessThanEqualToExpression.java | 3 +
.../conditional/LessThanExpression.java | 3 +
.../conditional/NotEqualsExpression.java | 3 +
.../expression/conditional/NotInExpression.java | 3 +
.../org/carbondata/scan/filter/FilterUtil.java | 4 +-
.../org/carbondata/scan/util/DataTypeUtil.java | 54 +-----------------
format/src/main/thrift/schema.thrift | 11 ++--
.../spark/sql/CarbonDatasourceRelation.scala | 1 +
.../spark/sql/CarbonDictionaryDecoder.scala | 1 +
.../execution/command/carbonTableSchema.scala | 2 +
.../spark/sql/hive/CarbonMetastoreCatalog.scala | 1 +
.../scala/org/carbondata/spark/package.scala | 2 +-
.../carbondata/spark/util/CarbonScalaUtil.scala | 46 +--------------
30 files changed, 130 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java b/core/src/main/java/org/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
index c9dff58..5c39eaa 100644
--- a/core/src/main/java/org/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
+++ b/core/src/main/java/org/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
@@ -223,6 +223,8 @@ public class ColumnDictionaryInfo extends AbstractColumnDictionaryInfo {
DataType dataType) {
try {
switch (dataType) {
+ case SHORT:
+ return Short.compare((Short.parseShort(dictionaryVal)), (Short.parseShort(memberVal)));
case INT:
return Integer.compare((Integer.parseInt(dictionaryVal)), (Integer.parseInt(memberVal)));
case DOUBLE:
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index 249d2a5..51b269a 100644
--- a/core/src/main/java/org/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -123,6 +123,8 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
return org.carbondata.format.DataType.STRING;
case INT:
return org.carbondata.format.DataType.INT;
+ case SHORT:
+ return org.carbondata.format.DataType.SHORT;
case LONG:
return org.carbondata.format.DataType.LONG;
case DOUBLE:
@@ -285,6 +287,8 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
return DataType.STRING;
case INT:
return DataType.INT;
+ case SHORT:
+ return DataType.SHORT;
case LONG:
return DataType.LONG;
case DOUBLE:
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/core/carbon/metadata/datatype/DataType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/metadata/datatype/DataType.java b/core/src/main/java/org/carbondata/core/carbon/metadata/datatype/DataType.java
index 7586022..9f2be8a 100644
--- a/core/src/main/java/org/carbondata/core/carbon/metadata/datatype/DataType.java
+++ b/core/src/main/java/org/carbondata/core/carbon/metadata/datatype/DataType.java
@@ -22,6 +22,7 @@ public enum DataType {
STRING,
INT,
+ SHORT,
LONG,
DOUBLE,
BOOLEAN,
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
index 13fce9f..96b0b39 100644
--- a/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
@@ -707,6 +707,7 @@ public final class CarbonCommonConstants {
public static final String COLUMNAR = "columnar";
public static final String INTEGER = "Integer";
+ public static final String SHORT = "Short";
public static final String NUMERIC = "Numeric";
public static final String TIMESTAMP = "Timestamp";
public static final String ARRAY = "ARRAY";
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java b/core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java
index 6165731..6641c18 100644
--- a/core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java
+++ b/core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java
@@ -301,6 +301,8 @@ class DataFileFooterConverter {
switch (dataTypeThrift) {
case STRING:
return DataType.STRING;
+ case SHORT:
+ return DataType.SHORT;
case INT:
return DataType.INT;
case LONG:
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java
index d3cd120..68bca6c 100644
--- a/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java
@@ -120,6 +120,9 @@ public final class DataTypeUtil {
case "INT":
dataType = DataType.INT;
break;
+ case "SHORT":
+ dataType = DataType.SHORT;
+ break;
case "LONG":
dataType = DataType.LONG;
break;
@@ -158,6 +161,7 @@ public final class DataTypeUtil {
try {
switch (actualDataType) {
case INT:
+ case SHORT:
case LONG:
case DOUBLE:
case DECIMAL:
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java b/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java
index e518107..8e66cc7 100644
--- a/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java
+++ b/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java
@@ -64,6 +64,7 @@ public class CarbonDictionarySortModel implements Comparable<CarbonDictionarySor
*/
@Override public int compareTo(CarbonDictionarySortModel o) {
switch (dataType) {
+ case SHORT:
case INT:
case LONG:
case DOUBLE:
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/DataType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/DataType.java b/core/src/main/java/org/carbondata/scan/expression/DataType.java
index fc9d0bf..062fad1 100644
--- a/core/src/main/java/org/carbondata/scan/expression/DataType.java
+++ b/core/src/main/java/org/carbondata/scan/expression/DataType.java
@@ -21,7 +21,8 @@ package org.carbondata.scan.expression;
public enum DataType {
StringType(0), DateType(1), TimestampType(2), BooleanType(1), IntegerType(3), FloatType(
- 4), LongType(5), DoubleType(6), NullType(7), DecimalType(8), ArrayType(9), StructType(10);
+ 4), LongType(5), DoubleType(6), NullType(7), DecimalType(8), ArrayType(9), StructType(
+ 10), ShortType(11);
private int presedenceOrder;
private DataType(int value) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/ExpressionResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/ExpressionResult.java b/core/src/main/java/org/carbondata/scan/expression/ExpressionResult.java
index decbdc0..a86052c 100644
--- a/core/src/main/java/org/carbondata/scan/expression/ExpressionResult.java
+++ b/core/src/main/java/org/carbondata/scan/expression/ExpressionResult.java
@@ -73,6 +73,8 @@ public class ExpressionResult implements Comparable<ExpressionResult> {
throw new FilterUnsupportedException(e);
}
+ case ShortType:
+ return ((Short) value).intValue();
case IntegerType:
case DoubleType:
@@ -100,6 +102,48 @@ public class ExpressionResult implements Comparable<ExpressionResult> {
}
}
+ public Short getShort() throws FilterUnsupportedException {
+ if (value == null) {
+ return null;
+ }
+ try {
+ switch (this.getDataType()) {
+ case StringType:
+ try {
+ return Short.parseShort(value.toString());
+ } catch (NumberFormatException e) {
+ throw new FilterUnsupportedException(e);
+ }
+ case ShortType:
+ case IntegerType:
+ case DoubleType:
+
+ if (value instanceof Double) {
+ return ((Double) value).shortValue();
+ } else if (value instanceof Integer) {
+ return ((Integer) value).shortValue();
+ }
+ return (Short) value;
+
+ case TimestampType:
+
+ if (value instanceof Timestamp) {
+ return (short) (((Timestamp) value).getTime() % 1000);
+ } else {
+ return (Short) value;
+ }
+
+ default:
+ throw new FilterUnsupportedException(
+ "Cannot convert" + this.getDataType().name() + " to integer type value");
+ }
+
+ } catch (ClassCastException e) {
+ throw new FilterUnsupportedException(
+ "Cannot convert" + this.getDataType().name() + " to Integer type value");
+ }
+ }
+
public String getString() {
if (value == null) {
return null;
@@ -132,7 +176,8 @@ public class ExpressionResult implements Comparable<ExpressionResult> {
} catch (NumberFormatException e) {
throw new FilterUnsupportedException(e);
}
-
+ case ShortType:
+ return ((Short) value).doubleValue();
case IntegerType:
return ((Integer) value).doubleValue();
case LongType:
@@ -168,7 +213,8 @@ public class ExpressionResult implements Comparable<ExpressionResult> {
} catch (NumberFormatException e) {
throw new FilterUnsupportedException(e);
}
-
+ case ShortType:
+ return ((Short) value).longValue();
case IntegerType:
return (Long) value;
case LongType:
@@ -205,7 +251,8 @@ public class ExpressionResult implements Comparable<ExpressionResult> {
} catch (NumberFormatException e) {
throw new FilterUnsupportedException(e);
}
-
+ case ShortType:
+ return new BigDecimal((short) value);
case IntegerType:
return new BigDecimal((int) value);
case LongType:
@@ -249,6 +296,8 @@ public class ExpressionResult implements Comparable<ExpressionResult> {
throw new FilterUnsupportedException(
"Cannot convert" + this.getDataType().name() + " to Time/Long type value");
}
+ case ShortType:
+ return ((Short) value).longValue();
case IntegerType:
case LongType:
return (Long) value;
@@ -354,6 +403,9 @@ public class ExpressionResult implements Comparable<ExpressionResult> {
case StringType:
result = this.getString().equals(objToCompare.getString());
break;
+ case ShortType:
+ result = this.getShort().equals(objToCompare.getShort());
+ break;
case IntegerType:
result = this.getInt().equals(objToCompare.getInt());
break;
@@ -381,6 +433,7 @@ public class ExpressionResult implements Comparable<ExpressionResult> {
@Override public int compareTo(ExpressionResult o) {
try {
switch (o.dataType) {
+ case ShortType:
case IntegerType:
case LongType:
case DoubleType:
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/arithmetic/AddExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/arithmetic/AddExpression.java b/core/src/main/java/org/carbondata/scan/expression/arithmetic/AddExpression.java
index e7ce159..62bab78 100644
--- a/core/src/main/java/org/carbondata/scan/expression/arithmetic/AddExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/arithmetic/AddExpression.java
@@ -55,6 +55,9 @@ public class AddExpression extends BinaryArithmeticExpression {
case DoubleType:
addExprRightRes.set(DataType.DoubleType, val1.getDouble() + val2.getDouble());
break;
+ case ShortType:
+ addExprRightRes.set(DataType.ShortType, val1.getShort() + val2.getShort());
+ break;
case IntegerType:
addExprRightRes.set(DataType.IntegerType, val1.getInt() + val2.getInt());
break;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/arithmetic/DivideExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/arithmetic/DivideExpression.java b/core/src/main/java/org/carbondata/scan/expression/arithmetic/DivideExpression.java
index f4df604..d98b301 100644
--- a/core/src/main/java/org/carbondata/scan/expression/arithmetic/DivideExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/arithmetic/DivideExpression.java
@@ -54,6 +54,9 @@ public class DivideExpression extends BinaryArithmeticExpression {
case DoubleType:
divideExprRightRes.set(DataType.DoubleType, val1.getDouble() / val2.getDouble());
break;
+ case ShortType:
+ divideExprRightRes.set(DataType.ShortType, val1.getShort() / val2.getShort());
+ break;
case IntegerType:
divideExprRightRes.set(DataType.IntegerType, val1.getInt() / val2.getInt());
break;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/arithmetic/MultiplyExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/arithmetic/MultiplyExpression.java b/core/src/main/java/org/carbondata/scan/expression/arithmetic/MultiplyExpression.java
index 3347250..7f8d8aa 100644
--- a/core/src/main/java/org/carbondata/scan/expression/arithmetic/MultiplyExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/arithmetic/MultiplyExpression.java
@@ -55,6 +55,9 @@ public class MultiplyExpression extends BinaryArithmeticExpression {
case DoubleType:
multiplyExprRightRes.set(DataType.DoubleType, val1.getDouble() * val2.getDouble());
break;
+ case ShortType:
+ multiplyExprRightRes.set(DataType.ShortType, val1.getShort() * val2.getShort());
+ break;
case IntegerType:
multiplyExprRightRes.set(DataType.IntegerType, val1.getInt() * val2.getInt());
break;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/arithmetic/SubstractExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/arithmetic/SubstractExpression.java b/core/src/main/java/org/carbondata/scan/expression/arithmetic/SubstractExpression.java
index 35fec1c..0e7ab2e 100644
--- a/core/src/main/java/org/carbondata/scan/expression/arithmetic/SubstractExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/arithmetic/SubstractExpression.java
@@ -55,6 +55,9 @@ public class SubstractExpression extends BinaryArithmeticExpression {
case DoubleType:
subtractExprRightRes.set(DataType.DoubleType, val1.getDouble() - val2.getDouble());
break;
+ case ShortType:
+ subtractExprRightRes.set(DataType.ShortType, val1.getShort() - val2.getShort());
+ break;
case IntegerType:
subtractExprRightRes.set(DataType.IntegerType, val1.getInt() - val2.getInt());
break;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/conditional/EqualToExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/conditional/EqualToExpression.java b/core/src/main/java/org/carbondata/scan/expression/conditional/EqualToExpression.java
index 7b2a32d..80ad32e 100644
--- a/core/src/main/java/org/carbondata/scan/expression/conditional/EqualToExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/conditional/EqualToExpression.java
@@ -61,6 +61,9 @@ public class EqualToExpression extends BinaryConditionalExpression {
case StringType:
result = val1.getString().equals(val2.getString());
break;
+ case ShortType:
+ result = val1.getShort().equals(val2.getShort());
+ break;
case IntegerType:
result = val1.getInt().equals(val2.getInt());
break;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanEqualToExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanEqualToExpression.java b/core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanEqualToExpression.java
index bb00467..4da578b 100644
--- a/core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanEqualToExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanEqualToExpression.java
@@ -52,6 +52,9 @@ public class GreaterThanEqualToExpression extends BinaryConditionalExpression {
case StringType:
result = elRes.getString().compareTo(erRes.getString()) >= 0;
break;
+ case ShortType:
+ result = elRes.getShort() >= (erRes.getShort());
+ break;
case IntegerType:
result = elRes.getInt() >= (erRes.getInt());
break;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanExpression.java b/core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanExpression.java
index f1bde04..5c3bd56 100644
--- a/core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanExpression.java
@@ -58,6 +58,9 @@ public class GreaterThanExpression extends BinaryConditionalExpression {
case DoubleType:
result = exprLeftRes.getDouble() > (exprRightRes.getDouble());
break;
+ case ShortType:
+ result = exprLeftRes.getShort() > (exprRightRes.getShort());
+ break;
case IntegerType:
result = exprLeftRes.getInt() > (exprRightRes.getInt());
break;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/conditional/InExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/conditional/InExpression.java b/core/src/main/java/org/carbondata/scan/expression/conditional/InExpression.java
index 6574d3e..ddb35af 100644
--- a/core/src/main/java/org/carbondata/scan/expression/conditional/InExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/conditional/InExpression.java
@@ -59,6 +59,9 @@ public class InExpression extends BinaryConditionalExpression {
case StringType:
val = new ExpressionResult(val.getDataType(), expressionResVal.getString());
break;
+ case ShortType:
+ val = new ExpressionResult(val.getDataType(), expressionResVal.getShort());
+ break;
case IntegerType:
val = new ExpressionResult(val.getDataType(), expressionResVal.getInt());
break;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/conditional/LessThanEqualToExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/conditional/LessThanEqualToExpression.java b/core/src/main/java/org/carbondata/scan/expression/conditional/LessThanEqualToExpression.java
index f194157..c8aa6a7 100644
--- a/core/src/main/java/org/carbondata/scan/expression/conditional/LessThanEqualToExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/conditional/LessThanEqualToExpression.java
@@ -53,6 +53,9 @@ public class LessThanEqualToExpression extends BinaryConditionalExpression {
case StringType:
result = elRes.getString().compareTo(erRes.getString()) <= 0;
break;
+ case ShortType:
+ result = elRes.getShort() <= (erRes.getShort());
+ break;
case IntegerType:
result = elRes.getInt() <= (erRes.getInt());
break;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/conditional/LessThanExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/conditional/LessThanExpression.java b/core/src/main/java/org/carbondata/scan/expression/conditional/LessThanExpression.java
index b55d0cc..7e6580a 100644
--- a/core/src/main/java/org/carbondata/scan/expression/conditional/LessThanExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/conditional/LessThanExpression.java
@@ -56,6 +56,9 @@ public class LessThanExpression extends BinaryConditionalExpression {
case StringType:
result = elRes.getString().compareTo(erRes.getString()) < 0;
break;
+ case ShortType:
+ result = elRes.getShort() < (erRes.getShort());
+ break;
case IntegerType:
result = elRes.getInt() < (erRes.getInt());
break;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/conditional/NotEqualsExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/conditional/NotEqualsExpression.java b/core/src/main/java/org/carbondata/scan/expression/conditional/NotEqualsExpression.java
index df40f7f..4f31c07 100644
--- a/core/src/main/java/org/carbondata/scan/expression/conditional/NotEqualsExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/conditional/NotEqualsExpression.java
@@ -60,6 +60,9 @@ public class NotEqualsExpression extends BinaryConditionalExpression {
case StringType:
result = !val1.getString().equals(val2.getString());
break;
+ case ShortType:
+ result = val1.getShort().shortValue() != val2.getShort().shortValue();
+ break;
case IntegerType:
result = val1.getInt().intValue() != val2.getInt().intValue();
break;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/conditional/NotInExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/conditional/NotInExpression.java b/core/src/main/java/org/carbondata/scan/expression/conditional/NotInExpression.java
index 5777c4d..caa722a 100644
--- a/core/src/main/java/org/carbondata/scan/expression/conditional/NotInExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/conditional/NotInExpression.java
@@ -59,6 +59,9 @@ public class NotInExpression extends BinaryConditionalExpression {
case StringType:
val = new ExpressionResult(val.getDataType(), exprResVal.getString());
break;
+ case ShortType:
+ val = new ExpressionResult(val.getDataType(), exprResVal.getShort());
+ break;
case IntegerType:
val = new ExpressionResult(val.getDataType(), exprResVal.getInt());
break;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/filter/FilterUtil.java b/core/src/main/java/org/carbondata/scan/filter/FilterUtil.java
index 802682b..2f394f7 100644
--- a/core/src/main/java/org/carbondata/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/carbondata/scan/filter/FilterUtil.java
@@ -1012,8 +1012,9 @@ public final class FilterUtil {
DataType dataType) {
try {
switch (dataType) {
+ case SHORT:
+ return Short.compare((Short.parseShort(dictionaryVal)), (Short.parseShort(memberVal)));
case INT:
-
return Integer.compare((Integer.parseInt(dictionaryVal)), (Integer.parseInt(memberVal)));
case DOUBLE:
return Double
@@ -1095,6 +1096,7 @@ public final class FilterUtil {
String filterMember2, org.carbondata.scan.expression.DataType dataType) {
try {
switch (dataType) {
+ case ShortType:
case IntegerType:
case LongType:
case DoubleType:
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/util/DataTypeUtil.java b/core/src/main/java/org/carbondata/scan/util/DataTypeUtil.java
index 8f301ca..5654b1d 100644
--- a/core/src/main/java/org/carbondata/scan/util/DataTypeUtil.java
+++ b/core/src/main/java/org/carbondata/scan/util/DataTypeUtil.java
@@ -18,7 +18,6 @@
*/
package org.carbondata.scan.util;
-import java.math.BigDecimal;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
@@ -52,30 +51,20 @@ public class DataTypeUtil {
*/
public static Object getDataBasedOnDataType(String data, DataType actualDataType) {
- if (null == data) {
+ if (null == data || data.isEmpty()) {
return null;
}
try {
switch (actualDataType) {
case INT:
- if (data.isEmpty()) {
- return null;
- }
return Integer.parseInt(data);
+ case SHORT:
+ return Short.parseShort(data);
case DOUBLE:
- if (data.isEmpty()) {
- return null;
- }
return Double.parseDouble(data);
case LONG:
- if (data.isEmpty()) {
- return null;
- }
return Long.parseLong(data);
case TIMESTAMP:
- if (data.isEmpty()) {
- return null;
- }
SimpleDateFormat parser = new SimpleDateFormat(CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
@@ -88,9 +77,6 @@ public class DataTypeUtil {
return null;
}
case DECIMAL:
- if (data.isEmpty()) {
- return null;
- }
java.math.BigDecimal javaDecVal = new java.math.BigDecimal(data);
scala.math.BigDecimal scalaDecVal = new scala.math.BigDecimal(javaDecVal);
org.apache.spark.sql.types.Decimal decConverter =
@@ -138,38 +124,4 @@ public class DataTypeUtil {
}
- public static int compareBasedOnDatatYpe(Object data1, Object data2, DataType dataType) {
- switch (dataType) {
- case INT:
- return ((Integer) data1).compareTo((Integer) data2);
- case LONG:
- case TIMESTAMP:
- return ((Long) data1).compareTo((Long) data2);
- case DOUBLE:
- return ((Double) data1).compareTo((Double) data2);
- case DECIMAL:
- return ((BigDecimal) data1).compareTo((BigDecimal) data2);
- default:
- return ((String) data1).compareTo((String) data2);
- }
- }
-
- /**
- * below method is to check whether data type is of numeric type or not
- *
- * @param dataType data type
- * @return true if numeric data type
- */
- public boolean isNumericDatatype(DataType dataType) {
- switch (dataType) {
- case INT:
- case LONG:
- case DOUBLE:
- case DECIMAL:
- return true;
- default:
- return false;
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/format/src/main/thrift/schema.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/schema.thrift b/format/src/main/thrift/schema.thrift
index 8e12361..9244dac 100644
--- a/format/src/main/thrift/schema.thrift
+++ b/format/src/main/thrift/schema.thrift
@@ -27,11 +27,12 @@ namespace java org.carbondata.format
*/
enum DataType {
STRING = 0,
- INT = 1,
- LONG = 2,
- DOUBLE = 3,
- DECIMAL = 4,
- TIMESTAMP = 5,
+ SHORT = 1,
+ INT = 2,
+ LONG = 3,
+ DOUBLE = 4,
+ DECIMAL = 5,
+ TIMESTAMP = 6,
ARRAY = 20,
STRUCT = 21,
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
index e14f86a..bbc9f25 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
@@ -242,6 +242,7 @@ case class CarbonRelation(
metaData.carbonTable.getMeasureByName(factTable, x.getColName).getDataType.toString
.toLowerCase match {
case "int" => "double"
+ case "short" => "double"
case "decimal" => "decimal(" + x.getPrecision + "," + x.getScale + ")"
case others => others
}),
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
index c2e1564..05149eb 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
@@ -92,6 +92,7 @@ case class CarbonDictionaryDecoder(
relation: CarbonRelation): types.DataType = {
carbonDimension.getDataType match {
case DataType.STRING => StringType
+ case DataType.SHORT => ShortType
case DataType.INT => IntegerType
case DataType.LONG => LongType
case DataType.DOUBLE => DoubleType
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 476f25d..cb94f72 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -379,6 +379,8 @@ class TableNewProcessor(cm: tableModel, sqlContext: SQLContext) {
case "String" => DataType.STRING
case "int" => DataType.INT
case "Integer" => DataType.INT
+ case "tinyint" => DataType.SHORT
+ case "short" => DataType.SHORT
case "Long" => DataType.LONG
case "BigInt" => DataType.LONG
case "Numeric" => DataType.DOUBLE
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
index dc44f28..d87e132 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
@@ -707,6 +707,7 @@ object CarbonMetastoreTypes extends RegexParsers {
"float" ^^^ FloatType |
"int" ^^^ IntegerType |
"tinyint" ^^^ ShortType |
+ "short" ^^^ ShortType |
"double" ^^^ DoubleType |
"long" ^^^ LongType |
"binary" ^^^ BinaryType |
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/integration/spark/src/main/scala/org/carbondata/spark/package.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/package.scala b/integration/spark/src/main/scala/org/carbondata/spark/package.scala
index 2745229..60c8880 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/package.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/package.scala
@@ -81,7 +81,7 @@ package object spark {
case StringType => CarbonType.STRING.name
case IntegerType => CarbonType.INT.name
case ByteType => CarbonType.INT.name
- case ShortType => CarbonType.INT.name
+ case ShortType => CarbonType.SHORT.name
case LongType => CarbonType.LONG.name
case FloatType => CarbonType.DOUBLE.name
case DoubleType => CarbonType.DOUBLE.name
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
index 593aadb..0930887 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
@@ -35,6 +35,7 @@ object CarbonScalaUtil {
dataType: org.apache.spark.sql.types.DataType): CarbonDataType = {
dataType match {
case StringType => CarbonDataType.StringType
+ case ShortType => CarbonDataType.ShortType
case IntegerType => CarbonDataType.IntegerType
case LongType => CarbonDataType.LongType
case DoubleType => CarbonDataType.DoubleType
@@ -54,7 +55,7 @@ object CarbonScalaUtil {
case CarbonCommonConstants.STRING_TYPE => CarbonCommonConstants.STRING
case CarbonCommonConstants.INTEGER_TYPE => CarbonCommonConstants.INTEGER
case CarbonCommonConstants.BYTE_TYPE => CarbonCommonConstants.INTEGER
- case CarbonCommonConstants.SHORT_TYPE => CarbonCommonConstants.INTEGER
+ case CarbonCommonConstants.SHORT_TYPE => CarbonCommonConstants.SHORT
case CarbonCommonConstants.LONG_TYPE => CarbonCommonConstants.NUMERIC
case CarbonCommonConstants.DOUBLE_TYPE => CarbonCommonConstants.NUMERIC
case CarbonCommonConstants.FLOAT_TYPE => CarbonCommonConstants.NUMERIC
@@ -66,36 +67,10 @@ object CarbonScalaUtil {
}
}
- def convertSparkColumnToCarbonLevel(field: (String, String)): Seq[Level] = {
- field._2 match {
- case CarbonCommonConstants.STRING_TYPE => Seq(
- Level(field._1, field._1, Int.MaxValue, CarbonCommonConstants.STRING))
- case CarbonCommonConstants.INTEGER_TYPE => Seq(
- Level(field._1, field._1, Int.MaxValue, CarbonCommonConstants.INTEGER))
- case CarbonCommonConstants.BYTE_TYPE => Seq(
- Level(field._1, field._1, Int.MaxValue, CarbonCommonConstants.INTEGER))
- case CarbonCommonConstants.SHORT_TYPE => Seq(
- Level(field._1, field._1, Int.MaxValue, CarbonCommonConstants.INTEGER))
- case CarbonCommonConstants.LONG_TYPE => Seq(
- Level(field._1, field._1, Int.MaxValue, CarbonCommonConstants.NUMERIC))
- case CarbonCommonConstants.DOUBLE_TYPE => Seq(
- Level(field._1, field._1, Int.MaxValue, CarbonCommonConstants.NUMERIC))
- case CarbonCommonConstants.FLOAT_TYPE => Seq(
- Level(field._1, field._1, Int.MaxValue, CarbonCommonConstants.NUMERIC))
- case CarbonCommonConstants.DECIMAL_TYPE => Seq(
- Level(field._1, field._1, Int.MaxValue, CarbonCommonConstants.NUMERIC))
- case CarbonCommonConstants.DATE_TYPE => Seq(
- Level(field._1, field._1, Int.MaxValue, CarbonCommonConstants.STRING))
- case CarbonCommonConstants.BOOLEAN_TYPE => Seq(
- Level(field._1, field._1, Int.MaxValue, CarbonCommonConstants.STRING))
- case CarbonCommonConstants.TIMESTAMP_TYPE => Seq(
- Level(field._1, field._1, Int.MaxValue, CarbonCommonConstants.TIMESTAMP))
- }
- }
-
def convertCarbonToSparkDataType(dataType: DataType): types.DataType = {
dataType match {
case DataType.STRING => StringType
+ case DataType.SHORT => ShortType
case DataType.INT => IntegerType
case DataType.LONG => LongType
case DataType.DOUBLE => DoubleType
@@ -105,21 +80,6 @@ object CarbonScalaUtil {
}
}
- def convertValueToSparkDataType(value: Any,
- dataType: org.apache.spark.sql.types.DataType): Any = {
- dataType match {
- case StringType => value.toString
- case IntegerType => value.toString.toInt
- case LongType => value.toString.toLong
- case DoubleType => value.toString.toDouble
- case FloatType => value.toString.toFloat
- case _ => value.toString.toDouble
- }
- }
-
-
- case class TransformHolder(rdd: Any, mataData: CarbonMetaData)
-
object CarbonSparkUtil {
def createSparkMeta(carbonTable: CarbonTable): CarbonMetaData = {
[2/2] incubator-carbondata git commit: [CARBONDATA-18] This closes #2
Posted by ch...@apache.org.
[CARBONDATA-18] This closes #2
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/634bdeef
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/634bdeef
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/634bdeef
Branch: refs/heads/master
Commit: 634bdeefcd186707f6c0fc6f98df4ef87bfdf40a
Parents: 02fcb19 a743757
Author: chenliang613 <ch...@apache.org>
Authored: Wed Jun 29 02:53:14 2016 +0530
Committer: chenliang613 <ch...@apache.org>
Committed: Wed Jun 29 02:53:14 2016 +0530
----------------------------------------------------------------------
.../cache/dictionary/ColumnDictionaryInfo.java | 2 +
.../ThriftWrapperSchemaConverterImpl.java | 4 ++
.../core/carbon/metadata/datatype/DataType.java | 1 +
.../core/constants/CarbonCommonConstants.java | 1 +
.../core/util/DataFileFooterConverter.java | 2 +
.../org/carbondata/core/util/DataTypeUtil.java | 4 ++
.../sortindex/CarbonDictionarySortModel.java | 1 +
.../carbondata/scan/expression/DataType.java | 3 +-
.../scan/expression/ExpressionResult.java | 59 +++++++++++++++++++-
.../expression/arithmetic/AddExpression.java | 3 +
.../expression/arithmetic/DivideExpression.java | 3 +
.../arithmetic/MultiplyExpression.java | 3 +
.../arithmetic/SubstractExpression.java | 3 +
.../conditional/EqualToExpression.java | 3 +
.../GreaterThanEqualToExpression.java | 3 +
.../conditional/GreaterThanExpression.java | 3 +
.../expression/conditional/InExpression.java | 3 +
.../conditional/LessThanEqualToExpression.java | 3 +
.../conditional/LessThanExpression.java | 3 +
.../conditional/NotEqualsExpression.java | 3 +
.../expression/conditional/NotInExpression.java | 3 +
.../org/carbondata/scan/filter/FilterUtil.java | 4 +-
.../org/carbondata/scan/util/DataTypeUtil.java | 54 +-----------------
format/src/main/thrift/schema.thrift | 11 ++--
.../spark/sql/CarbonDatasourceRelation.scala | 1 +
.../spark/sql/CarbonDictionaryDecoder.scala | 1 +
.../execution/command/carbonTableSchema.scala | 2 +
.../spark/sql/hive/CarbonMetastoreCatalog.scala | 1 +
.../scala/org/carbondata/spark/package.scala | 2 +-
.../carbondata/spark/util/CarbonScalaUtil.scala | 46 +--------------
30 files changed, 130 insertions(+), 105 deletions(-)
----------------------------------------------------------------------