You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/07/04 14:05:01 UTC
[4/8] carbondata git commit: [CARBONDATA-1253] Sort_columns should
not support float, double, decimal
[CARBONDATA-1253] Sort_columns should not support float,double,decimal
This closes #1122
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5f9741eb
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5f9741eb
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5f9741eb
Branch: refs/heads/encoding_override
Commit: 5f9741ebcb5b2b606f4d710785ce97c4d6b49229
Parents: 0d46976
Author: QiangCai <qi...@qq.com>
Authored: Fri Jun 30 19:51:19 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Mon Jul 3 20:53:20 2017 +0800
----------------------------------------------------------------------
...feVariableLengthDimensionDataChunkStore.java | 12 ---
...afeVariableLengthDimesionDataChunkStore.java | 12 ---
.../apache/carbondata/core/util/ByteUtil.java | 81 --------------------
.../carbondata/core/util/DataTypeUtil.java | 12 ---
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 12 ++-
5 files changed, 10 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f9741eb/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
index a65d745..2079811 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
@@ -25,10 +25,6 @@ import org.apache.carbondata.core.util.ByteUtil;
import org.apache.spark.sql.types.BooleanType;
import org.apache.spark.sql.types.DataType;
-import org.apache.spark.sql.types.Decimal;
-import org.apache.spark.sql.types.DecimalType;
-import org.apache.spark.sql.types.DoubleType;
-import org.apache.spark.sql.types.FloatType;
import org.apache.spark.sql.types.IntegerType;
import org.apache.spark.sql.types.LongType;
import org.apache.spark.sql.types.ShortType;
@@ -157,16 +153,8 @@ public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimens
vector.putShort(vectorRow, ByteUtil.toShort(data, currentDataOffset, length));
} else if (dt instanceof IntegerType) {
vector.putInt(vectorRow, ByteUtil.toInt(data, currentDataOffset, length));
- } else if (dt instanceof FloatType) {
- vector.putFloat(vectorRow, ByteUtil.toFloat(data, currentDataOffset));
- } else if (dt instanceof DoubleType) {
- vector.putDouble(vectorRow, ByteUtil.toDouble(data, currentDataOffset));
} else if (dt instanceof LongType) {
vector.putLong(vectorRow, ByteUtil.toLong(data, currentDataOffset, length));
- } else if (dt instanceof DecimalType) {
- vector.putDecimal(vectorRow,
- Decimal.apply(ByteUtil.toBigDecimal(data, currentDataOffset, length)),
- DecimalType.MAX_PRECISION());
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f9741eb/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
index 03ba34e..c6c98f9 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
@@ -26,10 +26,6 @@ import org.apache.carbondata.core.util.ByteUtil;
import org.apache.spark.sql.types.BooleanType;
import org.apache.spark.sql.types.DataType;
-import org.apache.spark.sql.types.Decimal;
-import org.apache.spark.sql.types.DecimalType;
-import org.apache.spark.sql.types.DoubleType;
-import org.apache.spark.sql.types.FloatType;
import org.apache.spark.sql.types.IntegerType;
import org.apache.spark.sql.types.LongType;
import org.apache.spark.sql.types.ShortType;
@@ -184,16 +180,8 @@ public class UnsafeVariableLengthDimesionDataChunkStore
vector.putShort(vectorRow, ByteUtil.toShort(value, 0, value.length));
} else if (dt instanceof IntegerType) {
vector.putInt(vectorRow, ByteUtil.toInt(value, 0, value.length));
- } else if (dt instanceof FloatType) {
- vector.putFloat(vectorRow, ByteUtil.toFloat(value, 0));
- } else if (dt instanceof DoubleType) {
- vector.putDouble(vectorRow, ByteUtil.toDouble(value, 0));
} else if (dt instanceof LongType) {
vector.putLong(vectorRow, ByteUtil.toLong(value, 0, value.length));
- } else if (dt instanceof DecimalType) {
- vector.putDecimal(vectorRow,
- Decimal.apply(ByteUtil.toBigDecimal(value, 0, value.length)),
- DecimalType.MAX_PRECISION());
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f9741eb/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
index 2761d71..b74f206 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
@@ -18,8 +18,6 @@
package org.apache.carbondata.core.util;
import java.io.UnsupportedEncodingException;
-import java.math.BigDecimal;
-import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
@@ -39,8 +37,6 @@ public final class ByteUtil {
public static final String UTF8_CSN = StandardCharsets.UTF_8.name();
- public static final byte[] ZERO_IN_BYTES = toBytes(0);
-
private ByteUtil() {
}
@@ -531,28 +527,6 @@ public final class ByteUtil {
}
/**
- * float => byte[]
- *
- * @param f
- * @return
- */
- public static byte[] toBytes(final float f) {
- // Encode it as int
- return toBytes(Float.floatToRawIntBits(f));
- }
-
- /**
- * byte[] => float
- *
- * @param bytes
- * @param offset
- * @return
- */
- public static float toFloat(byte[] bytes, int offset) {
- return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT));
- }
-
- /**
* long => byte[]
*
* @param val
@@ -593,61 +567,6 @@ public final class ByteUtil {
return l ^ Long.MIN_VALUE;
}
- /**
- * doube => byte[]
- *
- * @param d
- * @return
- */
- public static byte[] toBytes(final double d) {
- // Encode it as a long
- return toBytes(Double.doubleToRawLongBits(d));
- }
-
- /**
- * byte[] => double
- *
- * @param bytes
- * @param offset
- * @return
- */
- public static double toDouble(final byte[] bytes, final int offset) {
- return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG));
- }
-
- /**
- * BigDecimal => byte[]
- *
- * @param val
- * @return
- */
- public static byte[] toBytes(BigDecimal val) {
- byte[] valueBytes = val.unscaledValue().toByteArray();
- byte[] result = new byte[valueBytes.length + SIZEOF_INT];
- int offset = putInt(result, 0, val.scale());
- putBytes(result, offset, valueBytes, 0, valueBytes.length);
- return result;
- }
-
- /**
- * byte[] => BigDecimal
- *
- * @param bytes
- * @param offset
- * @param length
- * @return
- */
- public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) {
- if (bytes == null || length < SIZEOF_INT + 1 || (offset + length > bytes.length)) {
- return null;
- }
-
- int scale = toInt(bytes, offset, bytes.length);
- byte[] tcBytes = new byte[length - SIZEOF_INT];
- System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT);
- return new BigDecimal(new BigInteger(tcBytes), scale);
- }
-
private static IllegalArgumentException explainWrongLengthOrOffset(final byte[] bytes,
final int offset, final int length, final int expectedLength) {
String reason;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f9741eb/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 1b1884e..37ae5bb 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -334,14 +334,8 @@ public final class DataTypeUtil {
return ByteUtil.toBytes(Short.parseShort(dimensionValue));
case INT:
return ByteUtil.toBytes(Integer.parseInt(dimensionValue));
- case FLOAT:
- return ByteUtil.toBytes(Float.parseFloat(dimensionValue));
case LONG:
return ByteUtil.toBytes(Long.parseLong(dimensionValue));
- case DOUBLE:
- return ByteUtil.toBytes(Double.parseDouble(dimensionValue));
- case DECIMAL:
- return ByteUtil.toBytes(new BigDecimal(dimensionValue));
default:
return ByteUtil.toBytes(dimensionValue);
}
@@ -372,14 +366,8 @@ public final class DataTypeUtil {
return ByteUtil.toShort(dataInBytes, 0, dataInBytes.length);
case INT:
return ByteUtil.toInt(dataInBytes, 0, dataInBytes.length);
- case FLOAT:
- return ByteUtil.toFloat(dataInBytes, 0);
case LONG:
return ByteUtil.toLong(dataInBytes, 0, dataInBytes.length);
- case DOUBLE:
- return ByteUtil.toDouble(dataInBytes, 0);
- case DECIMAL:
- return ByteUtil.toBigDecimal(dataInBytes, 0, dataInBytes.length);
default:
return ByteUtil.toString(dataInBytes, 0, dataInBytes.length);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5f9741eb/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index c565c31..8207a9d 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -551,8 +551,8 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
} else {
val dataType = fields.find(x =>
x.column.equalsIgnoreCase(column)).get.dataType.get
- if (isComplexDimDictionaryExclude(dataType)) {
- val errormsg = "sort_columns is unsupported for complex datatype column: " + column
+ if (isDataTypeSupportedForSortColumn(dataType)) {
+ val errormsg = s"sort_columns is unsupported for ${dataType} datatype column: " + column
throw new MalformedCarbonCommandException(errormsg)
}
}
@@ -691,6 +691,14 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
}
/**
+ * detects whether datatype is part of sort_column
+ */
+ private def isDataTypeSupportedForSortColumn(columnDataType: String): Boolean = {
+ val dataTypes = Array("array", "struct", "double", "float", "decimal")
+ dataTypes.exists(x => x.equalsIgnoreCase(columnDataType))
+ }
+
+ /**
* detects whether datatype is part of dictionary_exclude
*/
def isDataTypeSupportedForDictionary_Exclude(columnDataType: String): Boolean = {