You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/07/31 11:49:18 UTC
[4/6] carbondata git commit: Rebased with new master and fixed binary
comparisions and comments.
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4e835095/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java
index 53cbc66..3469a54 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java
@@ -49,7 +49,9 @@ public class RowLevelRangeTypeExecuterFacory {
filterExpressionResolverTree.getFilterExpression(),
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
- .getFilterRangeValues(segmentProperties), segmentProperties);
+ .getFilterRangeValues(segmentProperties),
+ ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
+ .getMeasureFilterRangeValues(), segmentProperties);
case ROWLEVEL_LESSTHAN_EQUALTO:
return new RowLevelRangeLessThanEqualFilterExecuterImpl(
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
@@ -59,7 +61,9 @@ public class RowLevelRangeTypeExecuterFacory {
filterExpressionResolverTree.getFilterExpression(),
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
- .getFilterRangeValues(segmentProperties), segmentProperties);
+ .getFilterRangeValues(segmentProperties),
+ ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
+ .getMeasureFilterRangeValues(), segmentProperties);
case ROWLEVEL_GREATERTHAN_EQUALTO:
return new RowLevelRangeGrtrThanEquaToFilterExecuterImpl(
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
@@ -69,7 +73,9 @@ public class RowLevelRangeTypeExecuterFacory {
filterExpressionResolverTree.getFilterExpression(),
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
- .getFilterRangeValues(segmentProperties), segmentProperties);
+ .getFilterRangeValues(segmentProperties),
+ ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
+ .getMeasureFilterRangeValues(), segmentProperties);
case ROWLEVEL_GREATERTHAN:
return new RowLevelRangeGrtThanFiterExecuterImpl(
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
@@ -79,7 +85,9 @@ public class RowLevelRangeTypeExecuterFacory {
filterExpressionResolverTree.getFilterExpression(),
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
- .getFilterRangeValues(segmentProperties), segmentProperties);
+ .getFilterRangeValues(segmentProperties),
+ ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
+ .getMeasureFilterRangeValues(), segmentProperties);
default:
// Scenario wont come logic must break
return null;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4e835095/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/PartitionFilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/PartitionFilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/PartitionFilterUtil.java
index a056d94..d040c1b 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/PartitionFilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/partition/PartitionFilterUtil.java
@@ -30,8 +30,6 @@ import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.comparator.Comparator;
import org.apache.carbondata.core.util.comparator.SerializableComparator;
-import org.apache.spark.sql.types.Decimal;
-
public class PartitionFilterUtil {
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4e835095/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
index 95d3223..8d3c366 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
@@ -317,7 +317,7 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
} else if (null != dimColResolvedFilterInfo.getFilterValues() && dimColResolvedFilterInfo
.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
return FilterUtil.getKeyArray(this.dimColResolvedFilterInfo.getFilterValues(),
- this.dimColResolvedFilterInfo.getDimension(), null, segmentProperties);
+ this.dimColResolvedFilterInfo.getDimension(), segmentProperties);
}
return null;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4e835095/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
index 3bdca36..f64342f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
@@ -87,18 +87,23 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
.getDimensionFromCurrentBlock(this.dimColEvaluatorInfoList.get(0).getDimension());
if (null != dimensionFromCurrentBlock) {
return FilterUtil.getKeyArray(this.dimColEvaluatorInfoList.get(0).getFilterValues(),
- dimensionFromCurrentBlock, null, segmentProperties);
+ dimensionFromCurrentBlock, segmentProperties);
}
- } else if (msrColEvalutorInfoList.size() > 0 && null != msrColEvalutorInfoList.get(0)
+ }
+ return null;
+ }
+
+ public Object[] getMeasureFilterRangeValues() {
+ if (msrColEvalutorInfoList.size() > 0 && null != msrColEvalutorInfoList.get(0)
.getFilterValues()) {
- List<byte[]> measureFilterValuesList =
+ List<Object> measureFilterValuesList =
msrColEvalutorInfoList.get(0).getFilterValues().getMeasuresFilterValuesList();
- return measureFilterValuesList.toArray((new byte[measureFilterValuesList.size()][]));
+ return measureFilterValuesList.toArray((new Object[measureFilterValuesList.size()]));
}
return null;
-
}
+
/**
* method will get the start key based on the filter surrogates
*
@@ -179,23 +184,22 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
return filterValuesList;
}
- private List<byte[]> getMeasureRangeValues(CarbonMeasure carbonMeasure) {
+ private List<Object> getMeasureRangeValues(CarbonMeasure carbonMeasure) {
List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20);
if (this.getFilterExpression() instanceof BinaryConditionalExpression) {
listOfExpressionResults =
((BinaryConditionalExpression) this.getFilterExpression()).getLiterals();
}
- List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
+ List<Object> filterValuesList = new ArrayList<>(20);
boolean invalidRowsPresent = false;
for (ExpressionResult result : listOfExpressionResults) {
try {
if (result.getString() == null) {
- filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL.getBytes());
+ filterValuesList.add(null);
continue;
}
- filterValuesList.add(DataTypeUtil
- .getMeasureByteArrayBasedOnDataTypes(result.getString(),
- result.getDataType(), carbonMeasure));
+ filterValuesList.add(DataTypeUtil.getMeasureValueBasedOnDataType(result.getString(),
+ result.getDataType(), carbonMeasure));
} catch (FilterIllegalMemberException e) {
// Any invalid member while evaluation shall be ignored, system will log the
// error only once since all rows the evaluation happens so inorder to avoid
@@ -203,13 +207,8 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
FilterUtil.logError(e, invalidRowsPresent);
}
}
- Comparator<byte[]> filterMeasureComaparator = new Comparator<byte[]>() {
- @Override public int compare(byte[] filterMember1, byte[] filterMember2) {
- return ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterMember1, filterMember2);
- }
-
- };
- Collections.sort(filterValuesList, filterMeasureComaparator);
+ Collections.sort(filterValuesList, org.apache.carbondata.core.util.comparator.Comparator
+ .getComparatorByDataTypeForMeasure(carbonMeasure.getDataType()));
return filterValuesList;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4e835095/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index 60546ed..db5010f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -22,7 +22,6 @@ import java.io.ObjectOutputStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
@@ -440,29 +439,6 @@ public class CarbonMetadataUtil {
return false;
}
- private static ByteBuffer writeInfoIfDecimal(int blockIndex,
- SegmentProperties segmentProperties) {
- Map<Integer, Integer> blockMapping = segmentProperties.getMeasuresOrdinalToBlockMapping();
- List<CarbonMeasure> measures = segmentProperties.getMeasures();
- CarbonMeasure selectedMeasure = null;
- for (CarbonMeasure measure : measures) {
- Integer blockId = blockMapping.get(measure.getOrdinal());
- selectedMeasure = measure;
- if (blockId == blockIndex) {
- break;
- }
- }
- assert (selectedMeasure != null);
- if (selectedMeasure.getDataType() == DataType.DECIMAL) {
- ByteBuffer buffer = ByteBuffer.allocate(8);
- buffer.putInt(selectedMeasure.getScale());
- buffer.putInt(selectedMeasure.getPrecision());
- buffer.flip();
- return buffer;
- }
- return null;
- }
-
private static byte[] serializeEncoderMeta(ValueEncoderMeta encoderMeta) throws IOException {
// TODO : should remove the unnecessary fields.
ByteArrayOutputStream aos = new ByteArrayOutputStream();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4e835095/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 5f146f6..6e54f1f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -121,10 +121,15 @@ public final class DataTypeUtil {
}
public static Object getMeasureObjectFromDataType(byte[] data, DataType dataType) {
+ if (data == null || data.length == 0) {
+ return null;
+ }
ByteBuffer bb = ByteBuffer.wrap(data);
switch (dataType) {
case SHORT:
+ return (short)bb.getLong();
case INT:
+ return (int)bb.getLong();
case LONG:
return bb.getLong();
case DECIMAL:
@@ -134,113 +139,13 @@ public final class DataTypeUtil {
}
}
- /**
- * This method will convert a given ByteArray to its specific type
- *
- * @param msrValue
- * @param dataType
- * @param carbonMeasure
- * @return
- */
- // public static byte[] getMeasureByteArrayBasedOnDataType(String msrValue, DataType dataType,
- // CarbonMeasure carbonMeasure) {
- // switch (dataType) {
- // case DECIMAL:
- // BigDecimal bigDecimal =
- // new BigDecimal(msrValue).setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP);
- // return ByteUtil.toBytes(normalizeDecimalValue(bigDecimal, carbonMeasure.getPrecision()));
- // case SHORT:
- // return ByteUtil.toBytes((Short.parseShort(msrValue)));
- // case INT:
- // return ByteUtil.toBytes(Integer.parseInt(msrValue));
- // case LONG:
- // return ByteUtil.toBytes(Long.valueOf(msrValue));
- // default:
- // Double parsedValue = Double.valueOf(msrValue);
- // if (Double.isInfinite(parsedValue) || Double.isNaN(parsedValue)) {
- // return null;
- // }
- // return ByteUtil.toBytes(parsedValue);
- // }
- // }
- public static byte[] getMeasureByteArrayBasedOnDataTypes(String msrValue, DataType dataType,
- CarbonMeasure carbonMeasure) {
- ByteBuffer b;
- switch (dataType) {
- case BYTE:
- case SHORT:
- case INT:
- case LONG:
- b = ByteBuffer.allocate(8);
- b.putLong(Long.valueOf(msrValue));
- b.flip();
- return b.array();
- case DOUBLE:
- b = ByteBuffer.allocate(8);
- b.putDouble(Double.valueOf(msrValue));
- b.flip();
- return b.array();
- case DECIMAL:
- BigDecimal bigDecimal =
- new BigDecimal(msrValue).setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP);
- return DataTypeUtil
- .bigDecimalToByte(normalizeDecimalValue(bigDecimal, carbonMeasure.getPrecision()));
- default:
- throw new IllegalArgumentException("Invalid data type: " + dataType);
- }
- }
-
- /**
- * This method will convert a given ByteArray to its specific type
- *
- * @param msrValue
- * @param dataType
- * @param carbonMeasure
- * @return
- */
- public static byte[] getMeasureByteArrayBasedOnDataType(ColumnPage measurePage, int index,
- DataType dataType, CarbonMeasure carbonMeasure) {
- switch (dataType) {
- case DECIMAL:
- BigDecimal bigDecimal = new BigDecimal(measurePage.getDouble(index))
- .setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP);
- return ByteUtil.toBytes(normalizeDecimalValue(bigDecimal, carbonMeasure.getPrecision()));
- case SHORT:
- return ByteUtil.toBytes(measurePage.getShort(index));
- case INT:
- return ByteUtil.toBytes(measurePage.getInt(index));
- case LONG:
- return ByteUtil.toBytes(measurePage.getLong(index));
- default:
- Double parsedValue = Double.valueOf(measurePage.getDouble(index));
- if (Double.isInfinite(parsedValue) || Double.isNaN(parsedValue)) {
- return null;
- }
- return ByteUtil.toBytes(parsedValue);
- }
- }
-
public static Object getMeasureObjectBasedOnDataType(ColumnPage measurePage, int index,
DataType dataType, CarbonMeasure carbonMeasure) {
- // switch (dataType) {
- // case DECIMAL:
- // BigDecimal bigDecimal = new BigDecimal(measurePage.getDouble(index))
- // .setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP);
- // return normalizeDecimalValue(bigDecimal, carbonMeasure.getPrecision());
- // case SHORT:
- // case INT:
- // case LONG:
- // return measurePage.getLong(index);
- // default:
- // Double parsedValue = Double.valueOf(measurePage.getDouble(index));
- // if (Double.isInfinite(parsedValue) || Double.isNaN(parsedValue)) {
- // return null;
- // }
- // return parsedValue;
- // }
switch (dataType) {
case SHORT:
+ return (short)measurePage.getLong(index);
case INT:
+ return (int)measurePage.getLong(index);
case LONG:
return measurePage.getLong(index);
case DECIMAL:
@@ -249,7 +154,6 @@ public final class DataTypeUtil {
bigDecimalMsrValue =
bigDecimalMsrValue.setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP);
}
- //return org.apache.spark.sql.types.Decimal.apply(bigDecimalMsrValue);
return normalizeDecimalValue(bigDecimalMsrValue, carbonMeasure.getPrecision());
default:
return measurePage.getDouble(index);
@@ -379,6 +283,19 @@ public final class DataTypeUtil {
* @return actual data after conversion
*/
public static Object getDataBasedOnDataType(String data, DataType actualDataType) {
+ return getDataBasedOnDataType(data, actualDataType, getDataTypeConverter());
+ }
+
+ /**
+ * Below method will be used to convert the data passed to its actual data
+ * type
+ *
+ * @param data data
+ * @param actualDataType actual data type
+ * @return actual data after conversion
+ */
+ public static Object getDataBasedOnDataType(String data, DataType actualDataType,
+ DataTypeConverter converter) {
if (null == data || CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(data)) {
return null;
}
@@ -435,15 +352,14 @@ public final class DataTypeUtil {
if (data.isEmpty()) {
return null;
}
- return getDataTypeConverter().convertToDecimal(data);
+ return converter.convertToDecimal(data);
default:
- return getDataTypeConverter().convertFromStringToUTF8String(data);
+ return converter.convertFromStringToUTF8String(data);
}
} catch (NumberFormatException ex) {
LOGGER.error("Problem while converting data type" + data);
return null;
}
-
}
public static byte[] getBytesBasedOnDataTypeForNoDictionaryColumn(String dimensionValue,
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4e835095/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java b/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java
index adce04f..1f43c7c 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java
@@ -42,6 +42,29 @@ public final class Comparator {
return new ByteArraySerializableComparator();
}
}
+
+ /**
+ * create Comparator for Measure Datatype
+ *
+ * @param dataType
+ * @return
+ */
+ public static SerializableComparator getComparatorByDataTypeForMeasure(DataType dataType) {
+ switch (dataType) {
+ case INT:
+ return new IntSerializableComparator();
+ case SHORT:
+ return new ShortSerializableComparator();
+ case LONG:
+ return new LongSerializableComparator();
+ case DOUBLE:
+ return new DoubleSerializableComparator();
+ case DECIMAL:
+ return new BigDecimalSerializableComparator();
+ default:
+ throw new IllegalArgumentException("Unsupported data type");
+ }
+ }
}
class ByteArraySerializableComparator implements SerializableComparator {
@@ -52,6 +75,13 @@ class ByteArraySerializableComparator implements SerializableComparator {
class IntSerializableComparator implements SerializableComparator {
@Override public int compare(Object key1, Object key2) {
+ if (key1 == null && key2 == null) {
+ return 0;
+ } else if (key1 == null) {
+ return -1;
+ } else if (key2 == null) {
+ return 1;
+ }
if ((int) key1 < (int) key2) {
return -1;
} else if ((int) key1 > (int) key2) {
@@ -64,6 +94,13 @@ class IntSerializableComparator implements SerializableComparator {
class ShortSerializableComparator implements SerializableComparator {
@Override public int compare(Object key1, Object key2) {
+ if (key1 == null && key2 == null) {
+ return 0;
+ } else if (key1 == null) {
+ return -1;
+ } else if (key2 == null) {
+ return 1;
+ }
if ((short) key1 < (short) key2) {
return -1;
} else if ((short) key1 > (short) key2) {
@@ -76,6 +113,13 @@ class ShortSerializableComparator implements SerializableComparator {
class DoubleSerializableComparator implements SerializableComparator {
@Override public int compare(Object key1, Object key2) {
+ if (key1 == null && key2 == null) {
+ return 0;
+ } else if (key1 == null) {
+ return -1;
+ } else if (key2 == null) {
+ return 1;
+ }
if ((double) key1 < (double) key2) {
return -1;
} else if ((double) key1 > (double) key2) {
@@ -88,6 +132,13 @@ class DoubleSerializableComparator implements SerializableComparator {
class LongSerializableComparator implements SerializableComparator {
@Override public int compare(Object key1, Object key2) {
+ if (key1 == null && key2 == null) {
+ return 0;
+ } else if (key1 == null) {
+ return -1;
+ } else if (key2 == null) {
+ return 1;
+ }
if ((long) key1 < (long) key2) {
return -1;
} else if ((long) key1 > (long) key2) {
@@ -100,6 +151,13 @@ class LongSerializableComparator implements SerializableComparator {
class BigDecimalSerializableComparator implements SerializableComparator {
@Override public int compare(Object key1, Object key2) {
+ if (key1 == null && key2 == null) {
+ return 0;
+ } else if (key1 == null) {
+ return -1;
+ } else if (key2 == null) {
+ return 1;
+ }
return ((BigDecimal) key1).compareTo((BigDecimal) key2);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4e835095/core/src/main/java/org/apache/carbondata/core/util/comparator/SerializableComparator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/comparator/SerializableComparator.java b/core/src/main/java/org/apache/carbondata/core/util/comparator/SerializableComparator.java
index df0d3e2..a8887eb 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/comparator/SerializableComparator.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/comparator/SerializableComparator.java
@@ -19,6 +19,6 @@ package org.apache.carbondata.core.util.comparator;
import java.io.Serializable;
-public interface SerializableComparator extends Serializable {
+public interface SerializableComparator extends java.util.Comparator, Serializable {
int compare(Object key1, Object key2);
}
\ No newline at end of file