You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/10/31 07:00:11 UTC
[13/22] carbondata git commit: [CARBONDATA-1444] Support Boolean data
type
[CARBONDATA-1444] Support Boolean data type
Spark/Hive table support Boolean data type, the internal table also should support Boolean data type.
Boolean data type Range: TRUE or FALSE. Do not use quotation marks around the TRUE and FALSE literal values. You can write the literal values in uppercase, lowercase, or mixed case. The values queried from a table are always returned in lowercase, true or false.
This closes #1362
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/6abdd97a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/6abdd97a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/6abdd97a
Branch: refs/heads/fgdatamap
Commit: 6abdd97a42e945a0df61573aa477c9b3ddb7af02
Parents: 228ab2f
Author: xubo245 <60...@qq.com>
Authored: Mon Oct 16 19:59:50 2017 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Fri Oct 20 08:36:09 2017 +0800
----------------------------------------------------------------------
.../core/datastore/page/ColumnPage.java | 43 +-
.../core/datastore/page/LazyColumnPage.java | 4 +-
.../page/UnsafeFixLengthColumnPage.java | 3 +-
.../page/encoding/ColumnPageEncoderMeta.java | 6 +-
.../page/encoding/DefaultEncodingFactory.java | 5 +-
.../page/encoding/EncodingFactory.java | 5 +
.../page/encoding/bool/BooleanConvert.java | 63 ++
.../datastore/page/encoding/rle/RLECodec.java | 5 +-
.../statistics/PrimitivePageStatsCollector.java | 24 +-
.../ThriftWrapperSchemaConverterImpl.java | 6 +-
.../impl/AbstractScannedResultCollector.java | 4 +-
.../conditional/EqualToExpression.java | 4 +-
.../GreaterThanEqualToExpression.java | 4 +-
.../conditional/GreaterThanExpression.java | 4 +-
.../expression/conditional/InExpression.java | 4 +-
.../conditional/LessThanEqualToExpression.java | 4 +-
.../conditional/LessThanExpression.java | 4 +-
.../conditional/NotEqualsExpression.java | 4 +-
.../expression/conditional/NotInExpression.java | 4 +-
.../carbondata/core/scan/filter/FilterUtil.java | 5 +-
.../executer/ExcludeFilterExecuterImpl.java | 4 +-
.../executer/IncludeFilterExecuterImpl.java | 4 +-
.../executer/RowLevelFilterExecuterImpl.java | 8 +-
.../vector/MeasureDataVectorProcessor.java | 61 +-
.../util/AbstractDataFileFooterConverter.java | 2 +
.../core/util/CarbonMetadataUtil.java | 4 +-
.../apache/carbondata/core/util/CarbonUtil.java | 9 +-
.../carbondata/core/util/DataTypeUtil.java | 31 +-
.../core/util/comparator/Comparator.java | 28 +-
.../ThriftWrapperSchemaConverterImplTest.java | 12 +-
.../conditional/EqualToExpressionUnitTest.java | 2 +-
.../GreaterThanEqualToExpressionUnitTest.java | 2 +-
.../GreaterThanExpressionUnitTest.java | 2 +-
.../conditional/InExpressionUnitTest.java | 2 +-
.../LessThanEqualToExpressionUnitTest.java | 2 +-
.../conditional/LessThanExpressionUnitTest.java | 2 +-
.../NotEqualsExpressionUnitTest.java | 2 +-
.../conditional/NotInExpressionUnitTest.java | 2 +-
.../DataCompactionBoundaryConditionsTest.scala | 3 +
.../spark/util/DataTypeConverterUtil.scala | 4 +
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 3 +-
.../VectorizedCarbonRecordReader.java | 3 +-
.../src/test/resources/bool/supportBoolean.csv | 10 +
.../resources/bool/supportBooleanBadRecords.csv | 10 +
.../bool/supportBooleanDifferentFormat.csv | 20 +
.../bool/supportBooleanOnlyBoolean.csv | 11 +
.../bool/supportBooleanTwoBooleanColumns.csv | 10 +
.../bool/supportBooleanWithFileHeader.csv | 11 +
.../booleantype/BooleanDataTypesBaseTest.scala | 157 +++
.../BooleanDataTypesBigFileTest.scala | 729 ++++++++++++++
.../BooleanDataTypesFilterTest.scala | 416 ++++++++
.../BooleanDataTypesInsertTest.scala | 948 +++++++++++++++++++
.../booleantype/BooleanDataTypesLoadTest.scala | 744 +++++++++++++++
.../BooleanDataTypesParameterTest.scala | 288 ++++++
.../booleantype/BooleanDataTypesSortTest.scala | 145 +++
.../sort/unsafe/UnsafeCarbonRowPage.java | 12 +-
.../sort/sortdata/IntermediateFileMerger.java | 4 +-
.../processing/sort/sortdata/SortDataRows.java | 4 +-
.../sort/sortdata/SortTempFileChunkHolder.java | 4 +-
59 files changed, 3848 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
index 012413b..35bc560 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.core.datastore.TableSpec;
import org.apache.carbondata.core.datastore.compression.Compressor;
import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoderMeta;
+import org.apache.carbondata.core.datastore.page.encoding.bool.BooleanConvert;
import org.apache.carbondata.core.datastore.page.statistics.ColumnPageStatsCollector;
import org.apache.carbondata.core.datastore.page.statistics.SimpleStatsResult;
import org.apache.carbondata.core.memory.MemoryException;
@@ -184,7 +185,9 @@ public abstract class ColumnPage {
int pageSize) throws MemoryException {
ColumnPage instance;
if (unsafe) {
- if (dataType == DataTypes.BYTE ||
+ if (dataType == DataTypes.BOOLEAN) {
+ instance = new UnsafeFixLengthColumnPage(columnSpec, BYTE, pageSize);
+ } else if (dataType == DataTypes.BYTE ||
dataType == DataTypes.SHORT ||
dataType == DataTypes.SHORT_INT ||
dataType == DataTypes.INT ||
@@ -200,7 +203,7 @@ public abstract class ColumnPage {
throw new RuntimeException("Unsupported data dataType: " + dataType);
}
} else {
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
instance = newBytePage(columnSpec, new byte[pageSize]);
} else if (dataType == DataTypes.SHORT) {
instance = newShortPage(columnSpec, new short[pageSize]);
@@ -344,7 +347,10 @@ public abstract class ColumnPage {
nullBitSet.set(rowId);
return;
}
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
+ if (columnSpec.getSchemaDataType() == DataTypes.BOOLEAN) {
+ value = BooleanConvert.boolean2Byte((Boolean) value);
+ }
putByte(rowId, (byte) value);
statsCollector.update((byte) value);
} else if (dataType == DataTypes.SHORT) {
@@ -411,6 +417,13 @@ public abstract class ColumnPage {
public abstract void putShortInt(int rowId, int value);
/**
+ * Set boolean value at rowId
+ */
+ public void putBoolean(int rowId, boolean value) {
+ putByte(rowId, BooleanConvert.boolean2Byte(value));
+ }
+
+ /**
* Set byte array from offset to length at rowId
*/
public abstract void putBytes(int rowId, byte[] bytes, int offset, int length);
@@ -420,7 +433,9 @@ public abstract class ColumnPage {
* Set null at rowId
*/
private void putNull(int rowId) {
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN) {
+ putBoolean(rowId, false);
+ } else if (dataType == DataTypes.BYTE) {
putByte(rowId, (byte) 0);
} else if (dataType == DataTypes.SHORT) {
putShort(rowId, (short) 0);
@@ -453,6 +468,13 @@ public abstract class ColumnPage {
public abstract int getShortInt(int rowId);
/**
+ * Get boolean value at rowId
+ */
+ public boolean getBoolean(int rowId) {
+ return BooleanConvert.byte2Boolean(getByte(rowId));
+ }
+
+ /**
* Get int value at rowId
*/
public abstract int getInt(int rowId);
@@ -498,6 +520,13 @@ public abstract class ColumnPage {
public abstract byte[] getShortIntPage();
/**
+ * Get boolean value page
+ */
+ public byte[] getBooleanPage() {
+ return getBytePage();
+ }
+
+ /**
* Get int value page
*/
public abstract int[] getIntPage();
@@ -541,7 +570,9 @@ public abstract class ColumnPage {
* Compress page data using specified compressor
*/
public byte[] compress(Compressor compressor) throws MemoryException, IOException {
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return compressor.compressByte(getBooleanPage());
+ } else if (dataType == DataTypes.BYTE) {
return compressor.compressByte(getBytePage());
} else if (dataType == DataTypes.SHORT) {
return compressor.compressShort(getShortPage());
@@ -574,7 +605,7 @@ public abstract class ColumnPage {
Compressor compressor = CompressorFactory.getInstance().getCompressor(meta.getCompressorName());
TableSpec.ColumnSpec columnSpec = meta.getColumnSpec();
DataType storeDataType = meta.getStoreDataType();
- if (storeDataType == DataTypes.BYTE) {
+ if (storeDataType == DataTypes.BOOLEAN || storeDataType == DataTypes.BYTE) {
byte[] byteData = compressor.unCompressByte(compressedData, offset, length);
return newBytePage(columnSpec, byteData);
} else if (storeDataType == DataTypes.SHORT) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/datastore/page/LazyColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/LazyColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/LazyColumnPage.java
index cebb3c0..11bdaca 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/LazyColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/LazyColumnPage.java
@@ -53,7 +53,7 @@ public class LazyColumnPage extends ColumnPage {
@Override
public long getLong(int rowId) {
DataType dataType = columnPage.getDataType();
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
return converter.decodeLong(columnPage.getByte(rowId));
} else if (dataType == DataTypes.SHORT) {
return converter.decodeLong(columnPage.getShort(rowId));
@@ -71,7 +71,7 @@ public class LazyColumnPage extends ColumnPage {
@Override
public double getDouble(int rowId) {
DataType dataType = columnPage.getDataType();
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
return converter.decodeDouble(columnPage.getByte(rowId));
} else if (dataType == DataTypes.SHORT) {
return converter.decodeDouble(columnPage.getShort(rowId));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
index 055c27a..5695b70 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
@@ -56,7 +56,8 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
UnsafeFixLengthColumnPage(TableSpec.ColumnSpec columnSpec, DataType dataType, int pageSize)
throws MemoryException {
super(columnSpec, dataType, pageSize);
- if (dataType == DataTypes.BYTE ||
+ if (dataType == DataTypes.BOOLEAN ||
+ dataType == DataTypes.BYTE ||
dataType == DataTypes.SHORT ||
dataType == DataTypes.INT ||
dataType == DataTypes.LONG ||
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageEncoderMeta.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageEncoderMeta.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageEncoderMeta.java
index daccc32..a38da84 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageEncoderMeta.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnPageEncoderMeta.java
@@ -79,8 +79,6 @@ public class ColumnPageEncoderMeta extends ValueEncoderMeta implements Writable
}
}
-
-
public DataType getStoreDataType() {
return storeDataType;
}
@@ -108,7 +106,7 @@ public class ColumnPageEncoderMeta extends ValueEncoderMeta implements Writable
private void writeMinMax(DataOutput out) throws IOException {
DataType dataType = columnSpec.getSchemaDataType();
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
out.writeByte((byte) getMaxValue());
out.writeByte((byte) getMinValue());
out.writeLong(0L); // unique value is obsoleted, maintain for compatibility
@@ -151,7 +149,7 @@ public class ColumnPageEncoderMeta extends ValueEncoderMeta implements Writable
private void readMinMax(DataInput in) throws IOException {
DataType dataType = columnSpec.getSchemaDataType();
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
this.setMaxValue(in.readByte());
this.setMinValue(in.readByte());
in.readLong(); // for non exist value which is obsoleted, it is backward compatibility;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java
index d78d144..03af657 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DefaultEncodingFactory.java
@@ -32,6 +32,7 @@ import org.apache.carbondata.core.datastore.page.encoding.dimension.legacy.Compl
import org.apache.carbondata.core.datastore.page.encoding.dimension.legacy.DictDimensionIndexCodec;
import org.apache.carbondata.core.datastore.page.encoding.dimension.legacy.DirectDictDimensionIndexCodec;
import org.apache.carbondata.core.datastore.page.encoding.dimension.legacy.HighCardDictDimensionIndexCodec;
+import org.apache.carbondata.core.datastore.page.encoding.rle.RLECodec;
import org.apache.carbondata.core.datastore.page.statistics.SimpleStatsResult;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
@@ -113,7 +114,9 @@ public class DefaultEncodingFactory extends EncodingFactory {
private ColumnPageEncoder createEncoderForMeasure(ColumnPage columnPage) {
SimpleStatsResult stats = columnPage.getStatistics();
DataType dataType = stats.getDataType();
- if (dataType == DataTypes.BYTE ||
+ if (dataType == DataTypes.BOOLEAN) {
+ return new RLECodec().createEncoder(null);
+ } else if (dataType == DataTypes.BYTE ||
dataType == DataTypes.SHORT ||
dataType == DataTypes.INT ||
dataType == DataTypes.LONG) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
index 180228a..c298525 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/EncodingFactory.java
@@ -43,6 +43,7 @@ import org.apache.carbondata.format.Encoding;
import static org.apache.carbondata.format.Encoding.ADAPTIVE_DELTA_INTEGRAL;
import static org.apache.carbondata.format.Encoding.ADAPTIVE_FLOATING;
import static org.apache.carbondata.format.Encoding.ADAPTIVE_INTEGRAL;
+import static org.apache.carbondata.format.Encoding.BOOL_BYTE;
import static org.apache.carbondata.format.Encoding.DIRECT_COMPRESS;
import static org.apache.carbondata.format.Encoding.RLE_INTEGRAL;
@@ -94,6 +95,10 @@ public abstract class EncodingFactory {
RLEEncoderMeta metadata = new RLEEncoderMeta();
metadata.readFields(in);
return new RLECodec().createDecoder(metadata);
+ } else if (encoding == BOOL_BYTE) {
+ RLEEncoderMeta metadata = new RLEEncoderMeta();
+ metadata.readFields(in);
+ return new RLECodec().createDecoder(metadata);
} else {
// for backward compatibility
ValueEncoderMeta metadata = CarbonUtil.deserializeEncoderMetaV3(encoderMeta);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/bool/BooleanConvert.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/bool/BooleanConvert.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/bool/BooleanConvert.java
new file mode 100644
index 0000000..b373adf
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/bool/BooleanConvert.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page.encoding.bool;
+
+/**
+ * convert tools for boolean data type
+ */
+public class BooleanConvert {
+
+ public static final byte TRUE_VALUE = 1;
+ public static final byte FALSE_VALUE = 0;
+ /**
+ * convert boolean to byte
+ *
+ * @param data data of boolean data type
+ * @return byte type data by convert
+ */
+ public static byte boolean2Byte(boolean data) {
+ return data ? TRUE_VALUE : FALSE_VALUE;
+ }
+
+ /**
+ * convert byte to boolean
+ *
+ * @param data byte type data
+ * @return boolean type data
+ */
+ public static boolean byte2Boolean(int data) {
+ return data == TRUE_VALUE;
+ }
+
+ /**
+ * parse boolean, true and false to boolean
+ *
+ * @param input string type data
+ * @return Boolean type data
+ */
+ public static Boolean parseBoolean(String input) {
+ String value = input.toLowerCase();
+ if (("false").equals(value)) {
+ return Boolean.FALSE;
+ } else if (("true").equals(value)) {
+ return Boolean.TRUE;
+ } else {
+ throw new NumberFormatException("Not a valid Boolean type");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java
index 809bac0..7007084 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/rle/RLECodec.java
@@ -71,7 +71,8 @@ public class RLECodec implements ColumnPageCodec {
// This codec supports integral type only
private void validateDataType(DataType dataType) {
- if (! (dataType == DataTypes.BYTE || dataType == DataTypes.SHORT || dataType == DataTypes.INT ||
+ if (! (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE ||
+ dataType == DataTypes.SHORT || dataType == DataTypes.INT ||
dataType == DataTypes.LONG)) {
throw new UnsupportedOperationException(dataType + " is not supported for RLE");
}
@@ -293,7 +294,7 @@ public class RLECodec implements ColumnPageCodec {
DataType dataType = columnSpec.getSchemaDataType();
DataInputStream in = new DataInputStream(new ByteArrayInputStream(input, offset, length));
ColumnPage resultPage = ColumnPage.newPage(columnSpec, dataType, pageSize);
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
decodeBytePage(in, resultPage);
} else if (dataType == DataTypes.SHORT) {
decodeShortPage(in, resultPage);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
index 304d998..ed92622 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
@@ -20,10 +20,14 @@ package org.apache.carbondata.core.datastore.page.statistics;
import java.math.BigDecimal;
import org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoderMeta;
+import org.apache.carbondata.core.datastore.page.encoding.bool.BooleanConvert;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import static org.apache.carbondata.core.datastore.page.encoding.bool.BooleanConvert.FALSE_VALUE;
+import static org.apache.carbondata.core.datastore.page.encoding.bool.BooleanConvert.TRUE_VALUE;
+
/** statics for primitive column page */
public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, SimpleStatsResult {
private DataType dataType;
@@ -53,7 +57,7 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
meta.getScale(), meta.getPrecision());
// set min max from meta
DataType dataType = meta.getSchemaDataType();
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
instance.minByte = (byte) meta.getMinValue();
instance.maxByte = (byte) meta.getMaxValue();
} else if (dataType == DataTypes.SHORT) {
@@ -87,7 +91,7 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
new PrimitivePageStatsCollector(DataType.getDataType(meta.getType()), -1, -1);
// set min max from meta
DataType dataType = DataType.getDataType(meta.getType());
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
instance.minByte = (byte) meta.getMinValue();
instance.maxByte = (byte) meta.getMaxValue();
} else if (dataType == DataTypes.SHORT) {
@@ -118,7 +122,10 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
private PrimitivePageStatsCollector(DataType dataType, int scale, int precision) {
this.dataType = dataType;
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN) {
+ minByte = TRUE_VALUE;
+ maxByte = FALSE_VALUE;
+ } else if (dataType == DataTypes.BYTE) {
minByte = Byte.MAX_VALUE;
maxByte = Byte.MIN_VALUE;
} else if (dataType == DataTypes.SHORT) {
@@ -148,7 +155,7 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
@Override
public void updateNull(int rowId) {
long value = 0;
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
update((byte) value);
} else if (dataType == DataTypes.SHORT) {
update((short) value);
@@ -270,7 +277,10 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
@Override
public String toString() {
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return String.format("min: %s, max: %s ", BooleanConvert.byte2Boolean(minByte),
+ BooleanConvert.byte2Boolean(minByte));
+ } else if (dataType == DataTypes.BYTE) {
return String.format("min: %s, max: %s, decimal: %s ", minByte, maxByte, decimal);
} else if (dataType == DataTypes.SHORT) {
return String.format("min: %s, max: %s, decimal: %s ", minShort, maxShort, decimal);
@@ -286,7 +296,7 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
@Override
public Object getMin() {
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
return minByte;
} else if (dataType == DataTypes.SHORT) {
return minShort;
@@ -304,7 +314,7 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
@Override
public Object getMax() {
- if (dataType == DataTypes.BYTE) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.BYTE) {
return maxByte;
} else if (dataType == DataTypes.SHORT) {
return maxShort;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index 6b2cb90..7faa7e6 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -127,7 +127,9 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
return null;
}
// data type object maybe created by GSON, use id to compare the type instead of object address
- if (dataType.getId() == DataTypes.STRING.getId()) {
+ if (dataType.getId() == DataTypes.BOOLEAN.getId()) {
+ return org.apache.carbondata.format.DataType.BOOLEAN;
+ } else if (dataType.getId() == DataTypes.STRING.getId()) {
return org.apache.carbondata.format.DataType.STRING;
} else if (dataType.getId() == DataTypes.INT.getId()) {
return org.apache.carbondata.format.DataType.INT;
@@ -361,6 +363,8 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
return null;
}
switch (dataType) {
+ case BOOLEAN:
+ return DataTypes.BOOLEAN;
case STRING:
return DataTypes.STRING;
case INT:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
index 1fa42dc..7e24413 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
@@ -91,7 +91,9 @@ public abstract class AbstractScannedResultCollector implements ScannedResultCol
CarbonMeasure carbonMeasure) {
if (!dataChunk.getNullBits().get(index)) {
DataType dataType = carbonMeasure.getDataType();
- if (dataType == DataTypes.SHORT) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return dataChunk.getBoolean(index);
+ } else if (dataType == DataTypes.SHORT) {
return (short) dataChunk.getLong(index);
} else if (dataType == DataTypes.INT) {
return (int) dataChunk.getLong(index);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpression.java
index 780ca89..f143189 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpression.java
@@ -67,7 +67,9 @@ public class EqualToExpression extends BinaryConditionalExpression {
}
DataType dataType = val1.getDataType();
- if (dataType == DataTypes.STRING) {
+ if (dataType == DataTypes.BOOLEAN) {
+ result = val1.getBoolean().equals(val2.getBoolean());
+ } else if (dataType == DataTypes.STRING) {
result = val1.getString().equals(val2.getString());
} else if (dataType == DataTypes.SHORT) {
result = val1.getShort().equals(val2.getShort());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpression.java
index 469672e..1472959 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpression.java
@@ -50,7 +50,9 @@ public class GreaterThanEqualToExpression extends BinaryConditionalExpression {
}
boolean result = false;
DataType dataType = exprResVal1.getDataType();
- if (dataType == DataTypes.STRING) {
+ if (dataType == DataTypes.BOOLEAN) {
+ result = elRes.getBoolean().compareTo(erRes.getBoolean()) >= 0;
+ } else if (dataType == DataTypes.STRING) {
result = elRes.getString().compareTo(erRes.getString()) >= 0;
} else if (dataType == DataTypes.SHORT) {
result = elRes.getShort() >= (erRes.getShort());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpression.java
index 931de6c..b8a8a7c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpression.java
@@ -52,7 +52,9 @@ public class GreaterThanExpression extends BinaryConditionalExpression {
}
boolean result = false;
DataType dataType = val1.getDataType();
- if (dataType == DataTypes.STRING) {
+ if (dataType == DataTypes.BOOLEAN) {
+ result = exprLeftRes.getBoolean().compareTo(exprRightRes.getBoolean()) > 0;
+ } else if (dataType == DataTypes.STRING) {
result = exprLeftRes.getString().compareTo(exprRightRes.getString()) > 0;
} else if (dataType == DataTypes.DOUBLE) {
result = exprLeftRes.getDouble() > (exprRightRes.getDouble());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/InExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/InExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/InExpression.java
index bc714f7..7243741 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/InExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/InExpression.java
@@ -54,7 +54,9 @@ public class InExpression extends BinaryConditionalExpression {
val = expressionResVal;
}
DataType dataType = val.getDataType();
- if (dataType == DataTypes.STRING) {
+ if (dataType == DataTypes.BOOLEAN) {
+ val = new ExpressionResult(val.getDataType(), expressionResVal.getBoolean());
+ } else if (dataType == DataTypes.STRING) {
val = new ExpressionResult(val.getDataType(), expressionResVal.getString());
} else if (dataType == DataTypes.SHORT) {
val = new ExpressionResult(val.getDataType(), expressionResVal.getShort());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpression.java
index f416a6b..6a9fc3c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpression.java
@@ -50,7 +50,9 @@ public class LessThanEqualToExpression extends BinaryConditionalExpression {
}
boolean result = false;
DataType dataType = exprResValue1.getDataType();
- if (dataType == DataTypes.STRING) {
+ if (dataType == DataTypes.BOOLEAN) {
+ result = elRes.getBoolean().compareTo(erRes.getBoolean()) <= 0;
+ } else if (dataType == DataTypes.STRING) {
result = elRes.getString().compareTo(erRes.getString()) <= 0;
} else if (dataType == DataTypes.SHORT) {
result = elRes.getShort() <= (erRes.getShort());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpression.java
index c0d7c10..4283d83 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpression.java
@@ -54,7 +54,9 @@ public class LessThanExpression extends BinaryConditionalExpression {
}
DataType dataType = val1.getDataType();
- if (dataType == DataTypes.STRING) {
+ if (dataType == DataTypes.BOOLEAN) {
+ result = elRes.getBoolean().compareTo(erRes.getBoolean()) < 0;
+ } else if (dataType == DataTypes.STRING) {
result = elRes.getString().compareTo(erRes.getString()) < 0;
} else if (dataType == DataTypes.SHORT) {
result = elRes.getShort() < (erRes.getShort());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpression.java
index 8930c94..5055caf 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpression.java
@@ -63,7 +63,9 @@ public class NotEqualsExpression extends BinaryConditionalExpression {
}
}
DataType dataType = val1.getDataType();
- if (dataType == DataTypes.STRING) {
+ if (dataType == DataTypes.BOOLEAN) {
+ result = !val1.getBoolean().equals(val2.getBoolean());
+ } else if (dataType == DataTypes.STRING) {
result = !val1.getString().equals(val2.getString());
} else if (dataType == DataTypes.SHORT) {
result = val1.getShort().shortValue() != val2.getShort().shortValue();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpression.java
index 5f6359b..89a0374 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpression.java
@@ -75,7 +75,9 @@ public class NotInExpression extends BinaryConditionalExpression {
val = exprResVal;
}
DataType dataType = val.getDataType();
- if (dataType == DataTypes.STRING) {
+ if (dataType == DataTypes.BOOLEAN) {
+ val = new ExpressionResult(val.getDataType(), exprResVal.getBoolean());
+ } else if (dataType == DataTypes.STRING) {
val = new ExpressionResult(val.getDataType(), exprResVal.getString());
} else if (dataType == DataTypes.SHORT) {
val = new ExpressionResult(val.getDataType(), exprResVal.getShort());
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index cfe2cc8..ecac617 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -1296,7 +1296,10 @@ public final class FilterUtil {
public static int compareFilterKeyBasedOnDataType(String dictionaryVal, String memberVal,
DataType dataType) {
try {
- if (dataType == DataTypes.SHORT) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return Boolean.compare((Boolean.parseBoolean(dictionaryVal)),
+ (Boolean.parseBoolean(memberVal)));
+ } else if (dataType == DataTypes.SHORT) {
return Short.compare((Short.parseShort(dictionaryVal)), (Short.parseShort(memberVal)));
} else if (dataType == DataTypes.INT) {
return Integer.compare((Integer.parseInt(dictionaryVal)), (Integer.parseInt(memberVal)));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
index 4013578..7a80c88 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -124,7 +124,9 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
}
private DataType getMeasureDataType(MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo) {
- if (msrColumnEvaluatorInfo.getType() == DataTypes.SHORT) {
+ if (msrColumnEvaluatorInfo.getType() == DataTypes.BOOLEAN) {
+ return DataTypes.BOOLEAN;
+ } else if (msrColumnEvaluatorInfo.getType() == DataTypes.SHORT) {
return DataTypes.SHORT;
} else if (msrColumnEvaluatorInfo.getType() == DataTypes.INT) {
return DataTypes.INT;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
index 04fce4a..843da71 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -148,7 +148,9 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
}
private DataType getMeasureDataType(MeasureColumnResolvedFilterInfo msrColumnEvaluatorInfo) {
- if (msrColumnEvaluatorInfo.getType() == DataTypes.SHORT) {
+ if (msrColumnEvaluatorInfo.getType() == DataTypes.BOOLEAN) {
+ return DataTypes.BOOLEAN;
+ } else if (msrColumnEvaluatorInfo.getType() == DataTypes.SHORT) {
return DataTypes.SHORT;
} else if (msrColumnEvaluatorInfo.getType() == DataTypes.INT) {
return DataTypes.INT;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index c243368..dbf8d4c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -351,7 +351,9 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
for (int i = 0; i < msrColEvalutorInfoList.size(); i++) {
MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo = msrColEvalutorInfoList.get(i);
DataType dataType = msrColumnEvalutorInfo.getType();
- if (dataType == DataTypes.SHORT) {
+ if (dataType == DataTypes.BOOLEAN) {
+ msrType = DataTypes.BOOLEAN;
+ } else if (dataType == DataTypes.SHORT) {
msrType = DataTypes.SHORT;
} else if (dataType == DataTypes.INT) {
msrType = DataTypes.INT;
@@ -376,7 +378,9 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
ColumnPage columnPage =
blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]]
.convertToColumnPage(pageIndex);
- if (msrType == DataTypes.SHORT) {
+ if (msrType == DataTypes.BOOLEAN) {
+ msrValue = columnPage.getBoolean(index);
+ } else if (msrType == DataTypes.SHORT) {
msrValue = (short) columnPage.getLong(index);
} else if (msrType == DataTypes.INT) {
msrValue = (int) columnPage.getLong(index);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
index 9b01e1f..cf6c88b 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
@@ -89,6 +89,63 @@ public class MeasureDataVectorProcessor {
}
}
+ /**
+ * Fill Measure Vector For Boolean data type
+ */
+ public static class BooleanMeasureVectorFiller implements MeasureVectorFiller {
+
+ @Override
+ public void fillMeasureVector(ColumnPage dataChunk, ColumnVectorInfo info) {
+ int offset = info.offset;
+ int len = offset + info.size;
+ int vectorOffset = info.vectorOffset;
+ CarbonColumnVector vector = info.vector;
+ BitSet nullBitSet = dataChunk.getNullBits();
+ if (nullBitSet.isEmpty()) {
+ for (int i = offset; i < len; i++) {
+ vector.putBoolean(vectorOffset, dataChunk.getBoolean(i));
+ vectorOffset++;
+ }
+ } else {
+ for (int i = offset; i < len; i++) {
+ if (nullBitSet.get(i)) {
+ vector.putNull(vectorOffset);
+ } else {
+ vector.putBoolean(vectorOffset, dataChunk.getBoolean(i));
+ }
+ vectorOffset++;
+ }
+ }
+ }
+
+ @Override
+ public void fillMeasureVectorForFilter(int[] rowMapping,
+ ColumnPage dataChunk, ColumnVectorInfo info) {
+ int offset = info.offset;
+ int len = offset + info.size;
+ int vectorOffset = info.vectorOffset;
+ CarbonColumnVector vector = info.vector;
+ BitSet nullBitSet = dataChunk.getNullBits();
+ if (nullBitSet.isEmpty()) {
+ for (int i = offset; i < len; i++) {
+ int currentRow = rowMapping[i];
+ vector.putBoolean(vectorOffset, dataChunk.getBoolean(currentRow));
+ vectorOffset++;
+ }
+ } else {
+ for (int i = offset; i < len; i++) {
+ int currentRow = rowMapping[i];
+ if (nullBitSet.get(currentRow)) {
+ vector.putNull(vectorOffset);
+ } else {
+ vector.putBoolean(vectorOffset, dataChunk.getBoolean(currentRow));
+ }
+ vectorOffset++;
+ }
+ }
+ }
+ }
+
public static class ShortMeasureVectorFiller implements MeasureVectorFiller {
@Override
@@ -307,7 +364,9 @@ public class MeasureDataVectorProcessor {
public static class MeasureVectorFillerFactory {
public static MeasureVectorFiller getMeasureVectorFiller(DataType dataType) {
- if (dataType == DataTypes.SHORT) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return new BooleanMeasureVectorFiller();
+ } else if (dataType == DataTypes.SHORT) {
return new ShortMeasureVectorFiller();
} else if (dataType == DataTypes.INT) {
return new IntegralMeasureVectorFiller();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index 1020e5b..15f85d6 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -352,6 +352,8 @@ public abstract class AbstractDataFileFooterConverter {
protected DataType thriftDataTyopeToWrapperDataType(
org.apache.carbondata.format.DataType dataTypeThrift) {
switch (dataTypeThrift) {
+ case BOOLEAN:
+ return DataTypes.BOOLEAN;
case STRING:
return DataTypes.STRING;
case SHORT:
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index 2c9ef66..d232d61 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -344,7 +344,9 @@ public class CarbonMetadataUtil {
private static int compareMeasureData(byte[] first, byte[] second, DataType dataType) {
ByteBuffer firstBuffer = null;
ByteBuffer secondBuffer = null;
- if (dataType == DataTypes.DOUBLE) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return first[0] - second[0];
+ } else if (dataType == DataTypes.DOUBLE) {
firstBuffer = ByteBuffer.allocate(8);
firstBuffer.put(first);
secondBuffer = ByteBuffer.allocate(8);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 2924c09..ef3e71d 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -1938,11 +1938,10 @@ public final class CarbonUtil {
*/
public static byte[] getValueAsBytes(DataType dataType, Object value) {
ByteBuffer b;
- if (dataType == DataTypes.BYTE) {
- b = ByteBuffer.allocate(8);
- b.putLong((byte) value);
- b.flip();
- return b.array();
+ if (dataType == DataTypes.BYTE || dataType == DataTypes.BOOLEAN) {
+ byte[] bytes = new byte[1];
+ bytes[0] = (byte) value;
+ return bytes;
} else if (dataType == DataTypes.SHORT) {
b = ByteBuffer.allocate(8);
b.putLong((short) value);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index b8cd59d..01e34a7 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -34,6 +34,7 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.page.ColumnPage;
+import org.apache.carbondata.core.datastore.page.encoding.bool.BooleanConvert;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.metadata.datatype.DataType;
@@ -102,7 +103,9 @@ public final class DataTypeUtil {
*/
public static Object getMeasureValueBasedOnDataType(String msrValue, DataType dataType,
CarbonMeasure carbonMeasure) {
- if (dataType == DataTypes.DECIMAL) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return BooleanConvert.parseBoolean(msrValue);
+ } else if (dataType == DataTypes.DECIMAL) {
BigDecimal bigDecimal =
new BigDecimal(msrValue).setScale(carbonMeasure.getScale(), RoundingMode.HALF_UP);
return normalizeDecimalValue(bigDecimal, carbonMeasure.getPrecision());
@@ -126,7 +129,9 @@ public final class DataTypeUtil {
return null;
}
ByteBuffer bb = ByteBuffer.wrap(data);
- if (dataType == DataTypes.SHORT) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return BooleanConvert.byte2Boolean(bb.get());
+ } else if (dataType == DataTypes.SHORT) {
return (short) bb.getLong();
} else if (dataType == DataTypes.INT) {
return (int) bb.getLong();
@@ -141,7 +146,9 @@ public final class DataTypeUtil {
public static Object getMeasureObjectBasedOnDataType(ColumnPage measurePage, int index,
DataType dataType, CarbonMeasure carbonMeasure) {
- if (dataType == DataTypes.SHORT) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return measurePage.getBoolean(index);
+ } else if (dataType == DataTypes.SHORT) {
return (short) measurePage.getLong(index);
} else if (dataType == DataTypes.INT) {
return (int) measurePage.getLong(index);
@@ -240,6 +247,9 @@ public final class DataTypeUtil {
public static DataType getDataType(String dataTypeStr) {
DataType dataType = null;
switch (dataTypeStr) {
+ case "BOOLEAN":
+ dataType = DataTypes.BOOLEAN;
+ break;
case "DATE":
dataType = DataTypes.DATE;
break;
@@ -303,7 +313,12 @@ public final class DataTypeUtil {
return null;
}
try {
- if (actualDataType == DataTypes.INT) {
+ if (actualDataType == DataTypes.BOOLEAN) {
+ if (data.isEmpty()) {
+ return null;
+ }
+ return BooleanConvert.parseBoolean(data);
+ } else if (actualDataType == DataTypes.INT) {
if (data.isEmpty()) {
return null;
}
@@ -366,7 +381,9 @@ public final class DataTypeUtil {
public static byte[] getBytesBasedOnDataTypeForNoDictionaryColumn(String dimensionValue,
DataType actualDataType, String dateFormat) {
- if (actualDataType == DataTypes.STRING) {
+ if (actualDataType == DataTypes.BOOLEAN) {
+ return ByteUtil.toBytes(BooleanConvert.parseBoolean(dimensionValue));
+ } else if (actualDataType == DataTypes.STRING) {
return ByteUtil.toBytes(dimensionValue);
} else if (actualDataType == DataTypes.BOOLEAN) {
return ByteUtil.toBytes(Boolean.parseBoolean(dimensionValue));
@@ -411,7 +428,9 @@ public final class DataTypeUtil {
return null;
}
try {
- if (actualDataType == DataTypes.STRING) {
+ if (actualDataType == DataTypes.BOOLEAN) {
+ return ByteUtil.toBoolean(dataInBytes);
+ } else if (actualDataType == DataTypes.STRING) {
return getDataTypeConverter().convertFromByteToUTF8String(dataInBytes);
} else if (actualDataType == DataTypes.BOOLEAN) {
return ByteUtil.toBoolean(dataInBytes);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java b/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java
index a43ed0f..d1beb16 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/comparator/Comparator.java
@@ -26,7 +26,9 @@ import org.apache.carbondata.core.util.ByteUtil;
public final class Comparator {
public static SerializableComparator getComparator(DataType dataType) {
- if (dataType == DataTypes.INT) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return new BooleanSerializableComparator();
+ } else if (dataType == DataTypes.INT) {
return new IntSerializableComparator();
} else if (dataType == DataTypes.SHORT) {
return new ShortSerializableComparator();
@@ -49,7 +51,9 @@ public final class Comparator {
* @return
*/
public static SerializableComparator getComparatorByDataTypeForMeasure(DataType dataType) {
- if (dataType == DataTypes.INT) {
+ if (dataType == DataTypes.BOOLEAN) {
+ return new BooleanSerializableComparator();
+ } else if (dataType == DataTypes.INT) {
return new IntSerializableComparator();
} else if (dataType == DataTypes.SHORT) {
return new ShortSerializableComparator();
@@ -71,6 +75,26 @@ class ByteArraySerializableComparator implements SerializableComparator {
}
}
+class BooleanSerializableComparator implements SerializableComparator {
+ @Override
+ public int compare(Object key1, Object key2) {
+ if (key1 == null && key2 == null) {
+ return 0;
+ } else if (key1 == null) {
+ return -1;
+ } else if (key2 == null) {
+ return 1;
+ }
+ if (Boolean.compare((boolean) key1, (boolean) key2) < 0) {
+ return -1;
+ } else if (Boolean.compare((boolean) key1, (boolean) key2) > 0) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+}
+
class IntSerializableComparator implements SerializableComparator {
@Override public int compare(Object key1, Object key2) {
if (key1 == null && key2 == null) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
index 8bce684..4a3ef32 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
@@ -414,7 +414,11 @@ public class ThriftWrapperSchemaConverterImplTest {
assertEquals(expectedResult, actualResult);
}
- @Test public void testFromWrapperToExternalColumnSchema() {
+ @Test public void testFromWrapperToExternalColumnSchemaForBooleanDataType() {
+ org.apache.carbondata.format.ColumnSchema thriftColumnSchema =
+ new org.apache.carbondata.format.ColumnSchema(org.apache.carbondata.format.DataType.BOOLEAN,
+ "columnName", "1", true, encoders, true);
+ thriftColumnSchema.setSchemaOrdinal(1);
ColumnSchema wrapperColumnSchema = new ColumnSchema();
new MockUp<ColumnSchema>() {
@@ -1304,7 +1308,7 @@ public class ThriftWrapperSchemaConverterImplTest {
encoders.add(null);
org.apache.carbondata.format.ColumnSchema thriftColumnSchema = null;
thriftColumnSchema =
- new org.apache.carbondata.format.ColumnSchema(org.apache.carbondata.format.DataType.STRING,
+ new org.apache.carbondata.format.ColumnSchema(org.apache.carbondata.format.DataType.BOOLEAN,
"columnName", "1", true, encoders, true);
thriftColumnSchema.setSchemaOrdinal(1);
@@ -1327,7 +1331,7 @@ public class ThriftWrapperSchemaConverterImplTest {
}
@Mock public DataType getDataType() {
- return DataTypes.BOOLEAN;
+ return DataTypes.STRING;
}
@Mock public String getColumnName() {
@@ -1445,7 +1449,7 @@ public class ThriftWrapperSchemaConverterImplTest {
}
@Mock public DataType getDataType() {
- return DataTypes.BOOLEAN;
+ return DataTypes.STRING;
}
@Mock public String getColumnName() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java
index 0abfb18..51bb0fe 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java
@@ -182,7 +182,7 @@ public class EqualToExpressionUnitTest {
}
}
- @Test(expected = FilterUnsupportedException.class) public void testForEqualToExpressionForDefaultCase()
+ @Test public void testForEqualToExpressionForDefaultCase()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("contact", DataTypes.BOOLEAN);
right.setColIndex(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java
index 35d8886..0375a6d 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java
@@ -189,7 +189,7 @@ public class GreaterThanEqualToExpressionUnitTest {
}
}
- @Test(expected = FilterUnsupportedException.class) public void testForGreaterThanEqualToExpressionWithDefaultCase()
+ @Test public void testForGreaterThanEqualToExpressionWithDefaultCase()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("contact", DataTypes.BOOLEAN);
right.setColIndex(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
index 7d6a168..1940069 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
@@ -279,7 +279,7 @@ public class GreaterThanExpressionUnitTest {
assertTrue(result.getBoolean());
}
- @Test(expected = FilterUnsupportedException.class) public void testForGreaterThanExpressionWithDefaultCase()
+ @Test public void testForGreaterThanExpressionWithDefaultCase()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("contact", DataTypes.BOOLEAN);
right.setColIndex(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java
index 2fd6585..d31ec31 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java
@@ -224,7 +224,7 @@ public class InExpressionUnitTest {
assertTrue(result.getBoolean());
}
- @Test(expected = FilterUnsupportedException.class) public void testForInExpressionWithDefaultCase()
+ @Test public void testForInExpressionWithDefaultCase()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression left = new ColumnExpression("contact", DataTypes.BOOLEAN);
left.setColIndex(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java
index 7000ceb..a2a5d2a 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java
@@ -282,7 +282,7 @@ public class LessThanEqualToExpressionUnitTest {
assertTrue(result.getBoolean());
}
- @Test(expected = FilterUnsupportedException.class) public void testForLessThanEqualToExpressionWithDefaultCase()
+ @Test public void testForLessThanEqualToExpressionWithDefaultCase()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("contact", DataTypes.BOOLEAN);
right.setColIndex(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java
index 889ebbd..d385dc1 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java
@@ -279,7 +279,7 @@ public class LessThanExpressionUnitTest {
assertTrue(result.getBoolean());
}
- @Test(expected = FilterUnsupportedException.class) public void testForLessThanExpressionWithDefaultCase()
+ @Test public void testForLessThanExpressionWithDefaultCase()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("contact", DataTypes.BOOLEAN);
right.setColIndex(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java
index 1cb008c..e7fa544 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java
@@ -247,7 +247,7 @@ public class NotEqualsExpressionUnitTest {
}
}
- @Test(expected = FilterUnsupportedException.class) public void testForNotEqualsExpressionWithDefaultCase()
+ @Test public void testForNotEqualsExpressionWithDefaultCase()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression right = new ColumnExpression("contact", DataTypes.BOOLEAN);
right.setColIndex(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java
index 230e305..5758625 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java
@@ -223,7 +223,7 @@ public class NotInExpressionUnitTest {
assertTrue(result.getBoolean());
}
- @Test(expected = FilterUnsupportedException.class) public void testDefaultCaseForNotInExpression()
+ @Test public void testDefaultCaseForNotInExpression()
throws FilterUnsupportedException, FilterIllegalMemberException {
ColumnExpression left = new ColumnExpression("contact", DataTypes.BOOLEAN);
left.setColIndex(0);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala
index 5fadfb7..31cb4d0 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala
@@ -99,6 +99,9 @@ class DataCompactionBoundaryConditionsTest extends QueryTest with BeforeAndAfter
sql("drop table if exists boundarytest")
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+ CarbonProperties.getInstance().
+ addProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD,
+ CarbonCommonConstants.DEFAULT_SEGMENT_LEVEL_THRESHOLD)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
index 027c654..89a3ac3 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
@@ -26,6 +26,7 @@ object DataTypeConverterUtil {
def convertToCarbonType(dataType: String): DataType = {
dataType.toLowerCase match {
+ case "boolean" => DataTypes.BOOLEAN
case "string" => DataTypes.STRING
case "int" => DataTypes.INT
case "integer" => DataTypes.INT
@@ -48,6 +49,7 @@ object DataTypeConverterUtil {
def convertToCarbonTypeForSpark2(dataType: String): DataType = {
dataType.toLowerCase match {
+ case "booleantype" => DataTypes.BOOLEAN
case "stringtype" => DataTypes.STRING
case "inttype" => DataTypes.INT
case "integertype" => DataTypes.INT
@@ -79,6 +81,7 @@ object DataTypeConverterUtil {
def convertToString(dataType: DataType): String = {
dataType match {
+ case DataTypes.BOOLEAN => "boolean"
case DataTypes.STRING => "string"
case DataTypes.SHORT => "smallint"
case DataTypes.INT => "int"
@@ -106,6 +109,7 @@ object DataTypeConverterUtil {
dataType match {
case "string" => ThriftDataType.STRING
case "int" => ThriftDataType.INT
+ case "boolean" => ThriftDataType.BOOLEAN
case "short" => ThriftDataType.SHORT
case "long" | "bigint" => ThriftDataType.LONG
case "double" => ThriftDataType.DOUBLE
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 16301d6..4649082 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -163,6 +163,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
protected val FLOAT = carbonKeyWord("FLOAT")
protected val SHORT = carbonKeyWord("SHORT")
protected val INT = carbonKeyWord("INT")
+ protected val BOOLEAN = carbonKeyWord("BOOLEAN")
protected val BIGINT = carbonKeyWord("BIGINT")
protected val ARRAY = carbonKeyWord("ARRAY")
protected val STRUCT = carbonKeyWord("STRUCT")
@@ -954,7 +955,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
protected lazy val dimCol: Parser[Field] = anyFieldDef
protected lazy val primitiveTypes =
- STRING ^^^ "string" | INTEGER ^^^ "integer" |
+ STRING ^^^ "string" |BOOLEAN ^^^ "boolean" | INTEGER ^^^ "integer" |
TIMESTAMP ^^^ "timestamp" | NUMERIC ^^^ "numeric" |
BIGINT ^^^ "bigint" | (SHORT | SMALLINT) ^^^ "smallint" |
INT ^^^ "int" | DOUBLE ^^^ "double" | FLOAT ^^^ "double" | decimalType |
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
index 1183d94..1d3783e 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
@@ -206,7 +206,8 @@ class VectorizedCarbonRecordReader extends AbstractRecordReader<Object> {
for (int i = 0; i < queryMeasures.size(); i++) {
QueryMeasure msr = queryMeasures.get(i);
DataType dataType = msr.getMeasure().getDataType();
- if (dataType == DataTypes.SHORT || dataType == DataTypes.INT || dataType == DataTypes.LONG) {
+ if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.SHORT ||
+ dataType == DataTypes.INT || dataType == DataTypes.LONG) {
fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(),
CarbonScalaUtil.convertCarbonToSparkDataType(msr.getMeasure().getDataType()), true,
null);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/resources/bool/supportBoolean.csv
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/resources/bool/supportBoolean.csv b/integration/spark2/src/test/resources/bool/supportBoolean.csv
new file mode 100644
index 0000000..25e57d7
--- /dev/null
+++ b/integration/spark2/src/test/resources/bool/supportBoolean.csv
@@ -0,0 +1,10 @@
+1,true,10,1100,48.4,spark,2015/4/23 12:01:01,1.23,2015/4/23,aaa,2.5,'foo'#'bar'#'world'
+5,false,17,1140,43.4,spark,2015/7/27 12:01:02,3.45,2015/7/27,bbb,2.5,'foo'#'bar'#'world'
+1,false,11,1100,44.4,flink,2015/5/23 12:01:03,23.23,2015/5/23,ccc,2.5,'foo'#'bar'#'world'
+1,true,10,1150,43.4,spark,2015/7/24 12:01:04,254.12,2015/7/24,ddd,2.5,'foo'#'bar'#'world'
+1,true,10,1100,47.4,spark,2015/7/23 12:01:05,876.14,2015/7/23,eeee,3.5,'foo'#'bar'#'world'
+3,true,14,1160,43.4,hive,2015/7/26 12:01:06,3454.32,2015/7/26,ff,2.5,'foo'#'bar'#'world'
+2,false,10,1100,43.4,impala,2015/7/23 12:01:07,456.98,2015/7/23,ggg,2.5,'foo'#'bar'#'world'
+1,false,10,1100,43.4,spark,2015/5/23 12:01:08,32.53,2015/5/23,hhh,2.5,'foo'#'bar'#'world'
+4,false,16,1130,42.4,impala,2015/7/23 12:01:09,67.23,2015/7/23,iii,2.5,'foo'#'bar'#'world'
+1,false,10,1100,43.4,spark,2015/7/23 12:01:10,832.23,2015/7/23,jjj,2.5,'foo'#'bar'#'world'
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/resources/bool/supportBooleanBadRecords.csv
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/resources/bool/supportBooleanBadRecords.csv b/integration/spark2/src/test/resources/bool/supportBooleanBadRecords.csv
new file mode 100644
index 0000000..c9c769f
--- /dev/null
+++ b/integration/spark2/src/test/resources/bool/supportBooleanBadRecords.csv
@@ -0,0 +1,10 @@
+1,true,10,1100,48.4,spark,2015/4/23 12:01:01,1.23,2015/4/23,aaa,2.5,'foo'#'bar'#'world',true
+5,falsee,17,1140,43.4,spark,2015/7/27 12:01:02,3.45,2015/7/27,bbb,2.5,'foo'#'bar'#'world',true
+1,f,11,1100,44.4,flink,2015/5/23 12:01:03,23.23,2015/5/23,ccc,2.5,'foo'#'bar'#'world',true
+1,truee,10,1150,43.4,spark,2015/7/24 12:01:04,254.12,2015/7/24,ddd,2.5,'foo'#'bar'#'world',true
+1,truea,10,1100,47.4,spark,2015/7/23 12:01:05,876.14,2015/7/23,eeee,3.5,'foo'#'bar'#'world',true
+3,true,14,1160,43.4,hive,2015/7/26 12:01:06,3454.32,2015/7/26,ff,2.5,'foo'#'bar'#'world',falsee
+2,false,10,1100,43.4,impala,2015/7/23 12:01:07,456.98,2015/7/23,ggg,2.5,'foo'#'bar'#'world',falsea
+1,false,10,1100,43.4,spark,2015/5/23 12:01:08,32.53,2015/5/23,hhh,2.5,'foo'#'bar'#'world',falsef
+4,false,16,1130,42.4,impala,2015/7/23 12:01:09,67.23,2015/7/23,iii,2.5,'foo'#'bar'#'world',falsea
+1,false,10,1100,43.4,spark,2015/7/23 12:01:10,832.23,2015/7/23,jjj,2.5,'foo'#'bar'#'world',false
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/resources/bool/supportBooleanDifferentFormat.csv
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/resources/bool/supportBooleanDifferentFormat.csv b/integration/spark2/src/test/resources/bool/supportBooleanDifferentFormat.csv
new file mode 100644
index 0000000..032e92b
--- /dev/null
+++ b/integration/spark2/src/test/resources/bool/supportBooleanDifferentFormat.csv
@@ -0,0 +1,20 @@
+True
+TRUE
+true
+"true"
+false
+False
+FALSE
+"FALSE"
+
+null
+NULL
+TRUEA
+true"
+'true'
+truee
+'false'
+falsee
+FFALSE
+f
+t
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/resources/bool/supportBooleanOnlyBoolean.csv
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/resources/bool/supportBooleanOnlyBoolean.csv b/integration/spark2/src/test/resources/bool/supportBooleanOnlyBoolean.csv
new file mode 100644
index 0000000..3ce21d8
--- /dev/null
+++ b/integration/spark2/src/test/resources/bool/supportBooleanOnlyBoolean.csv
@@ -0,0 +1,11 @@
+True
+TRUE
+true
+"true"
+false
+False
+FALSE
+"FALSE"
+
+null
+NULL
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv b/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv
new file mode 100644
index 0000000..696befb
--- /dev/null
+++ b/integration/spark2/src/test/resources/bool/supportBooleanTwoBooleanColumns.csv
@@ -0,0 +1,10 @@
+1,true,10,1100,48.4,spark,2015/4/23 12:01:01,1.23,2015/4/23,aaa,2.5,'foo'#'bar'#'world',true
+5,false,17,1140,43.4,spark,2015/7/27 12:01:02,3.45,2015/7/27,bbb,2.5,'foo'#'bar'#'world',true
+1,false,11,1100,44.4,flink,2015/5/23 12:01:03,23.23,2015/5/23,ccc,2.5,'foo'#'bar'#'world',true
+1,true,10,1150,43.4,spark,2015/7/24 12:01:04,254.12,2015/7/24,ddd,2.5,'foo'#'bar'#'world',true
+1,true,10,1100,47.4,spark,2015/7/23 12:01:05,876.14,2015/7/23,eeee,3.5,'foo'#'bar'#'world',true
+3,true,14,1160,43.4,hive,2015/7/26 12:01:06,3454.32,2015/7/26,ff,2.5,'foo'#'bar'#'world',false
+2,false,10,1100,43.4,impala,2015/7/23 12:01:07,456.98,2015/7/23,ggg,2.5,'foo'#'bar'#'world',false
+1,false,10,1100,43.4,spark,2015/5/23 12:01:08,32.53,2015/5/23,hhh,2.5,'foo'#'bar'#'world',false
+4,false,16,1130,42.4,impala,2015/7/23 12:01:09,67.23,2015/7/23,iii,2.5,'foo'#'bar'#'world',false
+1,false,10,1100,43.4,spark,2015/7/23 12:01:10,832.23,2015/7/23,jjj,2.5,'foo'#'bar'#'world',false
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/6abdd97a/integration/spark2/src/test/resources/bool/supportBooleanWithFileHeader.csv
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/resources/bool/supportBooleanWithFileHeader.csv b/integration/spark2/src/test/resources/bool/supportBooleanWithFileHeader.csv
new file mode 100644
index 0000000..237eade
--- /dev/null
+++ b/integration/spark2/src/test/resources/bool/supportBooleanWithFileHeader.csv
@@ -0,0 +1,11 @@
+shortField,booleanField,intField,bigintField,doubleField,stringField,timestampField,decimalField,dateField,charField,floatField,complexData
+1,true,10,1100,48.4,spark,2015/4/23 12:01:01,1.23,2015/4/23,aaa,2.5,'foo'#'bar'#'world'
+5,false,17,1140,43.4,spark,2015/7/27 12:01:02,3.45,2015/7/27,bbb,2.5,'foo'#'bar'#'world'
+1,false,11,1100,44.4,flink,2015/5/23 12:01:03,23.23,2015/5/23,ccc,2.5,'foo'#'bar'#'world'
+1,true,10,1150,43.4,spark,2015/7/24 12:01:04,254.12,2015/7/24,ddd,2.5,'foo'#'bar'#'world'
+1,true,10,1100,47.4,spark,2015/7/23 12:01:05,876.14,2015/7/23,eeee,3.5,'foo'#'bar'#'world'
+3,true,14,1160,43.4,hive,2015/7/26 12:01:06,3454.32,2015/7/26,ff,2.5,'foo'#'bar'#'world'
+2,false,10,1100,43.4,impala,2015/7/23 12:01:07,456.98,2015/7/23,ggg,2.5,'foo'#'bar'#'world'
+1,false,10,1100,43.4,spark,2015/5/23 12:01:08,32.53,2015/5/23,hhh,2.5,'foo'#'bar'#'world'
+4,false,16,1130,42.4,impala,2015/7/23 12:01:09,67.23,2015/7/23,iii,2.5,'foo'#'bar'#'world'
+1,false,10,1100,43.4,spark,2015/7/23 12:01:10,832.23,2015/7/23,jjj,2.5,'foo'#'bar'#'world'