You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/05/02 14:13:08 UTC
[44/50] [abbrv] incubator-carbondata git commit: add SORT_COLUMNS
options
add SORT_COLUMNS options
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/7dafae7f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/7dafae7f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/7dafae7f
Branch: refs/heads/12-dev
Commit: 7dafae7f3770728c360885881884c10b69141a58
Parents: 1edabc8
Author: QiangCai <qi...@qq.com>
Authored: Tue Apr 18 17:12:14 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Tue May 2 09:42:20 2017 +0800
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 1 +
.../core/datastore/block/SegmentProperties.java | 42 +++
.../impl/VariableLengthDimensionDataChunk.java | 3 +-
.../impl/btree/BTreeDataRefNodeFinder.java | 24 +-
.../core/keygenerator/mdkey/Bits.java | 6 +-
.../ThriftWrapperSchemaConverterImpl.java | 18 +
.../core/metadata/schema/table/CarbonTable.java | 24 ++
.../schema/table/column/ColumnSchema.java | 10 +
.../impl/DictionaryBasedResultCollector.java | 12 +-
...structureBasedDictionaryResultCollector.java | 4 +-
.../scan/filter/FilterExpressionProcessor.java | 4 +-
.../carbondata/core/scan/filter/FilterUtil.java | 98 ++++--
...velRangeLessThanEqualFilterExecuterImpl.java | 2 +-
.../RowLevelRangeLessThanFiterExecuterImpl.java | 2 +-
.../visitor/NoDictionaryTypeVisitor.java | 3 +-
.../visitor/RangeNoDictionaryTypeVisitor.java | 3 +-
.../AbstractDetailQueryResultIterator.java | 4 +-
.../scan/result/vector/CarbonColumnVector.java | 7 +
.../util/AbstractDataFileFooterConverter.java | 7 +
.../apache/carbondata/core/util/ByteUtil.java | 336 ++++++++++++++++++-
.../apache/carbondata/core/util/CarbonUtil.java | 18 +
.../carbondata/core/util/DataTypeUtil.java | 70 ++++
.../impl/btree/BTreeBlockFinderTest.java | 10 +-
.../core/scan/filter/FilterUtilTest.java | 7 +-
.../carbondata/hadoop/CarbonInputFormat.java | 4 +-
.../internal/index/impl/InMemoryBTreeIndex.java | 4 +-
.../impl/DictionaryDecodeReadSupport.java | 3 +
.../hadoop/util/BlockLevelTraverser.java | 4 +-
.../hadoop/test/util/StoreCreator.java | 5 +
.../testsuite/sortcolumns/TestSortColumns.scala | 267 +++++++++++++++
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 125 +++----
.../execution/command/carbonTableSchema.scala | 66 ++--
.../apache/spark/sql/TestCarbonSqlParser.scala | 56 +---
.../readsupport/SparkRowReadSupportImpl.java | 13 +-
.../vectorreader/ColumnarVectorWrapper.java | 13 +
.../org/apache/spark/sql/TableCreator.scala | 145 ++++----
.../merger/CompactionResultSortProcessor.java | 10 +-
.../merger/RowResultMergerProcessor.java | 14 +-
.../newflow/CarbonDataLoadConfiguration.java | 24 ++
.../newflow/DataLoadProcessBuilder.java | 24 +-
.../impl/NonDictionaryFieldConverterImpl.java | 21 +-
.../sort/impl/ParallelReadMergeSorterImpl.java | 3 +-
...arallelReadMergeSorterWithBucketingImpl.java | 3 +-
.../sort/unsafe/UnsafeCarbonRowPage.java | 11 +-
.../newflow/sort/unsafe/UnsafeSortDataRows.java | 11 +-
.../unsafe/comparator/UnsafeRowComparator.java | 10 +-
.../UnsafeRowComparatorForNormalDIms.java | 8 +-
.../holder/UnsafeFinalMergePageHolder.java | 4 +-
.../unsafe/holder/UnsafeInmemoryHolder.java | 5 +-
.../holder/UnsafeSortTempFileChunkHolder.java | 2 +-
.../UnsafeSingleThreadFinalSortFilesMerger.java | 4 +-
.../newflow/steps/NoSortProcessorStepImpl.java | 153 +++++++++
.../sortdata/IntermediateFileMerger.java | 3 +-
.../sortdata/NewRowComparator.java | 12 +-
.../sortdata/NewRowComparatorForNormalDims.java | 10 +-
.../sortandgroupby/sortdata/RowComparator.java | 10 +-
.../sortdata/RowComparatorForNormalDims.java | 10 +-
.../sortandgroupby/sortdata/SortDataRows.java | 13 +-
.../sortandgroupby/sortdata/SortParameters.java | 43 +++
.../sortdata/SortTempFileChunkHolder.java | 11 +-
.../store/CarbonFactDataHandlerColumnar.java | 45 ++-
.../store/CarbonFactDataHandlerModel.java | 7 +
.../store/SingleThreadFinalSortFilesMerger.java | 8 +-
63 files changed, 1549 insertions(+), 350 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index ee18321..4a79e0c 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -805,6 +805,7 @@ public final class CarbonCommonConstants {
public static final String COLUMN_GROUPS = "column_groups";
public static final String DICTIONARY_EXCLUDE = "dictionary_exclude";
public static final String DICTIONARY_INCLUDE = "dictionary_include";
+ public static final String SORT_COLUMNS = "sort_columns";
public static final String PARTITIONCLASS = "partitionclass";
public static final String PARTITIONCOUNT = "partitioncount";
public static final String COLUMN_PROPERTIES = "columnproperties";
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
index 3bc208d..38e3c02 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
@@ -55,6 +55,14 @@ public class SegmentProperties {
private KeyGenerator dimensionKeyGenerator;
/**
+ * key generator which was used to generate the mdkey for dimensions in SORT_COLUMNS
+ * if SORT_COLUMNS contains all dimensions, it is same with dimensionKeyGenerator
+ * otherwise, it is different with dimensionKeyGenerator, the number of its dimensions is less
+ * than dimensionKeyGenerator.
+ */
+ private KeyGenerator sortColumnsGenerator;
+
+ /**
* list of dimension present in the block
*/
private List<CarbonDimension> dimensions;
@@ -151,6 +159,10 @@ public class SegmentProperties {
*/
private ColumnGroupModel colGroupModel;
+ private int numberOfSortColumns = 0;
+
+ private int numberOfNoDictSortColumns = 0;
+
public SegmentProperties(List<ColumnSchema> columnsInTable, int[] columnCardinality) {
dimensions = new ArrayList<CarbonDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
complexDimensions =
@@ -336,6 +348,9 @@ public class SegmentProperties {
if (CarbonUtil.hasEncoding(columnSchema.getEncodingList(), Encoding.DICTIONARY)
&& !isComplexDimensionStarted && columnSchema.getNumberOfChild() == 0) {
cardinalityIndexForNormalDimensionColumn.add(tableOrdinal);
+ if (columnSchema.isSortColumn()) {
+ this.numberOfSortColumns++;
+ }
if (columnSchema.isColumnar()) {
// if it is a columnar dimension participated in mdkey then added
// key ordinal and dimension ordinal
@@ -385,6 +400,10 @@ public class SegmentProperties {
// for no dictionary dimension
carbonDimension = new CarbonDimension(columnSchema, dimensonOrdinal++, -1, -1, -1);
numberOfNoDictionaryDimension++;
+ if (columnSchema.isSortColumn()) {
+ this.numberOfSortColumns++;
+ this.numberOfNoDictSortColumns++;
+ }
}
dimensions.add(carbonDimension);
} else {
@@ -511,6 +530,14 @@ public class SegmentProperties {
int[] bitLength = CarbonUtil.getDimensionBitLength(dimColumnsCardinality, dimensionPartitions);
// create a key generator
this.dimensionKeyGenerator = new MultiDimKeyVarLengthGenerator(bitLength);
+ if (this.getNumberOfDictSortColumns() == bitLength.length) {
+ this.sortColumnsGenerator = this.dimensionKeyGenerator;
+ } else {
+ int numberOfDictSortColumns = this.getNumberOfDictSortColumns();
+ int [] sortColumnBitLength = new int[numberOfDictSortColumns];
+ System.arraycopy(bitLength, 0, sortColumnBitLength, 0, numberOfDictSortColumns);
+ this.sortColumnsGenerator = new MultiDimKeyVarLengthGenerator(sortColumnBitLength);
+ }
this.fixedLengthKeySplitter =
new MultiDimKeyVarLengthVariableSplitGenerator(bitLength, dimensionPartitions);
// get the size of each value in file block
@@ -636,6 +663,10 @@ public class SegmentProperties {
return dimensionKeyGenerator;
}
+ public KeyGenerator getSortColumnsGenerator() {
+ return sortColumnsGenerator;
+ }
+
/**
* @return the dimensions
*/
@@ -797,4 +828,15 @@ public class SegmentProperties {
return CarbonUtil.getMeasureFromCurrentBlock(this.measures, columnId);
}
+ public int getNumberOfSortColumns() {
+ return numberOfSortColumns;
+ }
+
+ public int getNumberOfNoDictSortColumns() {
+ return numberOfNoDictSortColumns;
+ }
+
+ public int getNumberOfDictSortColumns() {
+ return this.numberOfSortColumns - this.numberOfNoDictSortColumns;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
index 6c47bf5..0ac5655 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
@@ -18,7 +18,8 @@ package org.apache.carbondata.core.datastore.chunk.impl;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory
+ .DimensionStoreType;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
index 2e1709b..2f8aadf 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
@@ -49,18 +49,20 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
private int[] eachColumnValueSize;
/**
- * this will be used during search for no dictionary column
+ * the number of no dictionary columns in SORT_COLUMNS
*/
- private int numberOfNoDictionaryColumns;
+ private int numberOfNoDictSortColumns;
- public BTreeDataRefNodeFinder(int[] eachColumnValueSize) {
- this.eachColumnValueSize = eachColumnValueSize;
+ /**
+ * the number of columns in SORT_COLUMNS
+ */
+ private int numberOfSortColumns;
- for (int i = 0; i < eachColumnValueSize.length; i++) {
- if (eachColumnValueSize[i] == -1) {
- numberOfNoDictionaryColumns++;
- }
- }
+ public BTreeDataRefNodeFinder(int[] eachColumnValueSize, int numberOfSortColumns,
+ int numberOfNoDictSortColumns) {
+ this.eachColumnValueSize = eachColumnValueSize;
+ this.numberOfNoDictSortColumns = numberOfNoDictSortColumns;
+ this.numberOfSortColumns = numberOfSortColumns;
}
/**
@@ -213,7 +215,7 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
int dictionaryKeyOffset = 0;
int nonDictionaryKeyOffset = 0;
int compareResult = 0;
- int processedNoDictionaryColumn = numberOfNoDictionaryColumns;
+ int processedNoDictionaryColumn = numberOfNoDictSortColumns;
ByteBuffer firstNoDictionaryKeyBuffer = ByteBuffer.wrap(first.getNoDictionaryKeys());
ByteBuffer secondNoDictionaryKeyBuffer = ByteBuffer.wrap(second.getNoDictionaryKeys());
int actualOffset = 0;
@@ -221,7 +223,7 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
int firstNoDcitionaryLength = 0;
int secondNodeDictionaryLength = 0;
- for (int i = 0; i < eachColumnValueSize.length; i++) {
+ for (int i = 0; i < numberOfSortColumns; i++) {
if (eachColumnValueSize[i] != NO_DCITIONARY_COLUMN_VALUE) {
compareResult = ByteUtil.UnsafeComparer.INSTANCE
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
index 7bd338d..044a77b 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/mdkey/Bits.java
@@ -119,7 +119,8 @@ public class Bits implements Serializable {
protected long[] get(long[] keys) {
long[] words = new long[wsize];
int ll = 0;
- for (int i = lens.length - 1; i >= 0; i--) {
+ int minLength = Math.min(lens.length, keys.length);
+ for (int i = minLength - 1; i >= 0; i--) {
long val = keys[i];
@@ -155,7 +156,8 @@ public class Bits implements Serializable {
protected long[] get(int[] keys) {
long[] words = new long[wsize];
int ll = 0;
- for (int i = lens.length - 1; i >= 0; i--) {
+ int minLength = Math.min(lens.length, keys.length);
+ for (int i = minLength - 1; i >= 0; i--) {
long val = keys[i];
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index 974cc81..3d114ec 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -17,8 +17,11 @@
package org.apache.carbondata.core.metadata.converter;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.datatype.DataType;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.BucketingInfo;
@@ -172,6 +175,13 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
thriftColumnSchema.setInvisible(wrapperColumnSchema.isInvisible());
thriftColumnSchema.setColumnReferenceId(wrapperColumnSchema.getColumnReferenceId());
thriftColumnSchema.setSchemaOrdinal(wrapperColumnSchema.getSchemaOrdinal());
+
+ if (wrapperColumnSchema.isSortColumn()) {
+ Map<String, String> properties = new HashMap<String, String>();
+ properties.put(CarbonCommonConstants.SORT_COLUMNS, "true");
+ thriftColumnSchema.setColumnProperties(properties);
+ }
+
return thriftColumnSchema;
}
@@ -360,6 +370,14 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
wrapperColumnSchema.setInvisible(externalColumnSchema.isInvisible());
wrapperColumnSchema.setColumnReferenceId(externalColumnSchema.getColumnReferenceId());
wrapperColumnSchema.setSchemaOrdinal(externalColumnSchema.getSchemaOrdinal());
+ wrapperColumnSchema.setSortColumn(false);
+ Map<String, String> properties = externalColumnSchema.getColumnProperties();
+ if (properties != null) {
+ String sortColumns = properties.get(CarbonCommonConstants.SORT_COLUMNS);
+ if (sortColumns != null) {
+ wrapperColumnSchema.setSortColumn(true);
+ }
+ }
return wrapperColumnSchema;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index dbc8836..fae9960 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -116,6 +116,16 @@ public class CarbonTable implements Serializable {
*/
private int blockSize;
+ /**
+ * the number of columns in SORT_COLUMNS
+ */
+ private int numberOfSortColumns;
+
+ /**
+ * the number of no dictionary columns in SORT_COLUMNS
+ */
+ private int numberOfNoDictSortColumns;
+
public CarbonTable() {
this.tableDimensionsMap = new HashMap<String, List<CarbonDimension>>();
this.tableImplicitDimensionsMap = new HashMap<String, List<CarbonDimension>>();
@@ -238,10 +248,16 @@ public class CarbonTable implements Serializable {
i = dimensionOrdinal - 1;
complexTypeOrdinal = assignComplexOrdinal(complexDimension, complexTypeOrdinal);
} else {
+ if (!columnSchema.isInvisible() && columnSchema.isSortColumn()) {
+ this.numberOfSortColumns++;
+ }
if (!columnSchema.getEncodingList().contains(Encoding.DICTIONARY)) {
CarbonDimension dimension =
new CarbonDimension(columnSchema, dimensionOrdinal++,
columnSchema.getSchemaOrdinal(), -1, -1, -1);
+ if (!columnSchema.isInvisible() && columnSchema.isSortColumn()) {
+ this.numberOfNoDictSortColumns++;
+ }
allDimensions.add(dimension);
primitiveDimensions.add(dimension);
} else if (columnSchema.getEncodingList().contains(Encoding.DICTIONARY)
@@ -639,4 +655,12 @@ public class CarbonTable implements Serializable {
}
tableMeasuresMap.put(tableName, visibleMeasures);
}
+
+ public int getNumberOfSortColumns() {
+ return numberOfSortColumns;
+ }
+
+ public int getNumberOfNoDictSortColumns() {
+ return numberOfNoDictSortColumns;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
index abe065c..6250dba 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
@@ -109,6 +109,8 @@ public class ColumnSchema implements Serializable {
*/
private boolean invisible = false;
+ private boolean isSortColumn = false;
+
/**
* @return the columnName
*/
@@ -411,4 +413,12 @@ public class ColumnSchema implements Serializable {
public void setSchemaOrdinal(int schemaOrdinal) {
this.schemaOrdinal = schemaOrdinal;
}
+
+ public boolean isSortColumn() {
+ return isSortColumn;
+ }
+
+ public void setSortColumn(boolean sortColumn) {
+ isSortColumn = sortColumn;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
index e5e4b3c..a7c93bb 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
@@ -88,7 +88,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
List<Object[]> listBasedResult = new ArrayList<>(batchSize);
int rowCounter = 0;
int[] surrogateResult;
- String[] noDictionaryKeys;
+ byte[][] noDictionaryKeys;
byte[][] complexTypeKeyArray;
BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache =
scannedResult.getDeleteDeltaDataCache();
@@ -96,7 +96,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
Object[] row = new Object[queryDimensions.length + queryMeasures.length];
if (isDimensionExists) {
surrogateResult = scannedResult.getDictionaryKeyIntegerArray();
- noDictionaryKeys = scannedResult.getNoDictionaryKeyStringArray();
+ noDictionaryKeys = scannedResult.getNoDictionaryKeyArray();
complexTypeKeyArray = scannedResult.getComplexTypeKeyArray();
dictionaryColumnIndex = 0;
noDictionaryColumnIndex = 0;
@@ -120,7 +120,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
}
protected void fillDimensionData(AbstractScannedResult scannedResult, int[] surrogateResult,
- String[] noDictionaryKeys, byte[][] complexTypeKeyArray,
+ byte[][] noDictionaryKeys, byte[][] complexTypeKeyArray,
Map<Integer, GenericQueryType> comlexDimensionInfoMap, Object[] row, int i) {
if (!dictionaryEncodingArray[i]) {
if (implictColumnArray[i]) {
@@ -134,9 +134,9 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
DataTypeUtil.getDataBasedOnDataType(scannedResult.getBlockletId(), DataType.STRING);
}
} else {
- row[order[i]] = DataTypeUtil
- .getDataBasedOnDataType(noDictionaryKeys[noDictionaryColumnIndex++],
- queryDimensions[i].getDimension().getDataType());
+ row[order[i]] = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(
+ noDictionaryKeys[noDictionaryColumnIndex++],
+ queryDimensions[i].getDimension().getDataType());
}
} else if (directDictionaryEncodingArray[i]) {
if (directDictionaryGenerators[i] != null) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
index 71045ff..7c07923 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RestructureBasedDictionaryResultCollector.java
@@ -48,7 +48,7 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe
List<Object[]> listBasedResult = new ArrayList<>(batchSize);
int rowCounter = 0;
int[] surrogateResult;
- String[] noDictionaryKeys;
+ byte[][] noDictionaryKeys;
byte[][] complexTypeKeyArray;
BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache =
scannedResult.getDeleteDeltaDataCache();
@@ -58,7 +58,7 @@ public class RestructureBasedDictionaryResultCollector extends DictionaryBasedRe
Object[] row = new Object[queryDimensions.length + queryMeasures.length];
if (isDimensionExists) {
surrogateResult = scannedResult.getDictionaryKeyIntegerArray();
- noDictionaryKeys = scannedResult.getNoDictionaryKeyStringArray();
+ noDictionaryKeys = scannedResult.getNoDictionaryKeyArray();
complexTypeKeyArray = scannedResult.getComplexTypeKeyArray();
dictionaryColumnIndex = 0;
noDictionaryColumnIndex = 0;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index 264c0ae..b5341a0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -116,7 +116,9 @@ public class FilterExpressionProcessor implements FilterProcessor {
+ searchEndKey.getNoDictionaryKeys());
long startTimeInMillis = System.currentTimeMillis();
DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(
- tableSegment.getSegmentProperties().getEachDimColumnValueSize());
+ tableSegment.getSegmentProperties().getEachDimColumnValueSize(),
+ tableSegment.getSegmentProperties().getNumberOfSortColumns(),
+ tableSegment.getSegmentProperties().getNumberOfNoDictSortColumns());
DataRefNode startBlock = blockFinder.findFirstDataBlock(btreeNode, searchStartKey);
DataRefNode endBlock = blockFinder.findLastDataBlock(btreeNode, searchEndKey);
FilterExecuter filterExecuter =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 7799b6a..d816c69 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -355,10 +355,23 @@ public final class FilterUtil {
* @return DimColumnFilterInfo
*/
public static DimColumnFilterInfo getNoDictionaryValKeyMemberForFilter(
- List<String> evaluateResultListFinal, boolean isIncludeFilter) {
+ List<String> evaluateResultListFinal, boolean isIncludeFilter, DataType dataType)
+ throws FilterUnsupportedException {
List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
- for (String result : evaluateResultListFinal) {
- filterValuesList.add(result.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
+ String result = null;
+ try {
+ int length = evaluateResultListFinal.size();
+ for (int i = 0; i < length; i++) {
+ result = evaluateResultListFinal.get(i);
+ if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(result)) {
+ filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY);
+ continue;
+ }
+ filterValuesList.add(
+ DataTypeUtil.getBytesBasedOnDataTypeForNoDictionaryColumn(result, dataType));
+ }
+ } catch (Throwable ex) {
+ throw new FilterUnsupportedException("Unsupported Filter condition: " + result, ex);
}
Comparator<byte[]> filterNoDictValueComaparator = new Comparator<byte[]>() {
@@ -879,8 +892,10 @@ public final class FilterUtil {
}
int keyOrdinalOfDimensionFromCurrentBlock = dimensionFromCurrentBlock.getKeyOrdinal();
for (DimColumnFilterInfo info : values) {
- if (startKey[keyOrdinalOfDimensionFromCurrentBlock] < info.getFilterList().get(0)) {
- startKey[keyOrdinalOfDimensionFromCurrentBlock] = info.getFilterList().get(0);
+ if (keyOrdinalOfDimensionFromCurrentBlock < startKey.length) {
+ if (startKey[keyOrdinalOfDimensionFromCurrentBlock] < info.getFilterList().get(0)) {
+ startKey[keyOrdinalOfDimensionFromCurrentBlock] = info.getFilterList().get(0);
+ }
}
}
long[] newStartKey = new long[startKey.length];
@@ -944,10 +959,12 @@ public final class FilterUtil {
}
int keyOrdinalOfDimensionFromCurrentBlock = dimensionFromCurrentBlock.getKeyOrdinal();
for (DimColumnFilterInfo info : values) {
- if (endKey[keyOrdinalOfDimensionFromCurrentBlock] > info.getFilterList()
- .get(info.getFilterList().size() - 1)) {
- endKey[keyOrdinalOfDimensionFromCurrentBlock] =
- info.getFilterList().get(info.getFilterList().size() - 1);
+ if (keyOrdinalOfDimensionFromCurrentBlock < endKey.length) {
+ if (endKey[keyOrdinalOfDimensionFromCurrentBlock] > info.getFilterList()
+ .get(info.getFilterList().size() - 1)) {
+ endKey[keyOrdinalOfDimensionFromCurrentBlock] =
+ info.getFilterList().get(info.getFilterList().size() - 1);
+ }
}
}
long[] newEndKey = new long[endKey.length];
@@ -1040,44 +1057,43 @@ public final class FilterUtil {
*/
public static IndexKey prepareDefaultEndIndexKey(SegmentProperties segmentProperties)
throws KeyGenException {
- long[] dictionarySurrogateKey =
- new long[segmentProperties.getDimensions().size() - segmentProperties
- .getNumberOfNoDictionaryDimension()];
+ long[] dictionarySurrogateKey = new long[segmentProperties.getNumberOfDictSortColumns()];
int index = 0;
int[] dimColumnsCardinality = segmentProperties.getDimColumnsCardinality();
- for (int i = 0; i < dimColumnsCardinality.length; i++) {
+ for (int i = 0; i < dictionarySurrogateKey.length; i++) {
dictionarySurrogateKey[index++] = dimColumnsCardinality[i];
}
IndexKey endIndexKey;
byte[] dictionaryendMdkey =
- segmentProperties.getDimensionKeyGenerator().generateKey(dictionarySurrogateKey);
+ segmentProperties.getSortColumnsGenerator().generateKey(dictionarySurrogateKey);
byte[] noDictionaryEndKeyBuffer = getNoDictionaryDefaultEndKey(segmentProperties);
endIndexKey = new IndexKey(dictionaryendMdkey, noDictionaryEndKeyBuffer);
return endIndexKey;
}
public static byte[] getNoDictionaryDefaultEndKey(SegmentProperties segmentProperties) {
+
+ int numberOfNoDictionaryDimension = segmentProperties.getNumberOfNoDictSortColumns();
// in case of non filter query when no dictionary columns are present we
// need to set the default end key, as for non filter query
// we need to get the last
// block of the btree so we are setting the max byte value in the end key
ByteBuffer noDictionaryEndKeyBuffer = ByteBuffer.allocate(
- (segmentProperties.getNumberOfNoDictionaryDimension()
- * CarbonCommonConstants.SHORT_SIZE_IN_BYTE) + segmentProperties
- .getNumberOfNoDictionaryDimension());
+ (numberOfNoDictionaryDimension * CarbonCommonConstants.SHORT_SIZE_IN_BYTE)
+ + numberOfNoDictionaryDimension);
// end key structure will be
//<Offset of first No Dictionary key in 2 Bytes><Offset of second No Dictionary key in 2 Bytes>
//<Offset of n No Dictionary key in 2 Bytes><first no dictionary column value>
// <second no dictionary column value> <N no dictionary column value>
//example if we have 2 no dictionary column
//<[0,4,0,5,127,127]>
- short startPoint = (short) (segmentProperties.getNumberOfNoDictionaryDimension()
- * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
- for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+ short startPoint =
+ (short) (numberOfNoDictionaryDimension * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+ for (int i = 0; i < numberOfNoDictionaryDimension; i++) {
noDictionaryEndKeyBuffer.putShort((startPoint));
startPoint++;
}
- for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+ for (int i = 0; i < numberOfNoDictionaryDimension; i++) {
noDictionaryEndKeyBuffer.put((byte) 127);
}
return noDictionaryEndKeyBuffer.array();
@@ -1094,11 +1110,9 @@ public final class FilterUtil {
public static IndexKey prepareDefaultStartIndexKey(SegmentProperties segmentProperties)
throws KeyGenException {
IndexKey startIndexKey;
- long[] dictionarySurrogateKey =
- new long[segmentProperties.getDimensions().size() - segmentProperties
- .getNumberOfNoDictionaryDimension()];
+ long[] dictionarySurrogateKey = new long[segmentProperties.getNumberOfDictSortColumns()];
byte[] dictionaryStartMdkey =
- segmentProperties.getDimensionKeyGenerator().generateKey(dictionarySurrogateKey);
+ segmentProperties.getSortColumnsGenerator().generateKey(dictionarySurrogateKey);
byte[] noDictionaryStartKeyArray = getNoDictionaryDefaultStartKey(segmentProperties);
startIndexKey = new IndexKey(dictionaryStartMdkey, noDictionaryStartKeyArray);
@@ -1106,26 +1120,27 @@ public final class FilterUtil {
}
public static byte[] getNoDictionaryDefaultStartKey(SegmentProperties segmentProperties) {
+
+ int numberOfNoDictionaryDimension = segmentProperties.getNumberOfNoDictSortColumns();
// in case of non filter query when no dictionary columns are present we
// need to set the default start key, as for non filter query we need to get the first
// block of the btree so we are setting the least byte value in the start key
ByteBuffer noDictionaryStartKeyBuffer = ByteBuffer.allocate(
- (segmentProperties.getNumberOfNoDictionaryDimension()
- * CarbonCommonConstants.SHORT_SIZE_IN_BYTE) + segmentProperties
- .getNumberOfNoDictionaryDimension());
+ (numberOfNoDictionaryDimension * CarbonCommonConstants.SHORT_SIZE_IN_BYTE)
+ + numberOfNoDictionaryDimension);
// end key structure will be
//<Offset of first No Dictionary key in 2 Bytes><Offset of second No Dictionary key in 2 Bytes>
//<Offset of n No Dictionary key in 2 Bytes><first no dictionary column value>
// <second no dictionary column value> <N no dictionary column value>
//example if we have 2 no dictionary column
//<[0,4,0,5,0,0]>
- short startPoint = (short) (segmentProperties.getNumberOfNoDictionaryDimension()
- * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
- for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+ short startPoint =
+ (short) (numberOfNoDictionaryDimension * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+ for (int i = 0; i < numberOfNoDictionaryDimension; i++) {
noDictionaryStartKeyBuffer.putShort((startPoint));
startPoint++;
}
- for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+ for (int i = 0; i < numberOfNoDictionaryDimension; i++) {
noDictionaryStartKeyBuffer.put((byte) 0);
}
return noDictionaryStartKeyBuffer.array();
@@ -1181,8 +1196,8 @@ public final class FilterUtil {
FilterResolverIntf filterResolver, List<IndexKey> listOfStartEndKeys) {
IndexKey searchStartKey = null;
IndexKey searchEndKey = null;
- long[] startKey = new long[segmentProperties.getDimensionKeyGenerator().getDimCount()];
- long[] endKey = new long[segmentProperties.getDimensionKeyGenerator().getDimCount()];
+ long[] startKey = new long[segmentProperties.getNumberOfDictSortColumns()];
+ long[] endKey = new long[segmentProperties.getNumberOfDictSortColumns()];
List<byte[]> listOfStartKeyByteArray =
new ArrayList<byte[]>(segmentProperties.getNumberOfNoDictionaryDimension());
List<byte[]> listOfEndKeyByteArray =
@@ -1234,12 +1249,23 @@ public final class FilterUtil {
pruneStartAndEndKeys(setOfStartKeyByteArray, listOfStartKeyByteArray);
pruneStartAndEndKeys(setOfEndKeyByteArray, listOfEndKeyByteArray);
+ if (segmentProperties.getNumberOfNoDictSortColumns() == 0) {
+ listOfStartKeyByteArray = new ArrayList<byte[]>();
+ listOfEndKeyByteArray = new ArrayList<byte[]>();
+ } else if (segmentProperties.getNumberOfNoDictSortColumns() < listOfStartKeyByteArray
+ .size()) {
+ while (segmentProperties.getNumberOfNoDictSortColumns() < listOfStartKeyByteArray.size()) {
+ listOfStartKeyByteArray.remove(listOfStartKeyByteArray.size() - 1);
+ listOfEndKeyByteArray.remove(listOfEndKeyByteArray.size() - 1);
+ }
+ }
+
searchStartKey = FilterUtil
- .createIndexKeyFromResolvedFilterVal(startKey, segmentProperties.getDimensionKeyGenerator(),
+ .createIndexKeyFromResolvedFilterVal(startKey, segmentProperties.getSortColumnsGenerator(),
FilterUtil.getKeyWithIndexesAndValues(listOfStartKeyByteArray));
searchEndKey = FilterUtil
- .createIndexKeyFromResolvedFilterVal(endKey, segmentProperties.getDimensionKeyGenerator(),
+ .createIndexKeyFromResolvedFilterVal(endKey, segmentProperties.getSortColumnsGenerator(),
FilterUtil.getKeyWithIndexesAndValues(listOfEndKeyByteArray));
listOfStartEndKeys.add(searchStartKey);
listOfStartEndKeys.add(searchEndKey);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index 306becf..bd83eec 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -157,7 +157,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
CarbonDimension currentBlockDimension =
segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension,
- this.segmentProperties.getDimensionKeyGenerator());
+ this.segmentProperties.getSortColumnsGenerator());
}
if (dimensionColumnDataChunk.isExplicitSorted()
&& dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index f2a49d9..a439228 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -157,7 +157,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
CarbonDimension currentBlockDimension =
segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension,
- this.segmentProperties.getDimensionKeyGenerator());
+ this.segmentProperties.getSortColumnsGenerator());
}
if (dimensionColumnDataChunk.isExplicitSorted()
&& dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
index 351d2c0..efedd93 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
@@ -67,7 +67,8 @@ public class NoDictionaryTypeVisitor implements ResolvedFilterInfoVisitorIntf {
throw new FilterUnsupportedException(e);
}
resolvedFilterObject = FilterUtil
- .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter());
+ .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(),
+ metadata.getColumnExpression().getDataType());
visitableObj.setFilterValues(resolvedFilterObject);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
index 8d0a8b4..d703ed0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/RangeNoDictionaryTypeVisitor.java
@@ -72,7 +72,8 @@ public class RangeNoDictionaryTypeVisitor extends NoDictionaryTypeVisitor
throw new FilterUnsupportedException(e);
}
resolvedFilterObject = FilterUtil
- .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter());
+ .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter(),
+ metadata.getColumnExpression().getDataType());
visitableObj.setFilterValues(resolvedFilterObject);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
index 4e5681c..cdab4e4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@ -104,7 +104,9 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
private void intialiseInfos() {
for (BlockExecutionInfo blockInfo : blockExecutionInfos) {
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(blockInfo.getEachColumnValueSize());
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(blockInfo.getEachColumnValueSize(),
+ blockInfo.getDataBlock().getSegmentProperties().getNumberOfSortColumns(),
+ blockInfo.getDataBlock().getSegmentProperties().getNumberOfNoDictSortColumns());
DataRefNode startDataBlock = finder
.findFirstDataBlock(blockInfo.getDataBlock().getDataRefNode(), blockInfo.getStartKey());
while (startDataBlock.nodeNumber() < blockInfo.getStartBlockletIndex()) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
index 4952e07..82a0b45 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
@@ -17,10 +17,15 @@
package org.apache.carbondata.core.scan.result.vector;
+import org.apache.spark.sql.types.DataType;
import org.apache.spark.sql.types.Decimal;
public interface CarbonColumnVector {
+ void putBoolean(int rowId, boolean value);
+
+ void putFloat(int rowId, float value);
+
void putShort(int rowId, short value);
void putShorts(int rowId, int count, short value);
@@ -59,4 +64,6 @@ public interface CarbonColumnVector {
void reset();
+ DataType getType();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index 48a081d..52f2f31 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -21,6 +21,7 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.List;
+import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.BlockInfo;
@@ -195,6 +196,12 @@ public abstract class AbstractDataFileFooterConverter {
wrapperColumnSchema.setColumnGroup(externalColumnSchema.getColumn_group_id());
wrapperColumnSchema.setScale(externalColumnSchema.getScale());
wrapperColumnSchema.setDefaultValue(externalColumnSchema.getDefault_value());
+ Map<String, String> properties = externalColumnSchema.getColumnProperties();
+ if (properties != null) {
+ if (properties.get(CarbonCommonConstants.SORT_COLUMNS) != null) {
+ wrapperColumnSchema.setSortColumn(true);
+ }
+ }
return wrapperColumnSchema;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
index 16c6760..f3f0fcd 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
@@ -17,7 +17,11 @@
package org.apache.carbondata.core.util;
+import java.io.UnsupportedEncodingException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.memory.CarbonUnsafe;
@@ -27,7 +31,13 @@ import org.apache.carbondata.core.memory.CarbonUnsafe;
*/
public final class ByteUtil {
- private static final int SIZEOF_LONG = 8;
+ public static final int SIZEOF_LONG = 8;
+
+ public static final int SIZEOF_INT = 4;
+
+ public static final int SIZEOF_SHORT = 2;
+
+ public static final String UTF8_CSN = StandardCharsets.UTF_8.name();
private ByteUtil() {
@@ -326,4 +336,328 @@ public final class ByteUtil {
}
+ /**
+ * Stirng => byte[]
+ *
+ * @param s
+ * @return
+ */
+ public static byte[] toBytes(String s) {
+ try {
+ return s.getBytes(UTF8_CSN);
+ } catch (UnsupportedEncodingException e) {
+ // should never happen!
+ throw new IllegalArgumentException("UTF8 decoding is not supported", e);
+ }
+ }
+
+ /**
+ * byte[] => String
+ *
+ * @param b
+ * @param off
+ * @param len
+ * @return
+ */
+ public static String toString(final byte[] b, int off, int len) {
+ if (b == null) {
+ return null;
+ }
+ if (len == 0) {
+ return "";
+ }
+ try {
+ return new String(b, off, len, UTF8_CSN);
+ } catch (UnsupportedEncodingException e) {
+ // should never happen!
+ throw new IllegalArgumentException("UTF8 encoding is not supported", e);
+ }
+ }
+
+ /**
+ * boolean => byte[]
+ *
+ * @param b
+ * @return
+ */
+ public static byte[] toBytes(final boolean b) {
+ return new byte[] { b ? (byte) -1 : (byte) 0 };
+ }
+
+ /**
+ * byte[] => boolean
+ *
+ * @param b
+ * @return
+ */
+ public static boolean toBoolean(final byte[] b) {
+ if (b.length != 1) {
+ throw new IllegalArgumentException("Array has wrong size: " + b.length);
+ }
+ return b[0] != (byte) 0;
+ }
+
+ /**
+ * short => byte[]
+ *
+ * @param val
+ * @return
+ */
+ public static byte[] toBytes(short val) {
+ byte[] b = new byte[SIZEOF_SHORT];
+ b[1] = (byte) val;
+ val >>= 8;
+ b[0] = (byte) val;
+ return b;
+ }
+
+ /**
+ * byte[] => short
+ *
+ * @param bytes
+ * @param offset
+ * @param length
+ * @return
+ */
+ public static short toShort(byte[] bytes, int offset, final int length) {
+ if (length != SIZEOF_SHORT || offset + length > bytes.length) {
+ throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
+ }
+ if (CarbonUnsafe.unsafe != null) {
+ if (CarbonUnsafe.ISLITTLEENDIAN) {
+ return Short.reverseBytes(
+ CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
+ } else {
+ return CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+ }
+ } else {
+ short n = 0;
+ n ^= bytes[offset] & 0xFF;
+ n <<= 8;
+ n ^= bytes[offset + 1] & 0xFF;
+ return n;
+ }
+ }
+
+ /**
+ * int => byte[]
+ *
+ * @param val
+ * @return
+ */
+ public static byte[] toBytes(int val) {
+ byte[] b = new byte[4];
+ for (int i = 3; i > 0; i--) {
+ b[i] = (byte) val;
+ val >>>= 8;
+ }
+ b[0] = (byte) val;
+ return b;
+ }
+
+ /**
+ * byte[] => int
+ *
+ * @param bytes
+ * @param offset
+ * @param length
+ * @return
+ */
+ public static int toInt(byte[] bytes, int offset, final int length) {
+ if (length != SIZEOF_INT || offset + length > bytes.length) {
+ throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
+ }
+ if (CarbonUnsafe.unsafe != null) {
+ if (CarbonUnsafe.ISLITTLEENDIAN) {
+ return Integer.reverseBytes(
+ CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
+ } else {
+ return CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+ }
+ } else {
+ int n = 0;
+ for (int i = offset; i < (offset + length); i++) {
+ n <<= 8;
+ n ^= bytes[i] & 0xFF;
+ }
+ return n;
+ }
+ }
+
+ /**
+ * float => byte[]
+ *
+ * @param f
+ * @return
+ */
+ public static byte[] toBytes(final float f) {
+ // Encode it as int
+ return toBytes(Float.floatToRawIntBits(f));
+ }
+
+ /**
+ * byte[] => float
+ *
+ * @param bytes
+ * @param offset
+ * @return
+ */
+ public static float toFloat(byte[] bytes, int offset) {
+ return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT));
+ }
+
+ /**
+ * long => byte[]
+ *
+ * @param val
+ * @return
+ */
+ public static byte[] toBytes(long val) {
+ byte[] b = new byte[8];
+ for (int i = 7; i > 0; i--) {
+ b[i] = (byte) val;
+ val >>>= 8;
+ }
+ b[0] = (byte) val;
+ return b;
+ }
+
+ /**
+ * byte[] => long
+ */
+ public static long toLong(byte[] bytes, int offset, final int length) {
+ if (length != SIZEOF_LONG || offset + length > bytes.length) {
+ throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
+ }
+ if (CarbonUnsafe.unsafe != null) {
+ if (CarbonUnsafe.ISLITTLEENDIAN) {
+ return Long.reverseBytes(
+ CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
+ } else {
+ return CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+ }
+ } else {
+ long l = 0;
+ for (int i = offset; i < offset + length; i++) {
+ l <<= 8;
+ l ^= bytes[i] & 0xFF;
+ }
+ return l;
+ }
+ }
+
+ /**
+ * doube => byte[]
+ *
+ * @param d
+ * @return
+ */
+ public static byte[] toBytes(final double d) {
+ // Encode it as a long
+ return toBytes(Double.doubleToRawLongBits(d));
+ }
+
+ /**
+ * byte[] => double
+ *
+ * @param bytes
+ * @param offset
+ * @return
+ */
+ public static double toDouble(final byte[] bytes, final int offset) {
+ return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG));
+ }
+
+ /**
+ * BigDecimal => byte[]
+ *
+ * @param val
+ * @return
+ */
+ public static byte[] toBytes(BigDecimal val) {
+ byte[] valueBytes = val.unscaledValue().toByteArray();
+ byte[] result = new byte[valueBytes.length + SIZEOF_INT];
+ int offset = putInt(result, 0, val.scale());
+ putBytes(result, offset, valueBytes, 0, valueBytes.length);
+ return result;
+ }
+
+ /**
+ * byte[] => BigDecimal
+ *
+ * @param bytes
+ * @param offset
+ * @param length
+ * @return
+ */
+ public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) {
+ if (bytes == null || length < SIZEOF_INT + 1 || (offset + length > bytes.length)) {
+ return null;
+ }
+
+ int scale = toInt(bytes, offset, bytes.length);
+ byte[] tcBytes = new byte[length - SIZEOF_INT];
+ System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT);
+ return new BigDecimal(new BigInteger(tcBytes), scale);
+ }
+
+ private static IllegalArgumentException explainWrongLengthOrOffset(final byte[] bytes,
+ final int offset, final int length, final int expectedLength) {
+ String reason;
+ if (length != expectedLength) {
+ reason = "Wrong length: " + length + ", expected " + expectedLength;
+ } else {
+ reason = "offset (" + offset + ") + length (" + length + ") exceed the"
+ + " capacity of the array: " + bytes.length;
+ }
+ return new IllegalArgumentException(reason);
+ }
+
+ /**
+ * Put an int value out to the specified byte array position.
+ *
+ * @param bytes the byte array
+ * @param offset position in the array
+ * @param val int to write out
+ * @return incremented offset
+ * @throws IllegalArgumentException if the byte array given doesn't have
+ * enough room at the offset specified.
+ */
+ public static int putInt(byte[] bytes, int offset, int val) {
+ if (bytes.length - offset < SIZEOF_INT) {
+ throw new IllegalArgumentException(
+ "Not enough room to put an int at" + " offset " + offset + " in a " + bytes.length
+ + " byte array");
+ }
+ if (CarbonUnsafe.unsafe != null) {
+ if (CarbonUnsafe.ISLITTLEENDIAN) {
+ val = Integer.reverseBytes(val);
+ }
+ CarbonUnsafe.unsafe.putInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET, val);
+ return offset + ByteUtil.SIZEOF_INT;
+ } else {
+ for (int i = offset + 3; i > offset; i--) {
+ bytes[i] = (byte) val;
+ val >>>= 8;
+ }
+ bytes[offset] = (byte) val;
+ return offset + SIZEOF_INT;
+ }
+ }
+
+ /**
+ * Put bytes at the specified byte array position.
+ *
+ * @param tgtBytes the byte array
+ * @param tgtOffset position in the array
+ * @param srcBytes array to write out
+ * @param srcOffset source offset
+ * @param srcLength source length
+ * @return incremented offset
+ */
+ public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes, int srcOffset,
+ int srcLength) {
+ System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength);
+ return tgtOffset + srcLength;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 8956549..95ee94b 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -641,6 +641,24 @@ public final class CarbonUtil {
}
/**
+ * remove the quote char for a string, e.g. "abc" => abc, 'abc' => abc
+ * @param parseStr
+ * @return
+ */
+ public static String unquoteChar(String parseStr) {
+ if (parseStr == null) {
+ return null;
+ }
+ if (parseStr.startsWith("'") && parseStr.endsWith("'")) {
+ return parseStr.substring(1, parseStr.length() - 1);
+ } else if (parseStr.startsWith("\"") && parseStr.endsWith("\"")) {
+ return parseStr.substring(1, parseStr.length() - 1);
+ } else {
+ return parseStr;
+ }
+ }
+
+ /**
* special char delimiter Converter
*
* @param delimiter
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index d8e0c82..2e16b0f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -323,6 +323,76 @@ public final class DataTypeUtil {
}
+ public static byte[] getBytesBasedOnDataTypeForNoDictionaryColumn(String dimensionValue,
+ DataType actualDataType) throws Throwable {
+ switch (actualDataType) {
+ case STRING:
+ return ByteUtil.toBytes(dimensionValue);
+ case BOOLEAN:
+ return ByteUtil.toBytes(Boolean.parseBoolean(dimensionValue));
+ case SHORT:
+ return ByteUtil.toBytes(Short.parseShort(dimensionValue));
+ case INT:
+ return ByteUtil.toBytes(Integer.parseInt(dimensionValue));
+ case FLOAT:
+ return ByteUtil.toBytes(Float.parseFloat(dimensionValue));
+ case LONG:
+ return ByteUtil.toBytes(Long.parseLong(dimensionValue));
+ case DOUBLE:
+ return ByteUtil.toBytes(Double.parseDouble(dimensionValue));
+ case DECIMAL:
+ return ByteUtil.toBytes(new BigDecimal(dimensionValue));
+ default:
+ return ByteUtil.toBytes(dimensionValue);
+ }
+ }
+
+
+ /**
+ * Below method will be used to convert the data passed to its actual data
+ * type
+ *
+ * @param dataInBytes data
+ * @param actualDataType actual data type
+ * @return actual data after conversion
+ */
+ public static Object getDataBasedOnDataTypeForNoDictionaryColumn(byte[] dataInBytes,
+ DataType actualDataType) {
+ if (null == dataInBytes || Arrays
+ .equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY, dataInBytes)) {
+ return null;
+ }
+ try {
+ switch (actualDataType) {
+ case STRING:
+ return UTF8String.fromBytes(dataInBytes);
+ case BOOLEAN:
+ return ByteUtil.toBoolean(dataInBytes);
+ case SHORT:
+ return ByteUtil.toShort(dataInBytes, 0, dataInBytes.length);
+ case INT:
+ return ByteUtil.toInt(dataInBytes, 0, dataInBytes.length);
+ case FLOAT:
+ return ByteUtil.toFloat(dataInBytes, 0);
+ case LONG:
+ return ByteUtil.toLong(dataInBytes, 0, dataInBytes.length);
+ case DOUBLE:
+ return ByteUtil.toDouble(dataInBytes, 0);
+ case DECIMAL:
+ return ByteUtil.toBigDecimal(dataInBytes, 0, dataInBytes.length);
+ default:
+ return ByteUtil.toString(dataInBytes, 0, dataInBytes.length);
+ }
+ } catch (Throwable ex) {
+ String data = new String(dataInBytes, CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
+ LOGGER.error("Cannot convert" + data + " to " + actualDataType.getName() + " type value" + ex
+ .getMessage());
+ LOGGER.error("Problem while converting data type" + data);
+ return null;
+ }
+ }
+
+
/**
* Below method will be used to convert the data passed to its actual data
* type
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
index 54bbe06..d6081a3 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
@@ -68,7 +68,7 @@ public class BTreeBlockFinderTest extends TestCase {
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 });
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }, 1, 1);
ByteBuffer buffer = ByteBuffer.allocate(4 + 2);
buffer.rewind();
buffer.putShort((short) 1);
@@ -88,7 +88,7 @@ public class BTreeBlockFinderTest extends TestCase {
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 });
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }, 1, 1);
ByteBuffer buffer = ByteBuffer.allocate(4 + 1);
buffer.rewind();
buffer.put((byte) 1);
@@ -109,7 +109,7 @@ public class BTreeBlockFinderTest extends TestCase {
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 });
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
int[] dimensionBitLength =
CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
KeyGenerator multiDimKeyVarLengthGenerator =
@@ -131,7 +131,7 @@ public class BTreeBlockFinderTest extends TestCase {
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 });
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
int[] dimensionBitLength =
CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
KeyGenerator multiDimKeyVarLengthGenerator =
@@ -159,7 +159,7 @@ public class BTreeBlockFinderTest extends TestCase {
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 });
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
int[] dimensionBitLength =
CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
KeyGenerator multiDimKeyVarLengthGenerator =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
index fee3d3d..078479f 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
@@ -37,6 +37,7 @@ import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.LiteralExpression;
import org.apache.carbondata.core.scan.expression.conditional.ListExpression;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.intf.RowImpl;
import org.apache.carbondata.core.util.BitSetGroup;
@@ -363,7 +364,7 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest {
assertFalse(result);
}
- @Test public void testGetNoDictionaryValKeyMemberForFilter() {
+ @Test public void testGetNoDictionaryValKeyMemberForFilter() throws FilterUnsupportedException {
boolean isIncludeFilter = true;
AbsoluteTableIdentifier absoluteTableIdentifier =
new AbsoluteTableIdentifier(this.carbonStorePath, carbonTableIdentifier);
@@ -371,7 +372,9 @@ public class FilterUtilTest extends AbstractDictionaryCacheTest {
List<String> evaluateResultListFinal = new ArrayList<>();
evaluateResultListFinal.add("test1");
evaluateResultListFinal.add("test2");
- assertTrue(FilterUtil.getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, isIncludeFilter) instanceof DimColumnFilterInfo);
+ assertTrue(FilterUtil
+ .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, isIncludeFilter,
+ DataType.STRING) instanceof DimColumnFilterInfo);
}
@Test public void testPrepareDefaultStartIndexKey() throws KeyGenException {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index cda34e4..05f2b67 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -598,7 +598,9 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
// Add all blocks of btree into result
DataRefNodeFinder blockFinder =
- new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize());
+ new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize(),
+ segmentProperties.getNumberOfSortColumns(),
+ segmentProperties.getNumberOfNoDictSortColumns());
DataRefNode startBlock =
blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), startIndexKey);
DataRefNode endBlock =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
index 7ba6133..d476f74 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
@@ -199,7 +199,9 @@ class InMemoryBTreeIndex implements Index {
// Add all blocks of btree into result
DataRefNodeFinder blockFinder =
- new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize());
+ new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize(),
+ segmentProperties.getNumberOfSortColumns(),
+ segmentProperties.getNumberOfNoDictSortColumns());
DataRefNode startBlock =
blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), startIndexKey);
DataRefNode endBlock =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java
index 79a1559..0239bce 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/DictionaryDecodeReadSupport.java
@@ -87,6 +87,9 @@ public class DictionaryDecodeReadSupport<T> implements CarbonReadSupport<T> {
* threshold is reached
*/
@Override public void close() {
+ if (dictionaries == null) {
+ return;
+ }
for (int i = 0; i < dictionaries.length; i++) {
CarbonUtil.clearDictionaryCache(dictionaries[i]);
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
index e39d214..b814468 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
@@ -53,7 +53,9 @@ public class BlockLevelTraverser {
FilterUtil.prepareDefaultStartIndexKey(abstractIndex.getSegmentProperties());
DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(
- abstractIndex.getSegmentProperties().getEachDimColumnValueSize());
+ abstractIndex.getSegmentProperties().getEachDimColumnValueSize(),
+ abstractIndex.getSegmentProperties().getNumberOfSortColumns(),
+ abstractIndex.getSegmentProperties().getNumberOfNoDictSortColumns());
DataRefNode currentBlock =
blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), searchStartKey);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7dafae7f/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
index c90b34c..1448957 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
@@ -203,6 +203,7 @@ public class StoreCreator {
date.setColumnUniqueId(UUID.randomUUID().toString());
date.setDimensionColumn(true);
date.setColumnGroup(2);
+ date.setSortColumn(true);
columnSchemas.add(date);
ColumnSchema country = new ColumnSchema();
@@ -213,6 +214,7 @@ public class StoreCreator {
country.setColumnUniqueId(UUID.randomUUID().toString());
country.setDimensionColumn(true);
country.setColumnGroup(3);
+ country.setSortColumn(true);
columnSchemas.add(country);
ColumnSchema name = new ColumnSchema();
@@ -223,6 +225,7 @@ public class StoreCreator {
name.setColumnUniqueId(UUID.randomUUID().toString());
name.setDimensionColumn(true);
name.setColumnGroup(4);
+ name.setSortColumn(true);
columnSchemas.add(name);
ColumnSchema phonetype = new ColumnSchema();
@@ -233,6 +236,7 @@ public class StoreCreator {
phonetype.setColumnUniqueId(UUID.randomUUID().toString());
phonetype.setDimensionColumn(true);
phonetype.setColumnGroup(5);
+ phonetype.setSortColumn(true);
columnSchemas.add(phonetype);
ColumnSchema serialname = new ColumnSchema();
@@ -243,6 +247,7 @@ public class StoreCreator {
serialname.setColumnUniqueId(UUID.randomUUID().toString());
serialname.setDimensionColumn(true);
serialname.setColumnGroup(6);
+ serialname.setSortColumn(true);
columnSchemas.add(serialname);
ColumnSchema salary = new ColumnSchema();