You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/04/05 13:46:35 UTC
[1/2] incubator-carbondata git commit: no_sort_columns
Repository: incubator-carbondata
Updated Branches:
refs/heads/12-dev 3f19978c0 -> dccd2ad0d
no_sort_columns
fix comments
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/dbd0cdd9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/dbd0cdd9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/dbd0cdd9
Branch: refs/heads/12-dev
Commit: dbd0cdd900e491d1e780ff6b84dbbabf781cf9c3
Parents: 3f19978
Author: QiangCai <qi...@qq.com>
Authored: Mon Apr 3 00:42:55 2017 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Wed Apr 5 21:44:56 2017 +0800
----------------------------------------------------------------------
.../core/datastore/block/SegmentProperties.java | 22 ++-
.../impl/btree/BTreeDataRefNodeFinder.java | 24 +--
.../core/metadata/schema/table/CarbonTable.java | 4 -
.../scan/filter/FilterExpressionProcessor.java | 4 +-
.../carbondata/core/scan/filter/FilterUtil.java | 8 +-
...velRangeLessThanEqualFilterExecuterImpl.java | 2 +-
.../RowLevelRangeLessThanFiterExecuterImpl.java | 2 +-
.../AbstractDetailQueryResultIterator.java | 4 +-
.../apache/carbondata/core/util/CarbonUtil.java | 18 +++
.../impl/btree/BTreeBlockFinderTest.java | 10 +-
.../carbondata/hadoop/CarbonInputFormat.java | 4 +-
.../internal/index/impl/InMemoryBTreeIndex.java | 4 +-
.../hadoop/util/BlockLevelTraverser.java | 4 +-
.../testsuite/sortcolumns/TestSortColumns.scala | 79 ++++++++++
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 11 +-
.../org/apache/spark/sql/TableCreator.scala | 11 +-
.../newflow/CarbonDataLoadConfiguration.java | 4 +
.../newflow/DataLoadProcessBuilder.java | 22 +--
.../newflow/steps/NoSortProcessorStepImpl.java | 153 +++++++++++++++++++
.../store/CarbonFactDataHandlerModel.java | 7 +
20 files changed, 350 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
index 3bef423..80785e0 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
@@ -55,6 +55,14 @@ public class SegmentProperties {
private KeyGenerator dimensionKeyGenerator;
/**
+ * key generator which was used to generate the mdkey for dimensions in SORT_COLUMNS
+ * if SORT_COLUMNS contains all dimensions, it is same with dimensionKeyGenerator
+ * otherwise, it is different with dimensionKeyGenerator, the number of its dimensions is less
+ * than dimensionKeyGenerator.
+ */
+ private KeyGenerator sortColumnsGenerator;
+
+ /**
* list of dimension present in the block
*/
private List<CarbonDimension> dimensions;
@@ -522,6 +530,14 @@ public class SegmentProperties {
int[] bitLength = CarbonUtil.getDimensionBitLength(dimColumnsCardinality, dimensionPartitions);
// create a key generator
this.dimensionKeyGenerator = new MultiDimKeyVarLengthGenerator(bitLength);
+ if (this.getNumberOfDictSortColumns() == bitLength.length) {
+ this.sortColumnsGenerator = this.dimensionKeyGenerator;
+ } else {
+ int numberOfDictSortColumns = this.getNumberOfDictSortColumns();
+ int [] sortColumnBitLength = new int[numberOfDictSortColumns];
+ System.arraycopy(bitLength, 0, sortColumnBitLength, 0, numberOfDictSortColumns);
+ this.sortColumnsGenerator = new MultiDimKeyVarLengthGenerator(sortColumnBitLength);
+ }
this.fixedLengthKeySplitter =
new MultiDimKeyVarLengthVariableSplitGenerator(bitLength, dimensionPartitions);
// get the size of each value in file block
@@ -647,6 +663,10 @@ public class SegmentProperties {
return dimensionKeyGenerator;
}
+ public KeyGenerator getSortColumnsGenerator() {
+ return sortColumnsGenerator;
+ }
+
/**
* @return the dimensions
*/
@@ -817,6 +837,6 @@ public class SegmentProperties {
}
public int getNumberOfDictSortColumns() {
- return this.numberOfSortColumns - this.numberOfNoDictionaryDimension;
+ return this.numberOfSortColumns - this.numberOfNoDictSortColumns;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
index 2e1709b..2f8aadf 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeDataRefNodeFinder.java
@@ -49,18 +49,20 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
private int[] eachColumnValueSize;
/**
- * this will be used during search for no dictionary column
+ * the number of no dictionary columns in SORT_COLUMNS
*/
- private int numberOfNoDictionaryColumns;
+ private int numberOfNoDictSortColumns;
- public BTreeDataRefNodeFinder(int[] eachColumnValueSize) {
- this.eachColumnValueSize = eachColumnValueSize;
+ /**
+ * the number of columns in SORT_COLUMNS
+ */
+ private int numberOfSortColumns;
- for (int i = 0; i < eachColumnValueSize.length; i++) {
- if (eachColumnValueSize[i] == -1) {
- numberOfNoDictionaryColumns++;
- }
- }
+ public BTreeDataRefNodeFinder(int[] eachColumnValueSize, int numberOfSortColumns,
+ int numberOfNoDictSortColumns) {
+ this.eachColumnValueSize = eachColumnValueSize;
+ this.numberOfNoDictSortColumns = numberOfNoDictSortColumns;
+ this.numberOfSortColumns = numberOfSortColumns;
}
/**
@@ -213,7 +215,7 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
int dictionaryKeyOffset = 0;
int nonDictionaryKeyOffset = 0;
int compareResult = 0;
- int processedNoDictionaryColumn = numberOfNoDictionaryColumns;
+ int processedNoDictionaryColumn = numberOfNoDictSortColumns;
ByteBuffer firstNoDictionaryKeyBuffer = ByteBuffer.wrap(first.getNoDictionaryKeys());
ByteBuffer secondNoDictionaryKeyBuffer = ByteBuffer.wrap(second.getNoDictionaryKeys());
int actualOffset = 0;
@@ -221,7 +223,7 @@ public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
int firstNoDcitionaryLength = 0;
int secondNodeDictionaryLength = 0;
- for (int i = 0; i < eachColumnValueSize.length; i++) {
+ for (int i = 0; i < numberOfSortColumns; i++) {
if (eachColumnValueSize[i] != NO_DCITIONARY_COLUMN_VALUE) {
compareResult = ByteUtil.UnsafeComparer.INSTANCE
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 1271b6a..fae9960 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -656,10 +656,6 @@ public class CarbonTable implements Serializable {
tableMeasuresMap.put(tableName, visibleMeasures);
}
- public boolean isSortByColumns() {
- return numberOfSortColumns > 0;
- }
-
public int getNumberOfSortColumns() {
return numberOfSortColumns;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index 264c0ae..b5341a0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -116,7 +116,9 @@ public class FilterExpressionProcessor implements FilterProcessor {
+ searchEndKey.getNoDictionaryKeys());
long startTimeInMillis = System.currentTimeMillis();
DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(
- tableSegment.getSegmentProperties().getEachDimColumnValueSize());
+ tableSegment.getSegmentProperties().getEachDimColumnValueSize(),
+ tableSegment.getSegmentProperties().getNumberOfSortColumns(),
+ tableSegment.getSegmentProperties().getNumberOfNoDictSortColumns());
DataRefNode startBlock = blockFinder.findFirstDataBlock(btreeNode, searchStartKey);
DataRefNode endBlock = blockFinder.findLastDataBlock(btreeNode, searchEndKey);
FilterExecuter filterExecuter =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index bb26ea5..b116d9a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -1065,7 +1065,7 @@ public final class FilterUtil {
}
IndexKey endIndexKey;
byte[] dictionaryendMdkey =
- segmentProperties.getDimensionKeyGenerator().generateKey(dictionarySurrogateKey);
+ segmentProperties.getSortColumnsGenerator().generateKey(dictionarySurrogateKey);
byte[] noDictionaryEndKeyBuffer = getNoDictionaryDefaultEndKey(segmentProperties);
endIndexKey = new IndexKey(dictionaryendMdkey, noDictionaryEndKeyBuffer);
return endIndexKey;
@@ -1112,7 +1112,7 @@ public final class FilterUtil {
IndexKey startIndexKey;
long[] dictionarySurrogateKey = new long[segmentProperties.getNumberOfDictSortColumns()];
byte[] dictionaryStartMdkey =
- segmentProperties.getDimensionKeyGenerator().generateKey(dictionarySurrogateKey);
+ segmentProperties.getSortColumnsGenerator().generateKey(dictionarySurrogateKey);
byte[] noDictionaryStartKeyArray = getNoDictionaryDefaultStartKey(segmentProperties);
startIndexKey = new IndexKey(dictionaryStartMdkey, noDictionaryStartKeyArray);
@@ -1261,11 +1261,11 @@ public final class FilterUtil {
}
searchStartKey = FilterUtil
- .createIndexKeyFromResolvedFilterVal(startKey, segmentProperties.getDimensionKeyGenerator(),
+ .createIndexKeyFromResolvedFilterVal(startKey, segmentProperties.getSortColumnsGenerator(),
FilterUtil.getKeyWithIndexesAndValues(listOfStartKeyByteArray));
searchEndKey = FilterUtil
- .createIndexKeyFromResolvedFilterVal(endKey, segmentProperties.getDimensionKeyGenerator(),
+ .createIndexKeyFromResolvedFilterVal(endKey, segmentProperties.getSortColumnsGenerator(),
FilterUtil.getKeyWithIndexesAndValues(listOfEndKeyByteArray));
listOfStartEndKeys.add(searchStartKey);
listOfStartEndKeys.add(searchEndKey);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index 306becf..bd83eec 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -157,7 +157,7 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
CarbonDimension currentBlockDimension =
segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension,
- this.segmentProperties.getDimensionKeyGenerator());
+ this.segmentProperties.getSortColumnsGenerator());
}
if (dimensionColumnDataChunk.isExplicitSorted()
&& dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index f2a49d9..a439228 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -157,7 +157,7 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
CarbonDimension currentBlockDimension =
segmentProperties.getDimensions().get(dimensionBlocksIndex[0]);
defaultValue = FilterUtil.getMaskKey(key, currentBlockDimension,
- this.segmentProperties.getDimensionKeyGenerator());
+ this.segmentProperties.getSortColumnsGenerator());
}
if (dimensionColumnDataChunk.isExplicitSorted()
&& dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
index 4e5681c..cdab4e4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@ -104,7 +104,9 @@ public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterato
private void intialiseInfos() {
for (BlockExecutionInfo blockInfo : blockExecutionInfos) {
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(blockInfo.getEachColumnValueSize());
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(blockInfo.getEachColumnValueSize(),
+ blockInfo.getDataBlock().getSegmentProperties().getNumberOfSortColumns(),
+ blockInfo.getDataBlock().getSegmentProperties().getNumberOfNoDictSortColumns());
DataRefNode startDataBlock = finder
.findFirstDataBlock(blockInfo.getDataBlock().getDataRefNode(), blockInfo.getStartKey());
while (startDataBlock.nodeNumber() < blockInfo.getStartBlockletIndex()) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index a442087..7c63f28 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -641,6 +641,24 @@ public final class CarbonUtil {
}
/**
+ * remove the quote char for a string, e.g. "abc" => abc, 'abc' => abc
+ * @param parseStr
+ * @return
+ */
+ public static String unquoteChar(String parseStr) {
+ if (parseStr == null) {
+ return null;
+ }
+ if (parseStr.startsWith("'") && parseStr.endsWith("'")) {
+ return parseStr.substring(1, parseStr.length() - 1);
+ } else if (parseStr.startsWith("\"") && parseStr.endsWith("\"")) {
+ return parseStr.substring(1, parseStr.length() - 1);
+ } else {
+ return parseStr;
+ }
+ }
+
+ /**
* special char delimiter Converter
*
* @param delimiter
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
index a4adf92..245ff48 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/impl/btree/BTreeBlockFinderTest.java
@@ -68,7 +68,7 @@ public class BTreeBlockFinderTest extends TestCase {
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 });
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }, 1, 1);
ByteBuffer buffer = ByteBuffer.allocate(4 + 2);
buffer.rewind();
buffer.putShort((short) 1);
@@ -88,7 +88,7 @@ public class BTreeBlockFinderTest extends TestCase {
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 });
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }, 1, 1);
ByteBuffer buffer = ByteBuffer.allocate(4 + 1);
buffer.rewind();
buffer.put((byte) 1);
@@ -109,7 +109,7 @@ public class BTreeBlockFinderTest extends TestCase {
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 });
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
int[] dimensionBitLength =
CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
KeyGenerator multiDimKeyVarLengthGenerator =
@@ -131,7 +131,7 @@ public class BTreeBlockFinderTest extends TestCase {
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 });
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
int[] dimensionBitLength =
CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
KeyGenerator multiDimKeyVarLengthGenerator =
@@ -160,7 +160,7 @@ public class BTreeBlockFinderTest extends TestCase {
builder.build(infos);
DataRefNode dataBlock = builder.get();
assertTrue(dataBlock != null);
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 });
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
int[] dimensionBitLength =
CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
KeyGenerator multiDimKeyVarLengthGenerator =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index b330f12..540a69c 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -578,7 +578,9 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
// Add all blocks of btree into result
DataRefNodeFinder blockFinder =
- new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize());
+ new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize(),
+ segmentProperties.getNumberOfSortColumns(),
+ segmentProperties.getNumberOfNoDictSortColumns());
DataRefNode startBlock =
blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), startIndexKey);
DataRefNode endBlock =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
index 7ba6133..d476f74 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
@@ -199,7 +199,9 @@ class InMemoryBTreeIndex implements Index {
// Add all blocks of btree into result
DataRefNodeFinder blockFinder =
- new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize());
+ new BTreeDataRefNodeFinder(segmentProperties.getEachDimColumnValueSize(),
+ segmentProperties.getNumberOfSortColumns(),
+ segmentProperties.getNumberOfNoDictSortColumns());
DataRefNode startBlock =
blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), startIndexKey);
DataRefNode endBlock =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
index e39d214..b814468 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
@@ -53,7 +53,9 @@ public class BlockLevelTraverser {
FilterUtil.prepareDefaultStartIndexKey(abstractIndex.getSegmentProperties());
DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(
- abstractIndex.getSegmentProperties().getEachDimColumnValueSize());
+ abstractIndex.getSegmentProperties().getEachDimColumnValueSize(),
+ abstractIndex.getSegmentProperties().getNumberOfSortColumns(),
+ abstractIndex.getSegmentProperties().getNumberOfNoDictSortColumns());
DataRefNode currentBlock =
blockFinder.findFirstDataBlock(abstractIndex.getDataRefNode(), searchStartKey);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
index 3b28453..88b0421 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
@@ -154,6 +154,79 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
checkAnswer(sql("select * from sorttable6 where empname = 'madhan'"), sql("select * from origintable1 where empname = 'madhan'"))
}
+ test("unsorted table creation, query data loading with heap and safe sort config") {
+ try {
+ setLoadingProperties("false", "false", "false")
+ sql("CREATE TABLE unsortedtable_heap_safe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select * from unsortedtable_heap_safe where empno = 11"), sql("select * from origintable1 where empno = 11"))
+ checkAnswer(sql("select * from unsortedtable_heap_safe order by empno"), sql("select * from origintable1 order by empno"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("unsorted table creation, query and data loading with heap and unsafe sort config") {
+ try {
+ setLoadingProperties("false", "true", "false")
+ sql("CREATE TABLE unsortedtable_heap_unsafe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_unsafe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select * from unsortedtable_heap_unsafe where empno = 11"), sql("select * from origintable1 where empno = 11"))
+ checkAnswer(sql("select * from unsortedtable_heap_unsafe order by empno"), sql("select * from origintable1 order by empno"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("unsorted table creation, query and loading with heap and inmemory sort config") {
+ try {
+ setLoadingProperties("false", "false", "true")
+ sql("CREATE TABLE unsortedtable_heap_inmemory (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_heap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select * from unsortedtable_heap_inmemory where empno = 11"), sql("select * from origintable1 where empno = 11"))
+ checkAnswer(sql("select * from unsortedtable_heap_inmemory order by empno"), sql("select * from origintable1 order by empno"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("unsorted table creation, query and data loading with offheap and safe sort config") {
+ try {
+ setLoadingProperties("true", "false", "false")
+ sql("CREATE TABLE unsortedtable_offheap_safe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_safe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select * from unsortedtable_offheap_safe where empno = 11"), sql("select * from origintable1 where empno = 11"))
+ checkAnswer(sql("select * from unsortedtable_offheap_safe order by empno"), sql("select * from origintable1 order by empno"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("unsorted table creation, query and data loading with offheap and unsafe sort config") {
+ try {
+ setLoadingProperties("true", "true", "false")
+ sql("CREATE TABLE unsortedtable_offheap_unsafe (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_unsafe OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select * from unsortedtable_offheap_unsafe where empno = 11"), sql("select * from origintable1 where empno = 11"))
+ checkAnswer(sql("select * from unsortedtable_offheap_unsafe order by empno"), sql("select * from origintable1 order by empno"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+ test("unsorted table creation, query and data loading with offheap and inmemory sort config") {
+ try {
+ setLoadingProperties("true", "false", "true")
+ sql("CREATE TABLE unsortedtable_offheap_inmemory (empno int, empname String, designation String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
+ sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_inmemory OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno = 11"), sql("select * from origintable1 where empno = 11"))
+ checkAnswer(sql("select * from unsortedtable_offheap_inmemory order by empno"), sql("select * from origintable1 order by empno"))
+ } finally {
+ defaultLoadingProperties
+ }
+ }
+
+
override def afterAll = {
dropTable
}
@@ -172,6 +245,12 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
sql("drop table if exists sorttable4_heap_inmemory")
sql("drop table if exists sorttable5")
sql("drop table if exists sorttable6")
+ sql("drop table if exists unsortedtable_offheap_safe")
+ sql("drop table if exists unsortedtable_offheap_unsafe")
+ sql("drop table if exists unsortedtable_offheap_inmemory")
+ sql("drop table if exists unsortedtable_heap_safe")
+ sql("drop table if exists unsortedtable_heap_unsafe")
+ sql("drop table if exists unsortedtable_heap_inmemory")
}
def setLoadingProperties(offheap: String, unsafe: String, useBatch: String): Unit = {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 4e95073..d5c1bcd 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -33,7 +33,7 @@ import org.apache.spark.sql.execution.command._
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.datatype.DataType
-import org.apache.carbondata.core.util.DataTypeUtil
+import org.apache.carbondata.core.util.{CarbonUtil, DataTypeUtil}
import org.apache.carbondata.processing.constants.LoggerAction
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.util.CommonUtil
@@ -490,8 +490,13 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
// All columns in sortkey should be there in create table cols
val sortKeyOption = tableProperties.get(CarbonCommonConstants.SORT_COLUMNS)
var sortKeyDimsTmp: Seq[String] = Seq[String]()
- if (sortKeyOption.isDefined) {
- var sortKey = sortKeyOption.get.split(',').map(_.trim)
+ val sortKeyString: String = if (sortKeyOption.isDefined) {
+ CarbonUtil.unquoteChar(sortKeyOption.get) trim
+ } else {
+ ""
+ }
+ if (!sortKeyString.isEmpty) {
+ val sortKey = sortKeyString.split(',').map(_.trim)
sortKey.foreach { column =>
if (!fields.exists(x => x.column.equalsIgnoreCase(column))) {
val errormsg = "sort_columns: " + column +
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
index 9321706..9e14f16 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.execution.command._
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.datatype.DataType
-import org.apache.carbondata.core.util.DataTypeUtil
+import org.apache.carbondata.core.util.{CarbonUtil, DataTypeUtil}
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.util.CommonUtil
@@ -62,8 +62,13 @@ object TableCreator {
// All columns in sortkey should be there in create table cols
val sortKeyOption = tableProperties.get(CarbonCommonConstants.SORT_COLUMNS)
var sortKeyDimsTmp: Seq[String] = Seq[String]()
- if (sortKeyOption.isDefined) {
- var sortKey = sortKeyOption.get.split(',').map(_.trim)
+ val sortKeyString: String = if (sortKeyOption.isDefined) {
+ CarbonUtil.unquoteChar(sortKeyOption.get) trim
+ } else {
+ ""
+ }
+ if (!sortKeyString.isEmpty) {
+ val sortKey = sortKeyString.split(',').map(_.trim)
sortKey.foreach { column =>
if (!fields.exists(x => x.column.equalsIgnoreCase(column))) {
val errormsg = "sort_columns: " + column +
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
index 407ac2f..4a6c421 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
@@ -133,6 +133,10 @@ public class CarbonDataLoadConfiguration {
return this.numberOfSortColumns;
}
+ public boolean isSortTable() {
+ return this.numberOfSortColumns > 0;
+ }
+
public void setNumberOfNoDictSortColumns(int numberOfNoDictSortColumns) {
this.numberOfNoDictSortColumns = numberOfNoDictSortColumns;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
index 1932888..e5f6a5f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
@@ -40,6 +40,7 @@ import org.apache.carbondata.processing.newflow.steps.DataConverterProcessorWith
import org.apache.carbondata.processing.newflow.steps.DataWriterBatchProcessorStepImpl;
import org.apache.carbondata.processing.newflow.steps.DataWriterProcessorStepImpl;
import org.apache.carbondata.processing.newflow.steps.InputProcessorStepImpl;
+import org.apache.carbondata.processing.newflow.steps.NoSortProcessorStepImpl;
import org.apache.carbondata.processing.newflow.steps.SortProcessorStepImpl;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
@@ -76,9 +77,10 @@ public final class DataLoadProcessBuilder {
// data types and configurations.
AbstractDataLoadProcessorStep converterProcessorStep =
new DataConverterProcessorStepImpl(configuration, inputProcessorStep);
- // 3. Sorts the data which are part of key (all dimensions except complex types)
- AbstractDataLoadProcessorStep sortProcessorStep =
- new SortProcessorStepImpl(configuration, converterProcessorStep);
+ // 3. Sorts the data by SortColumn or not
+ AbstractDataLoadProcessorStep sortProcessorStep = configuration.isSortTable() ?
+ new SortProcessorStepImpl(configuration, converterProcessorStep) :
+ new NoSortProcessorStepImpl(configuration, converterProcessorStep);
// 4. Writes the sorted data in carbondata format.
AbstractDataLoadProcessorStep writerProcessorStep =
new DataWriterProcessorStepImpl(configuration, sortProcessorStep);
@@ -94,9 +96,10 @@ public final class DataLoadProcessBuilder {
// data types and configurations.
AbstractDataLoadProcessorStep converterProcessorStep =
new DataConverterProcessorStepImpl(configuration, inputProcessorStep);
- // 3. Sorts the data which are part of key (all dimensions except complex types)
- AbstractDataLoadProcessorStep sortProcessorStep =
- new SortProcessorStepImpl(configuration, converterProcessorStep);
+ // 3. Sorts the data by SortColumn or not
+ AbstractDataLoadProcessorStep sortProcessorStep = configuration.isSortTable() ?
+ new SortProcessorStepImpl(configuration, converterProcessorStep) :
+ new NoSortProcessorStepImpl(configuration, converterProcessorStep);
// 4. Writes the sorted data in carbondata format.
AbstractDataLoadProcessorStep writerProcessorStep =
new DataWriterBatchProcessorStepImpl(configuration, sortProcessorStep);
@@ -112,9 +115,10 @@ public final class DataLoadProcessBuilder {
// data types and configurations.
AbstractDataLoadProcessorStep converterProcessorStep =
new DataConverterProcessorWithBucketingStepImpl(configuration, inputProcessorStep);
- // 3. Sorts the data which are part of key (all dimensions except complex types)
- AbstractDataLoadProcessorStep sortProcessorStep =
- new SortProcessorStepImpl(configuration, converterProcessorStep);
+ // 3. Sorts the data by SortColumn or not
+ AbstractDataLoadProcessorStep sortProcessorStep = configuration.isSortTable() ?
+ new SortProcessorStepImpl(configuration, converterProcessorStep) :
+ new NoSortProcessorStepImpl(configuration, converterProcessorStep);
// 4. Writes the sorted data in carbondata format.
AbstractDataLoadProcessorStep writerProcessorStep =
new DataWriterProcessorStepImpl(configuration, sortProcessorStep);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java
new file mode 100644
index 0000000..d44f954
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/NoSortProcessorStepImpl.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.processing.newflow.steps;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.processing.newflow.AbstractDataLoadProcessorStep;
+import org.apache.carbondata.processing.newflow.CarbonDataLoadConfiguration;
+import org.apache.carbondata.processing.newflow.DataField;
+import org.apache.carbondata.processing.newflow.row.CarbonRow;
+import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
+import org.apache.carbondata.processing.util.NonDictionaryUtil;
+
+/**
+ * if the table doesn't have sort_columns, just convert row format.
+ */
+public class NoSortProcessorStepImpl extends AbstractDataLoadProcessorStep {
+
+ private int dimensionCount;
+
+ private int dimensionWithComplexCount;
+
+ private int noDictCount;
+
+ private int measureCount;
+
+ private boolean[] isNoDictionaryDimensionColumn;
+
+ private char[] aggType;
+
+ public NoSortProcessorStepImpl(CarbonDataLoadConfiguration configuration,
+ AbstractDataLoadProcessorStep child) {
+ super(configuration, child);
+ this.dimensionWithComplexCount = configuration.getDimensionCount();
+ this.noDictCount =
+ configuration.getNoDictionaryCount() + configuration.getComplexDimensionCount();
+ this.dimensionCount = configuration.getDimensionCount() - this.noDictCount;
+ this.measureCount = configuration.getMeasureCount();
+ this.isNoDictionaryDimensionColumn =
+ CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields());
+ this.aggType = CarbonDataProcessorUtil
+ .getAggType(configuration.getMeasureCount(), configuration.getMeasureFields());
+ }
+
+ @Override public DataField[] getOutput() {
+ return child.getOutput();
+ }
+
+ @Override public void initialize() throws IOException {
+ child.initialize();
+ }
+
+ /**
+ * convert input CarbonRow to output CarbonRow
+ * e.g. There is a table as following,
+ * the number of dictionary dimensions is a,
+ * the number of no-dictionary dimensions is b,
+ * the number of complex dimensions is c,
+ * the number of measures is d.
+ * input CarbonRow format: the length of Object[] data is a+b+c+d, the number of all columns.
+ * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ * | Part | Object item | describe |
+ * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ * | Object[0 ~ a+b-1] | Integer, byte[], Integer, ... | dict + no dict dimensions|
+ * ----------------------------------------------------------------------------------------
+ * | Object[a+b ~ a+b+c-1] | byte[], byte[], ... | complex dimensions |
+ * ----------------------------------------------------------------------------------------
+ * | Object[a+b+c ~ a+b+c+d-1]| int, byte[], ... | measures |
+ * ----------------------------------------------------------------------------------------
+ * output CarbonRow format: the length of object[] data is 3.
+ * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ * | Part | Object item | describe |
+ * ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+ * | Object[0] | int[a] | dict dimension array |
+ * ----------------------------------------------------------------------------------------
+ * | Object[1] | byte[b+c][] | no dict + complex dim |
+ * ----------------------------------------------------------------------------------------
+ * | Object[2] | Object[d] | measures |
+ * ----------------------------------------------------------------------------------------
+ * @param row
+ * @return
+ */
+ @Override protected CarbonRow processRow(CarbonRow row) {
+ int dictIndex = 0;
+ int nonDicIndex = 0;
+ int[] dim = new int[this.dimensionCount];
+ byte[][] nonDicArray = new byte[this.noDictCount][];
+ Object[] measures = new Object[this.measureCount];
+ // read dimension values
+ int dimCount = 0;
+ for (; dimCount < isNoDictionaryDimensionColumn.length; dimCount++) {
+ if (isNoDictionaryDimensionColumn[dimCount]) {
+ nonDicArray[nonDicIndex++] = (byte[]) row.getObject(dimCount);
+ } else {
+ dim[dictIndex++] = (int) row.getObject(dimCount);
+ }
+ }
+
+ for (; dimCount < this.dimensionWithComplexCount; dimCount++) {
+ nonDicArray[nonDicIndex++] = (byte[]) row.getObject(dimCount);
+ }
+
+ // measure values
+ for (int mesCount = 0; mesCount < this.measureCount; mesCount++) {
+ Object value = row.getObject(mesCount + this.dimensionWithComplexCount);
+ if (null != value) {
+ if (aggType[mesCount] == CarbonCommonConstants.SUM_COUNT_VALUE_MEASURE) {
+ measures[mesCount] = value;
+ } else if (aggType[mesCount] == CarbonCommonConstants.BIG_INT_MEASURE) {
+ measures[mesCount] = value;
+ } else if (aggType[mesCount] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
+ BigDecimal val = (BigDecimal) value;
+ measures[mesCount] = DataTypeUtil.bigDecimalToByte(val);
+ }
+ } else {
+ measures[mesCount] = null;
+ }
+ }
+ // create new row of size 3 (1 for dims , 1 for high card , 1 for measures)
+ Object[] holder = new Object[3];
+ NonDictionaryUtil.prepareOutObj(holder, dim, nonDicArray, measures);
+ //return out row
+ return new CarbonRow(holder);
+ }
+
+ @Override
+ public void close() {
+ if (!closed) {
+ super.close();
+ }
+ }
+
+ @Override protected String getStepName() {
+ return "No Sort Processor";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dbd0cdd9/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index ffd23a2..e746514 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -199,6 +199,13 @@ public class CarbonFactDataHandlerModel {
CarbonDataProcessorUtil.getIsUseInvertedIndex(configuration.getDataFields());
int[] dimLensWithComplex = configuration.getCardinalityFinder().getCardinality();
+ if (!configuration.isSortTable()) {
+ for (int i = 0; i < dimLensWithComplex.length; i++) {
+ if (dimLensWithComplex[i] != 0) {
+ dimLensWithComplex[i] = Integer.MAX_VALUE;
+ }
+ }
+ }
List<Integer> dimsLenList = new ArrayList<Integer>();
for (int eachDimLen : dimLensWithComplex) {
if (eachDimLen != 0) dimsLenList.add(eachDimLen);
[2/2] incubator-carbondata git commit: [CARBONDATA-782]support
unsorted table (when SORT_COLUMNS is empty) This closes #722
Posted by ja...@apache.org.
[CARBONDATA-782]support unsorted table (when SORT_COLUMNS is empty) This closes #722
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/dccd2ad0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/dccd2ad0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/dccd2ad0
Branch: refs/heads/12-dev
Commit: dccd2ad0d0909561bbe5433b28be5012ae8f6425
Parents: 3f19978 dbd0cdd
Author: jackylk <ja...@huawei.com>
Authored: Wed Apr 5 21:46:16 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Wed Apr 5 21:46:16 2017 +0800
----------------------------------------------------------------------
.../core/datastore/block/SegmentProperties.java | 22 ++-
.../impl/btree/BTreeDataRefNodeFinder.java | 24 +--
.../core/metadata/schema/table/CarbonTable.java | 4 -
.../scan/filter/FilterExpressionProcessor.java | 4 +-
.../carbondata/core/scan/filter/FilterUtil.java | 8 +-
...velRangeLessThanEqualFilterExecuterImpl.java | 2 +-
.../RowLevelRangeLessThanFiterExecuterImpl.java | 2 +-
.../AbstractDetailQueryResultIterator.java | 4 +-
.../apache/carbondata/core/util/CarbonUtil.java | 18 +++
.../impl/btree/BTreeBlockFinderTest.java | 10 +-
.../carbondata/hadoop/CarbonInputFormat.java | 4 +-
.../internal/index/impl/InMemoryBTreeIndex.java | 4 +-
.../hadoop/util/BlockLevelTraverser.java | 4 +-
.../testsuite/sortcolumns/TestSortColumns.scala | 79 ++++++++++
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 11 +-
.../org/apache/spark/sql/TableCreator.scala | 11 +-
.../newflow/CarbonDataLoadConfiguration.java | 4 +
.../newflow/DataLoadProcessBuilder.java | 22 +--
.../newflow/steps/NoSortProcessorStepImpl.java | 153 +++++++++++++++++++
.../store/CarbonFactDataHandlerModel.java | 7 +
20 files changed, 350 insertions(+), 47 deletions(-)
----------------------------------------------------------------------