You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/03/20 01:26:24 UTC
[3/3] incubator-carbondata git commit: [CARBONDATA-742] Added batch
sort to improve the loading performance This closes #620
[CARBONDATA-742] Added batch sort to improve the loading performance This closes #620
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8d0a672b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8d0a672b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8d0a672b
Branch: refs/heads/master
Commit: 8d0a672b9b2372fd2caffe84c07ddb6ce6b21ece
Parents: 0b44d0e b13ead9
Author: jackylk <ja...@huawei.com>
Authored: Mon Mar 20 09:26:10 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Mon Mar 20 09:26:10 2017 +0800
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 19 +
.../core/datastore/block/TableBlockInfo.java | 9 +-
.../carbondata/core/memory/CarbonUnsafe.java | 4 +
.../core/mutate/CarbonUpdateUtil.java | 3 +-
.../apache/carbondata/core/util/ByteUtil.java | 136 +++---
.../core/util/path/CarbonTablePath.java | 17 +-
.../datastore/block/TableBlockInfoTest.java | 2 +-
.../CarbonFormatDirectoryStructureTest.java | 4 +-
.../spark/merger/TupleConversionAdapter.java | 4 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 2 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 2 +-
.../graphgenerator/GraphGenerator.java | 8 +-
.../processing/mdkeygen/MDKeyGenStep.java | 18 +-
.../newflow/CarbonDataLoadConfiguration.java | 11 +
.../newflow/DataLoadProcessBuilder.java | 24 +
.../constants/DataLoadProcessorConstants.java | 8 -
.../converter/DictionaryCardinalityFinder.java | 26 +
.../newflow/converter/RowConverter.java | 2 +-
.../converter/impl/RowConverterImpl.java | 32 +-
.../processing/newflow/row/CarbonRowBatch.java | 4 +-
.../processing/newflow/row/CarbonSortBatch.java | 44 ++
.../UnsafeBatchParallelReadMergeSorterImpl.java | 270 +++++++++++
.../sort/unsafe/UnsafeCarbonRowPage.java | 3 +-
.../sort/unsafe/UnsafeMemoryManager.java | 4 +
.../newflow/sort/unsafe/UnsafeSortDataRows.java | 30 +-
.../unsafe/comparator/UnsafeRowComparator.java | 34 +-
.../merger/UnsafeIntermediateFileMerger.java | 3 +-
.../UnsafeSingleThreadFinalSortFilesMerger.java | 10 +-
.../steps/DataConverterProcessorStepImpl.java | 1 +
...ConverterProcessorWithBucketingStepImpl.java | 1 +
.../steps/DataWriterBatchProcessorStepImpl.java | 191 ++++++++
.../steps/DataWriterProcessorStepImpl.java | 28 +-
.../newflow/steps/SortProcessorStepImpl.java | 9 +-
.../sortdata/IntermediateFileMerger.java | 26 +-
.../sortandgroupby/sortdata/RowComparator.java | 10 +-
.../sortdata/RowComparatorForNormalDims.java | 6 +-
.../sortandgroupby/sortdata/SortDataRows.java | 14 +-
.../sortdata/SortTempFileChunkHolder.java | 14 +-
.../UnCompressedTempSortFileWriter.java | 8 +-
.../sortdatastep/SortKeyStep.java | 8 +-
.../store/CarbonFactDataHandlerColumnar.java | 13 +-
.../store/CarbonFactDataHandlerModel.java | 13 +-
.../store/writer/AbstractFactDataWriter.java | 5 +-
.../store/writer/CarbonDataWriterVo.java | 10 +
.../csvbased/CarbonCSVBasedSeqGenMeta.java | 6 +-
.../csvbased/CarbonCSVBasedSeqGenStep.java | 8 +-
.../processing/util/NonDictionaryUtil.java | 479 +++++++++++++++++++
.../processing/util/RemoveDictionaryUtil.java | 479 -------------------
48 files changed, 1359 insertions(+), 703 deletions(-)
----------------------------------------------------------------------