You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/06/13 08:28:29 UTC
[3/3] carbondata git commit: [CARBONDATA-1163] Use spark
multi-threads model to load data This closes #910
[CARBONDATA-1163] Use spark multi-threads model to load data This closes #910
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/73c3525c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/73c3525c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/73c3525c
Branch: refs/heads/master
Commit: 73c3525ce3f8aac5d968d1c703e0ff8190168431
Parents: 47d0780 82741c1
Author: jackylk <ja...@huawei.com>
Authored: Tue Jun 13 16:28:13 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Tue Jun 13 16:28:13 2017 +0800
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 11 +
.../core/datastore/row/CarbonRow.java | 3 +-
docs/configuration-parameters.md | 1 +
.../src/test/resources/globalsort/sample1.csv | 5 +
.../src/test/resources/globalsort/sample2.csv | 5 +
.../src/test/resources/globalsort/sample3.csv | 5 +
.../dataload/TestGlobalSortDataLoad.scala | 296 +++++++++++++++++++
.../load/DataLoadProcessBuilderOnSpark.scala | 159 ++++++++++
.../load/DataLoadProcessorStepOnSpark.scala | 235 +++++++++++++++
.../spark/load/GlobalSortHelper.scala | 40 +++
.../carbondata/spark/load/ValidateUtil.scala | 79 +++++
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 3 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 11 +
.../execution/command/carbonTableSchema.scala | 32 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 11 +
.../execution/command/carbonTableSchema.scala | 34 +--
.../processing/model/CarbonLoadModel.java | 62 ++++
.../newflow/DataLoadProcessBuilder.java | 13 +-
.../newflow/sort/SortScopeOptions.java | 6 +-
.../newflow/sort/SortStepRowUtil.java | 73 +++++
.../UnsafeSingleThreadFinalSortFilesMerger.java | 48 +--
.../steps/DataConverterProcessorStepImpl.java | 35 ++-
.../steps/DataWriterProcessorStepImpl.java | 32 +-
.../schema/metadata/SortObserver.java | 4 +-
.../sortandgroupby/sortdata/SortParameters.java | 3 +-
.../store/CarbonFactDataHandlerColumnar.java | 11 +-
.../store/CarbonFactDataHandlerModel.java | 8 +
.../util/CarbonDataProcessorUtil.java | 25 ++
28 files changed, 1128 insertions(+), 122 deletions(-)
----------------------------------------------------------------------