You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2016/11/29 09:15:48 UTC
[2/2] incubator-carbondata git commit: [CARBONDATA-368]Imporve
performance of dataframe loading This closes #278
[CARBONDATA-368]Imporve performance of dataframe loading This closes #278
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/567fa513
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/567fa513
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/567fa513
Branch: refs/heads/master
Commit: 567fa5131628b70c8c4829368fda6d48cb013af3
Parents: 879bfe7 f8a0c87
Author: jackylk <ja...@huawei.com>
Authored: Tue Nov 29 17:15:20 2016 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Tue Nov 29 17:15:20 2016 +0800
----------------------------------------------------------------------
.../spark/rdd/CarbonDataLoadRDD.scala | 96 ++---
.../spark/rdd/CarbonDataRDDFactory.scala | 88 +++--
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 11 +-
.../carbondata/spark/util/CarbonScalaUtil.scala | 51 +++
.../spark/util/GlobalDictionaryUtil.scala | 11 +-
.../apache/spark/rdd/DataLoadCoalescedRDD.scala | 68 ++++
.../spark/rdd/DataLoadPartitionCoalescer.scala | 363 +++++++++++++++++++
.../spark/sql/hive/DistributionUtil.scala | 19 +-
.../org/apache/spark/util/TaskContextUtil.scala | 29 ++
.../TestDataLoadPartitionCoalescer.scala | 170 +++++++++
.../spark/util/AllDictionaryTestCase.scala | 9 +-
.../util/ExternalColumnDictionaryTestCase.scala | 14 +-
...GlobalDictionaryUtilConcurrentTestCase.scala | 23 +-
.../util/GlobalDictionaryUtilTestCase.scala | 10 +-
.../processing/csvreaderstep/CsvInput.java | 73 +++-
.../csvreaderstep/JavaRddIterator.java | 32 ++
.../processing/csvreaderstep/RddInputUtils.java | 11 +-
17 files changed, 921 insertions(+), 157 deletions(-)
----------------------------------------------------------------------