You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/02/21 10:18:22 UTC
[1/2] incubator-carbondata git commit: Made partition distribution as
configurable
Repository: incubator-carbondata
Updated Branches:
refs/heads/master b400b82ce -> b4db25164
Made partition distribution as configurable
Fixed comment
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/e60a7b66
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/e60a7b66
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/e60a7b66
Branch: refs/heads/master
Commit: e60a7b669384c27467d7b5c86abc307c42a5a9f3
Parents: b400b82
Author: ravipesala <ra...@gmail.com>
Authored: Wed Feb 15 15:26:39 2017 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Tue Feb 21 18:17:21 2017 +0800
----------------------------------------------------------------------
.../org/apache/carbondata/spark/rdd/CarbonScanRDD.scala | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e60a7b66/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index 3415a81..09612cb 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -116,7 +116,8 @@ class CarbonScanRDD(
i += 1
result.add(partition)
}
- } else {
+ } else if (sparkContext.getConf.contains("spark.carbon.custom.distribution") &&
+ sparkContext.getConf.getBoolean("spark.carbon.custom.distribution", false)) {
// create a list of block based on split
val blockList = splits.asScala.map(_.asInstanceOf[Distributable])
@@ -141,6 +142,15 @@ class CarbonScanRDD(
}
}
noOfNodes = nodeBlockMapping.size
+ } else {
+ splits.asScala.zipWithIndex.foreach { splitWithIndex =>
+ val multiBlockSplit =
+ new CarbonMultiBlockSplit(identifier,
+ Seq(splitWithIndex._1.asInstanceOf[CarbonInputSplit]).asJava,
+ splitWithIndex._1.getLocations)
+ val partition = new CarbonSparkPartition(id, splitWithIndex._2, multiBlockSplit)
+ result.add(partition)
+ }
}
noOfBlocks = splits.size
[2/2] incubator-carbondata git commit: [CARBONDATA-705]Make partition
distribution as configurable This closes #600
Posted by ja...@apache.org.
[CARBONDATA-705]Make partition distribution as configurable This closes #600
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/b4db2516
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/b4db2516
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/b4db2516
Branch: refs/heads/master
Commit: b4db25164d2300e815fab649591a10106f319785
Parents: b400b82 e60a7b6
Author: jackylk <ja...@huawei.com>
Authored: Tue Feb 21 18:17:59 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Tue Feb 21 18:17:59 2017 +0800
----------------------------------------------------------------------
.../org/apache/carbondata/spark/rdd/CarbonScanRDD.scala | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------