You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by su...@apache.org on 2017/04/24 16:57:02 UTC
hive git commit: HIVE-16483: HoS should populate split related
configurations to HiveConf (Chao Sun, reviewed by Xuefu Zhang)
Repository: hive
Updated Branches:
refs/heads/master 19a1a4b42 -> 26dd70e20
HIVE-16483: HoS should populate split related configurations to HiveConf (Chao Sun, reviewed by Xuefu Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/26dd70e2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/26dd70e2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/26dd70e2
Branch: refs/heads/master
Commit: 26dd70e2074d1878f0d7962fc3b9c782803eca7e
Parents: 19a1a4b
Author: Chao Sun <su...@apache.org>
Authored: Mon Apr 24 09:56:02 2017 -0700
Committer: Chao Sun <su...@apache.org>
Committed: Mon Apr 24 09:56:02 2017 -0700
----------------------------------------------------------------------
.../hive/ql/exec/spark/SparkPlanGenerator.java | 23 +++++++++++++++-----
1 file changed, 18 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/26dd70e2/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
index 12a76a7..5f85f9e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java
@@ -298,12 +298,13 @@ public class SparkPlanGenerator {
throw new IllegalArgumentException(msg, e);
}
if (work instanceof MapWork) {
+ MapWork mapWork = (MapWork) work;
cloned.setBoolean("mapred.task.is.map", true);
- List<Path> inputPaths = Utilities.getInputPaths(cloned, (MapWork) work,
+ List<Path> inputPaths = Utilities.getInputPaths(cloned, mapWork,
scratchDir, context, false);
Utilities.setInputPaths(cloned, inputPaths);
- Utilities.setMapWork(cloned, (MapWork) work, scratchDir, false);
- Utilities.createTmpDirs(cloned, (MapWork) work);
+ Utilities.setMapWork(cloned, mapWork, scratchDir, false);
+ Utilities.createTmpDirs(cloned, mapWork);
if (work instanceof MergeFileWork) {
MergeFileWork mergeFileWork = (MergeFileWork) work;
cloned.set(Utilities.MAPRED_MAPPER_CLASS, MergeFileMapper.class.getName());
@@ -313,9 +314,21 @@ public class SparkPlanGenerator {
} else {
cloned.set(Utilities.MAPRED_MAPPER_CLASS, ExecMapper.class.getName());
}
- if (((MapWork) work).getMinSplitSize() != null) {
+ if (mapWork.getMaxSplitSize() != null) {
+ HiveConf.setLongVar(cloned, HiveConf.ConfVars.MAPREDMAXSPLITSIZE,
+ mapWork.getMaxSplitSize());
+ }
+ if (mapWork.getMinSplitSize() != null) {
HiveConf.setLongVar(cloned, HiveConf.ConfVars.MAPREDMINSPLITSIZE,
- ((MapWork) work).getMinSplitSize());
+ mapWork.getMinSplitSize());
+ }
+ if (mapWork.getMinSplitSizePerNode() != null) {
+ HiveConf.setLongVar(cloned, HiveConf.ConfVars.MAPREDMINSPLITSIZEPERNODE,
+ mapWork.getMinSplitSizePerNode());
+ }
+ if (mapWork.getMinSplitSizePerRack() != null) {
+ HiveConf.setLongVar(cloned, HiveConf.ConfVars.MAPREDMINSPLITSIZEPERRACK,
+ mapWork.getMinSplitSizePerRack());
}
// remember the JobConf cloned for each MapWork, so we won't clone for it again
workToJobConf.put(work, cloned);