You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by li...@apache.org on 2016/01/28 08:20:39 UTC
[08/11] hive git commit: HIVE-12828: Update Spark version to 1.6 (Rui
reviewed by Xuefu)
HIVE-12828: Update Spark version to 1.6 (Rui reviewed by Xuefu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/34b41e38
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/34b41e38
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/34b41e38
Branch: refs/heads/master
Commit: 34b41e389ab331f1d61b077fbea8cb855775698f
Parents: e47e114
Author: Rui Li <ru...@intel.com>
Authored: Fri Jan 15 21:24:39 2016 +0800
Committer: Rui Li <ru...@intel.com>
Committed: Thu Jan 28 14:51:55 2016 +0800
----------------------------------------------------------------------
data/conf/spark/yarn-client/hive-site.xml | 17 ++++++++++++++++-
pom.xml | 2 +-
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 2 ++
3 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/34b41e38/data/conf/spark/yarn-client/hive-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/spark/yarn-client/hive-site.xml b/data/conf/spark/yarn-client/hive-site.xml
index bc42f0d..f1d9ddc 100644
--- a/data/conf/spark/yarn-client/hive-site.xml
+++ b/data/conf/spark/yarn-client/hive-site.xml
@@ -231,7 +231,22 @@
<property>
<name>spark.executor.memory</name>
- <value>512m</value>
+ <value>1g</value>
+</property>
+
+<property>
+ <name>spark.yarn.executor.memoryOverhead</name>
+ <value>0</value>
+</property>
+
+<property>
+ <name>spark.driver.memory</name>
+ <value>1g</value>
+</property>
+
+<property>
+ <name>spark.yarn.driver.memoryOverhead</name>
+ <value>0</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hive/blob/34b41e38/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2066518..2d2a3de 100644
--- a/pom.xml
+++ b/pom.xml
@@ -166,7 +166,7 @@
<ST4.version>4.0.4</ST4.version>
<tez.version>0.8.2</tez.version>
<super-csv.version>2.2.0</super-csv.version>
- <spark.version>1.5.0</spark.version>
+ <spark.version>1.6.0</spark.version>
<scala.binary.version>2.10</scala.binary.version>
<scala.version>2.10.4</scala.version>
<tempus-fugit.version>1.1</tempus-fugit.version>
http://git-wip-us.apache.org/repos/asf/hive/blob/34b41e38/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index c03ae35..31060a2 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -526,6 +526,8 @@ public class Hadoop23Shims extends HadoopShimsSecure {
mr = new MiniSparkOnYARNCluster("sparkOnYarn");
conf.set("fs.defaultFS", nameNode);
conf.set("yarn.resourcemanager.scheduler.class", "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
+ // disable resource monitoring, although it should be off by default
+ conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING, false);
configureImpersonation(conf);
mr.init(conf);
mr.start();