You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/10/28 02:33:17 UTC
[09/11] hive git commit: HIVE-14887. Reduce the memory used by MiniMr, MiniTez, MiniLlap tests. (Siddharth Seth, reviewed by Sergio Peña)
HIVE-14887. Reduce the memory used by MiniMr, MiniTez, MiniLlap tests. (Siddharth Seth, reviewed by Sergio Pe�a)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a977c368
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a977c368
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a977c368
Branch: refs/heads/hive-14535
Commit: a977c3680246f03d8abbcb1210a78f15c68dac2d
Parents: dcfb1ed
Author: Siddharth Seth <ss...@apache.org>
Authored: Thu Oct 27 13:22:33 2016 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Oct 27 13:22:33 2016 -0700
----------------------------------------------------------------------
data/conf/llap/tez-site.xml | 5 ++++
data/conf/perf-reg/tez-site.xml | 4 +++
data/conf/tez/tez-site.xml | 4 +++
.../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 7 +++---
.../org/apache/hadoop/hive/ql/QTestUtil.java | 8 +++---
pom.xml | 3 ++-
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 26 ++++++++++++++++++++
.../ptest2/src/main/resources/batch-exec.vm | 2 +-
.../ptest2/src/main/resources/source-prep.vm | 2 +-
9 files changed, 50 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a977c368/data/conf/llap/tez-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/llap/tez-site.xml b/data/conf/llap/tez-site.xml
index 6f1b9d2..62f97c5 100644
--- a/data/conf/llap/tez-site.xml
+++ b/data/conf/llap/tez-site.xml
@@ -5,6 +5,11 @@
<value>org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled</value>
</property>
+ <property>
+ <name>tez.am.resource.memory.mb</name>
+ <value>128</value>
+ </property>
+
<!-- Fail fast during tests -->
<property>
<name>tez.am.task.max.failed.attempts</name>
http://git-wip-us.apache.org/repos/asf/hive/blob/a977c368/data/conf/perf-reg/tez-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/perf-reg/tez-site.xml b/data/conf/perf-reg/tez-site.xml
index 940f390..c575544 100644
--- a/data/conf/perf-reg/tez-site.xml
+++ b/data/conf/perf-reg/tez-site.xml
@@ -1,5 +1,9 @@
<configuration>
<property>
+ <name>tez.am.resource.memory.mb</name>
+ <value>128</value>
+ </property>
+ <property>
<name>tez.am.dag.scheduler.class</name>
<value>org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled</value>
</property>
http://git-wip-us.apache.org/repos/asf/hive/blob/a977c368/data/conf/tez/tez-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/tez/tez-site.xml b/data/conf/tez/tez-site.xml
index 940f390..c575544 100644
--- a/data/conf/tez/tez-site.xml
+++ b/data/conf/tez/tez-site.xml
@@ -1,5 +1,9 @@
<configuration>
<property>
+ <name>tez.am.resource.memory.mb</name>
+ <value>128</value>
+ </property>
+ <property>
<name>tez.am.dag.scheduler.class</name>
<value>org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrderControlled</value>
</property>
http://git-wip-us.apache.org/repos/asf/hive/blob/a977c368/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
index f02f0e3..d06c869 100644
--- a/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
+++ b/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
@@ -223,7 +223,8 @@ public class MiniHS2 extends AbstractHiveService {
// Initialize the execution engine based on cluster type
switch (miniClusterType) {
case TEZ:
- mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString);
+ // TODO: This should be making use of confDir to load configs setup for Tez, etc.
+ mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 2, uriString);
break;
case LLAP:
if (usePortsFromConf) {
@@ -231,10 +232,10 @@ public class MiniHS2 extends AbstractHiveService {
}
llapCluster = LlapItUtils.startAndGetMiniLlapCluster(hiveConf, null, null);
- mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 4, uriString);
+ mr = ShimLoader.getHadoopShims().getMiniTezCluster(hiveConf, 2, uriString);
break;
case MR:
- mr = ShimLoader.getHadoopShims().getMiniMrCluster(hiveConf, 4, uriString, 1);
+ mr = ShimLoader.getHadoopShims().getMiniMrCluster(hiveConf, 2, uriString, 1);
break;
default:
throw new IllegalArgumentException("Unsupported cluster type " + mr);
http://git-wip-us.apache.org/repos/asf/hive/blob/a977c368/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index a873721..bfb9645 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -654,12 +654,10 @@ public class QTestUtil {
conf.addResource(new URL("file://" + new File(confDir).toURI().getPath()
+ "/tez-site.xml"));
}
- int numTrackers;
+ int numTrackers = 2;
if (EnumSet.of(MiniClusterType.llap, MiniClusterType.llap_local).contains(clusterType)) {
llapCluster = LlapItUtils.startAndGetMiniLlapCluster(conf, setup.zooKeeperCluster, confDir);
- numTrackers = 2;
} else {
- numTrackers = 4;
}
if (EnumSet.of(MiniClusterType.llap_local, MiniClusterType.tez_local).contains(clusterType)) {
mr = shims.getLocalMiniTezCluster(conf, clusterType == MiniClusterType.llap_local);
@@ -667,9 +665,9 @@ public class QTestUtil {
mr = shims.getMiniTezCluster(conf, numTrackers, uriString);
}
} else if (clusterType == MiniClusterType.miniSparkOnYarn) {
- mr = shims.getMiniSparkCluster(conf, 4, uriString, 1);
+ mr = shims.getMiniSparkCluster(conf, 2, uriString, 1);
} else if (clusterType == MiniClusterType.mr) {
- mr = shims.getMiniMrCluster(conf, 4, uriString, 1);
+ mr = shims.getMiniMrCluster(conf, 2, uriString, 1);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a977c368/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 6dad4cd..3d8fa1a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -91,6 +91,7 @@
<!-- Plugin and Plugin Dependency Versions -->
<ant.contrib.version>1.0b3</ant.contrib.version>
<datanucleus.maven.plugin.version>3.3.0-release</datanucleus.maven.plugin.version>
+ <maven.test.jvm.args>-Xmx2048m</maven.test.jvm.args>
<maven.antrun.plugin.version>1.7</maven.antrun.plugin.version>
<maven.assembly.plugin.version>2.3</maven.assembly.plugin.version>
<maven.checkstyle.plugin.version>2.12.1</maven.checkstyle.plugin.version>
@@ -1003,7 +1004,7 @@
<redirectTestOutputToFile>true</redirectTestOutputToFile>
<reuseForks>false</reuseForks>
<failIfNoTests>false</failIfNoTests>
- <argLine>-Xmx2048m -XX:MaxPermSize=512m</argLine>
+ <argLine>${maven.test.jvm.args}</argLine>
<additionalClasspathElements>
<additionalClasspathElement>${test.conf.dir}</additionalClasspathElement>
<additionalClasspathElement>${basedir}/${hive.path.to.root}/conf</additionalClasspathElement>
http://git-wip-us.apache.org/repos/asf/hive/blob/a977c368/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index ff760c8..e56463b 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -293,6 +293,12 @@ public class Hadoop23Shims extends HadoopShimsSecure {
JobConf jConf = new JobConf(conf);
jConf.set("yarn.scheduler.capacity.root.queues", "default");
jConf.set("yarn.scheduler.capacity.root.default.capacity", "100");
+ jConf.setInt(MRJobConfig.MAP_MEMORY_MB, 128);
+ jConf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 128);
+ jConf.setInt(MRJobConfig.MR_AM_VMEM_MB, 128);
+ jConf.setInt(YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB, 512);
+ jConf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
+ jConf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 512);
mr = new MiniMRCluster(numberOfTaskTrackers, nameNode, numDir, null, null, jConf);
}
@@ -320,6 +326,9 @@ public class Hadoop23Shims extends HadoopShimsSecure {
for (Map.Entry<String, String> pair: jConf) {
conf.set(pair.getKey(), pair.getValue());
}
+ conf.setInt(MRJobConfig.MAP_MEMORY_MB, 128);
+ conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, 128);
+ conf.setInt(MRJobConfig.MR_AM_VMEM_MB, 128);
}
}
@@ -385,6 +394,16 @@ public class Hadoop23Shims extends HadoopShimsSecure {
public MiniTezShim(Configuration conf, int numberOfTaskTrackers, String nameNode) throws IOException {
mr = new MiniTezCluster("hive", numberOfTaskTrackers);
+ conf.setInt(YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB, 512);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 512);
+ // Overrides values from the hive/tez-site.
+ conf.setInt("hive.tez.container.size", 128);
+ conf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 128);
+ conf.setInt(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB, 128);
+ conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 24);
+ conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 10);
+ conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f);
conf.set("fs.defaultFS", nameNode);
conf.set("tez.am.log.level", "DEBUG");
conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/apps_staging_dir");
@@ -416,6 +435,13 @@ public class Hadoop23Shims extends HadoopShimsSecure {
for (Map.Entry<String, String> pair: config) {
conf.set(pair.getKey(), pair.getValue());
}
+ // Overrides values from the hive/tez-site.
+ conf.setInt("hive.tez.container.size", 128);
+ conf.setInt(TezConfiguration.TEZ_AM_RESOURCE_MEMORY_MB, 128);
+ conf.setInt(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB, 128);
+ conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 24);
+ conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_BUFFER_SIZE_MB, 10);
+ conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.4f);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a977c368/testutils/ptest2/src/main/resources/batch-exec.vm
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/resources/batch-exec.vm b/testutils/ptest2/src/main/resources/batch-exec.vm
index 00487ce..2d16ca3 100644
--- a/testutils/ptest2/src/main/resources/batch-exec.vm
+++ b/testutils/ptest2/src/main/resources/batch-exec.vm
@@ -35,7 +35,7 @@ then
export PATH=$JAVA_HOME/bin/:$PATH
fi
export ANT_OPTS="-Xmx1g -XX:MaxPermSize=256m -Djava.io.tmpdir=$logDir/tmp ${antEnvOpts}"
-export M2_OPTS="-Xmx1g -XX:MaxPermSize=256m -Djava.io.tmpdir=$logDir/tmp ${mavenEnvOpts}"
+export MAVEN_OPTS="-Xmx256m -Djava.io.tmpdir=$logDir/tmp ${mavenEnvOpts}"
export HADOOP_ROOT_LOGGER=INFO,console
export HADOOP_OPTS="-Dhive.log.dir=$logDir -Dhive.query.id=hadoop -Djava.io.tmpdir=$logDir/tmp"
cd $localDir/$instanceName/${repositoryName}-source || exit 1
http://git-wip-us.apache.org/repos/asf/hive/blob/a977c368/testutils/ptest2/src/main/resources/source-prep.vm
----------------------------------------------------------------------
diff --git a/testutils/ptest2/src/main/resources/source-prep.vm b/testutils/ptest2/src/main/resources/source-prep.vm
index 0fc22be..7ad5024 100644
--- a/testutils/ptest2/src/main/resources/source-prep.vm
+++ b/testutils/ptest2/src/main/resources/source-prep.vm
@@ -23,7 +23,7 @@ then
export PATH=$JAVA_HOME/bin/:$PATH
fi
export ANT_OPTS="-Xmx1g -XX:MaxPermSize=256m ${antEnvOpts}"
-export M2_OPTS="-Xmx1g -XX:MaxPermSize=256m ${mavenEnvOpts}"
+export MAVEN_OPTS="-Xmx1g ${mavenEnvOpts}"
cd $workingDir/
(
if [[ "$clearLibraryCache" == "true" ]]