You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2015/09/11 16:16:35 UTC
[2/2] ambari git commit: HADOOP-11764. [ HADOOP-11764] NodeManager
should use directory other than tmp for extracting and loading leveldbjni
(aonishuk)
HADOOP-11764. [ HADOOP-11764] NodeManager should use directory other than tmp for extracting and loading leveldbjni (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d9f600e2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d9f600e2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d9f600e2
Branch: refs/heads/branch-2.1
Commit: d9f600e23d42544a3d8f67e2da75dc3db98bc555
Parents: cfa8023
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Fri Sep 11 17:16:26 2015 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Fri Sep 11 17:16:26 2015 +0300
----------------------------------------------------------------------
.../common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml | 2 ++
.../YARN/2.1.0.2.0/package/scripts/params_linux.py | 1 +
.../stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py | 2 ++
.../2.0.6/hooks/before-ANY/scripts/shared_initialization.py | 7 +++++++
.../stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml | 2 ++
.../HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml | 2 ++
.../stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml | 2 ++
.../python/stacks/2.0.6/hooks/before-ANY/test_before_any.py | 5 +++++
8 files changed, 23 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
index a1dfa57..6eb1a1d 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-env.xml
@@ -85,6 +85,7 @@ export YARN_LOG_DIR={{yarn_log_dir_prefix}}/$USER
export YARN_PID_DIR={{yarn_pid_dir_prefix}}/$USER
export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}
export JAVA_HOME={{java64_home}}
+export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}"
# User for YARN daemons
export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
@@ -190,6 +191,7 @@ if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
fi
YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
+YARN_OPTS="$YARN_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}"
</value>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index 70ae4b8..9b242ea 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -74,6 +74,7 @@ hadoop_mapred2_jar_location = "/usr/lib/hadoop-mapreduce"
mapred_bin = "/usr/lib/hadoop-mapreduce/sbin"
yarn_bin = "/usr/lib/hadoop-yarn/sbin"
yarn_container_bin = "/usr/lib/hadoop-yarn/bin"
+hadoop_java_io_tmpdir = os.path.join(tmp_dir, "hadoop_java_io_tmpdir")
# hadoop parameters for 2.2+
if Script.is_hdp_stack_greater_or_equal("2.2"):
http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py
index 8abd72b..d871751 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py
@@ -19,6 +19,7 @@ limitations under the License.
import collections
import re
+import os
import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
@@ -94,6 +95,7 @@ hadoop_conf_empty_dir = "/etc/hadoop/conf.empty"
hadoop_secure_dn_user = hdfs_user
hadoop_dir = "/etc/hadoop"
versioned_hdp_root = '/usr/hdp/current'
+hadoop_java_io_tmpdir = os.path.join(tmp_dir, "hadoop_java_io_tmpdir")
# HDP 2.2+ params
if Script.is_hdp_stack_greater_or_equal("2.2"):
http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py
index bd44caf..c2cbe3c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/shared_initialization.py
@@ -155,6 +155,13 @@ def setup_hadoop_env():
group=params.user_group,
content=InlineTemplate(params.hadoop_env_sh_template))
+ # Create tmp dir for java.io.tmpdir
+ # Handle a situation when /tmp is set to noexec
+ Directory(params.hadoop_java_io_tmpdir,
+ owner=params.hdfs_user,
+ group=params.user_group,
+ mode=0777
+ )
def setup_java():
"""
http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml
index 4c99823..9ad95ac 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/YARN/configuration/yarn-env.xml
@@ -37,6 +37,7 @@ export YARN_LOG_DIR={{yarn_log_dir_prefix}}/$USER
export YARN_PID_DIR={{yarn_pid_dir_prefix}}/$USER
export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}
export JAVA_HOME={{java64_home}}
+export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}"
# User for YARN daemons
export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}
@@ -142,6 +143,7 @@ if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
fi
YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
+YARN_OPTS="$YARN_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}"
</value>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml
index b5280ce..0046eb9 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-env.xml
@@ -41,6 +41,8 @@ export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA
#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default
#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0.
export HADOOP_OPTS="-Dhdp.version=$HDP_VERSION $HADOOP_OPTS"
+export HADOOP_OPTS="-Djava.io.tmpdir={{hadoop_java_io_tmpdir}} $HADOOP_OPTS"
+export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}"
</value>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml
index 44f3dc2..699463b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/YARN/configuration/yarn-env.xml
@@ -35,6 +35,7 @@
export YARN_PID_DIR={{yarn_pid_dir_prefix}}/$USER
export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}
export JAVA_HOME={{java64_home}}
+ export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}"
# We need to add the EWMA appender for the yarn daemons only;
# however, YARN_ROOT_LOGGER is shared by the yarn client and the
@@ -148,6 +149,7 @@
YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
fi
YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE"
+ YARN_OPTS="$YARN_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}"
</value>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d9f600e2/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py
index 39fd3d5..242dc44 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/before-ANY/test_before_any.py
@@ -163,6 +163,11 @@ class TestHookBeforeInstall(RMFTestCase):
owner = 'hdfs',
group = 'hadoop'
)
+ self.assertResourceCalled('Directory', '/tmp/hadoop_java_io_tmpdir',
+ owner = 'hdfs',
+ group = 'hadoop',
+ mode = 0777
+ )
self.assertResourceCalled('Directory', '/tmp/AMBARI-artifacts/',
recursive = True,