You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2014/04/04 21:15:36 UTC
git commit: AMBARI-5356 File hadoop-mapreduce.jobsummary.log is
missing on host with JobTracker (dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk 82e34855a -> e63a05a85
AMBARI-5356 File hadoop-mapreduce.jobsummary.log is missing on host with JobTracker (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e63a05a8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e63a05a8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e63a05a8
Branch: refs/heads/trunk
Commit: e63a05a85191aa33eb024089518614afccb9d023
Parents: 82e3485
Author: Dmitry Sen <ds...@hortonworks.com>
Authored: Fri Apr 4 21:55:28 2014 +0300
Committer: Dmitry Sen <ds...@hortonworks.com>
Committed: Fri Apr 4 22:12:31 2014 +0300
----------------------------------------------------------------------
.../HDP/1.3.2/hooks/before-START/scripts/params.py | 6 +-----
.../hooks/before-START/templates/hadoop-env.sh.j2 | 8 ++++----
.../MAPREDUCE/package/scripts/mapreduce.py | 17 ++++++++++++++++-
.../services/MAPREDUCE/package/scripts/params.py | 2 +-
.../1.3.2/MAPREDUCE/test_mapreduce_jobtracker.py | 10 ++++++++++
5 files changed, 32 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/e63a05a8/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/params.py
index 60c8298..2db9b4b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/params.py
@@ -134,16 +134,12 @@ dtnode_heapsize = config['configurations']['global']['dtnode_heapsize']
mapred_pid_dir_prefix = default("mapred_pid_dir_prefix","/var/run/hadoop-mapreduce")
mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
hadoop_libexec_dir = "/usr/lib/hadoop/libexec"
-mapred_log_dir_prefix = default("mapred_log_dir_prefix","/var/log/hadoop-mapreduce")
+mapred_log_dir_prefix = default("mapred_log_dir_prefix",hdfs_log_dir_prefix)
#taskcontroller.cfg
mapred_local_dir = "/tmp/hadoop-mapred/mapred/local"
-#log4j.properties
-
-yarn_log_dir_prefix = default("yarn_log_dir_prefix","/var/log/hadoop-yarn")
-
dfs_hosts = default('/configurations/hdfs-site/dfs.hosts', None)
#log4j.properties
http://git-wip-us.apache.org/repos/asf/ambari/blob/e63a05a8/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/templates/hadoop-env.sh.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/templates/hadoop-env.sh.j2 b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/templates/hadoop-env.sh.j2
index a2ae4f4..76ac3f3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/templates/hadoop-env.sh.j2
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/templates/hadoop-env.sh.j2
@@ -62,9 +62,12 @@ export HADOOP_NAMENODE_INIT_HEAPSIZE="-Xms{{namenode_heapsize}}"
# Extra Java runtime options. Empty by default.
export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}"
+# History server logs
+export HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER
+
# Command specific options appended to HADOOP_OPTS when specified
export HADOOP_NAMENODE_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}"
-HADOOP_JOBTRACKER_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}"
+export HADOOP_JOBTRACKER_OPTS="-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA -Dmapred.log.dir=$HADOOP_MAPRED_LOG_DIR ${HADOOP_JOBTRACKER_OPTS}"
HADOOP_TASKTRACKER_OPTS="-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}"
HADOOP_DATANODE_OPTS="-Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=ERROR,DRFAS ${HADOOP_DATANODE_OPTS}"
@@ -83,9 +86,6 @@ export HADOOP_SSH_OPTS="-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR"
# Where log files are stored. $HADOOP_HOME/logs by default.
export HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER
-# History server logs
-export HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER
-
# Where log files are stored in the secure data environment.
export HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_DN_USER
http://git-wip-us.apache.org/repos/asf/ambari/blob/e63a05a8/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/mapreduce.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/mapreduce.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/mapreduce.py
index 2918188..94fe24e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/mapreduce.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/mapreduce.py
@@ -22,6 +22,7 @@ Ambari Agent
from resource_management import *
import sys
+import os
def mapreduce(name=None):
@@ -55,12 +56,26 @@ def mapreduce(name=None):
)
params.HdfsDirectory(None, action="create")
- Directory([params.mapred_pid_dir,params.mapred_log_dir],
+ Directory(params.mapred_pid_dir,
owner=params.mapred_user,
group=params.user_group,
recursive=True
)
+ mapred_log_dir = os.path.join(params.mapred_log_dir_prefix, params.mapred_user)
+ Directory(mapred_log_dir,
+ recursive=True,
+ owner=params.mapred_user,
+ group=params.user_group
+ )
+
+ if name == 'jobtracker':
+ File(os.path.join(mapred_log_dir, 'hadoop-mapreduce.jobsummary.log'),
+ owner=params.mapred_user,
+ group=params.user_group,
+ mode=0664
+ )
+
Directory(params.mapred_local_dir.split(','),
owner=params.mapred_user,
mode=0755,
http://git-wip-us.apache.org/repos/asf/ambari/blob/e63a05a8/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/params.py
index 3073400..1a77f95 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/params.py
@@ -39,7 +39,7 @@ hadoop_libexec_dir = '/usr/lib/hadoop/libexec'
hadoop_bin = "/usr/lib/hadoop/bin"
user_group = config['configurations']['global']['user_group']
hdfs_log_dir_prefix = config['configurations']['global']['hdfs_log_dir_prefix']
-mapred_log_dir = format("{hdfs_log_dir_prefix}/{mapred_user}")
+mapred_log_dir_prefix = default("mapred_log_dir_prefix",hdfs_log_dir_prefix)
mapred_local_dir = config['configurations']['mapred-site']['mapred.local.dir']
update_exclude_file_only = config['commandParams']['update_exclude_file_only']
http://git-wip-us.apache.org/repos/asf/ambari/blob/e63a05a8/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_jobtracker.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_jobtracker.py b/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_jobtracker.py
index 415c409..faa27fd 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_jobtracker.py
+++ b/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_jobtracker.py
@@ -228,6 +228,11 @@ class TestJobtracker(RMFTestCase):
group = 'hadoop',
recursive = True,
)
+ self.assertResourceCalled('File', '/var/log/hadoop/mapred/hadoop-mapreduce.jobsummary.log',
+ owner = 'mapred',
+ group = 'hadoop',
+ mode = 0664
+ )
self.assertResourceCalled('Directory', '/hadoop/mapred',
owner = 'mapred',
recursive = True,
@@ -317,6 +322,11 @@ class TestJobtracker(RMFTestCase):
group = 'hadoop',
recursive = True,
)
+ self.assertResourceCalled('File', '/var/log/hadoop/mapred/hadoop-mapreduce.jobsummary.log',
+ owner = 'mapred',
+ group = 'hadoop',
+ mode = 0664
+ )
self.assertResourceCalled('Directory', '/hadoop/mapred',
owner = 'mapred',
recursive = True,