You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2014/03/19 16:23:50 UTC
git commit: AMBARI-5112 hadoop-mapreduce.jobsummary.log is empty when
specified custom YARN Log Dir (dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk 17ead15a7 -> 8449cd1d3
AMBARI-5112 hadoop-mapreduce.jobsummary.log is empty when specified custom YARN Log Dir (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8449cd1d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8449cd1d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8449cd1d
Branch: refs/heads/trunk
Commit: 8449cd1d34929c823a90001e7e8d22590c2052e9
Parents: 17ead15
Author: Dmitry Sen <ds...@hortonworks.com>
Authored: Wed Mar 19 17:23:55 2014 +0200
Committer: Dmitry Sen <ds...@hortonworks.com>
Committed: Wed Mar 19 17:23:55 2014 +0200
----------------------------------------------------------------------
.../MAPREDUCE/configuration/mapreduce-log4j.xml | 4 +++-
.../services/YARN/configuration/yarn-log4j.xml | 4 +++-
.../YARN/package/scripts/resourcemanager.py | 24 +-------------------
.../2.0.6/services/YARN/package/scripts/yarn.py | 10 +++-----
.../stacks/2.0.6/YARN/test_historyserver.py | 16 -------------
.../stacks/2.0.6/YARN/test_mapreduce2_client.py | 16 -------------
.../stacks/2.0.6/YARN/test_nodemanager.py | 16 -------------
.../stacks/2.0.6/YARN/test_resourcemanager.py | 8 -------
.../stacks/2.0.6/YARN/test_yarn_client.py | 24 --------------------
.../stacks/2.1/YARN/test_apptimelineserver.py | 8 -------
10 files changed, 10 insertions(+), 120 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/8449cd1d/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/configuration/mapreduce-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/configuration/mapreduce-log4j.xml b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/configuration/mapreduce-log4j.xml
index 529e063..c76c088 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/configuration/mapreduce-log4j.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/configuration/mapreduce-log4j.xml
@@ -25,6 +25,8 @@
<property>
<name>content</name>
<value>
+#Relative to Mapred LOG DIR Prefix
+mapred.log.dir=.
#
# Job Summary Appender
#
@@ -36,7 +38,7 @@ hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}
hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
log4j.appender.JSA=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.JSA.File={{hdfs_log_dir_prefix}}/{{mapred_user}}/${hadoop.mapreduce.jobsummary.log.file}
+log4j.appender.JSA.File=${mapred.log.dir}/${hadoop.mapreduce.jobsummary.log.file}
log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
http://git-wip-us.apache.org/repos/asf/ambari/blob/8449cd1d/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-log4j.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-log4j.xml
index 99501de..e04973e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-log4j.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration/yarn-log4j.xml
@@ -25,6 +25,8 @@
<property>
<name>content</name>
<value>
+#Relative to Yarn Log Dir Prefix
+yarn.log.dir=.
#
# Job Summary Appender
#
@@ -51,7 +53,7 @@ yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger}
# - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename)
# - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender)
log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender
-log4j.appender.RMSUMMARY.File={{yarn_log_dir_prefix}}/{{yarn_user}}/${yarn.server.resourcemanager.appsummary.log.file}
+log4j.appender.RMSUMMARY.File=${yarn.log.dir}/${yarn.server.resourcemanager.appsummary.log.file}
log4j.appender.RMSUMMARY.MaxFileSize=256MB
log4j.appender.RMSUMMARY.MaxBackupIndex=20
log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout
http://git-wip-us.apache.org/repos/asf/ambari/blob/8449cd1d/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/resourcemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/resourcemanager.py
index 4af82c7..6e6b34f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/resourcemanager.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/resourcemanager.py
@@ -20,28 +20,6 @@ Ambari Agent
"""
-#!/usr/bin/env python2.6
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
import sys
from resource_management import *
@@ -58,7 +36,7 @@ class Resourcemanager(Script):
import params
env.set_params(params)
- yarn()
+ yarn(name='resourcemanager')
def start(self, env):
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/8449cd1d/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py
index aa7f139..9387022 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py
@@ -116,21 +116,17 @@ def yarn(name = None):
mode=0644
)
- File(params.yarn_job_summary_log,
+ if name == 'resourcemanager':
+ File(params.yarn_job_summary_log,
owner=params.yarn_user,
group=params.user_group
- )
+ )
File(params.rm_nodes_exclude_path,
owner=params.yarn_user,
group=params.user_group
)
- File(params.mapred_job_summary_log,
- owner=params.mapred_user,
- group=params.user_group
- )
-
File(format("{limits_conf_dir}/yarn.conf"),
mode=0644,
content=Template('yarn.conf.j2')
http://git-wip-us.apache.org/repos/asf/ambari/blob/8449cd1d/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
index 5c924f8..369e686 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py
@@ -238,18 +238,10 @@ class TestHistoryServer(RMFTestCase):
conf_dir = '/etc/hadoop/conf',
configurations = self.getConfig()['configurations']['capacity-scheduler'],
)
- self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log',
- owner = 'yarn',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
owner = 'yarn',
group = 'hadoop',
)
- self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log',
- owner = 'mapred',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf',
content = Template('yarn.conf.j2'),
mode = 0644,
@@ -386,18 +378,10 @@ class TestHistoryServer(RMFTestCase):
conf_dir = '/etc/hadoop/conf',
configurations = self.getConfig()['configurations']['capacity-scheduler'],
)
- self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log',
- owner = 'yarn',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
owner = 'yarn',
group = 'hadoop',
)
- self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log',
- owner = 'mapred',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf',
content = Template('yarn.conf.j2'),
mode = 0644,
http://git-wip-us.apache.org/repos/asf/ambari/blob/8449cd1d/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py
index e376850..e69dcfc 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py
@@ -97,18 +97,10 @@ class TestMapReduce2Client(RMFTestCase):
conf_dir = '/etc/hadoop/conf',
configurations = self.getConfig()['configurations']['capacity-scheduler'],
)
- self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log',
- owner = 'yarn',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
owner = 'yarn',
group = 'hadoop',
)
- self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log',
- owner = 'mapred',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf',
content = Template('yarn.conf.j2'),
mode = 0644,
@@ -192,18 +184,10 @@ class TestMapReduce2Client(RMFTestCase):
conf_dir = '/etc/hadoop/conf',
configurations = self.getConfig()['configurations']['capacity-scheduler'],
)
- self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log',
- owner = 'yarn',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
owner = 'yarn',
group = 'hadoop',
)
- self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log',
- owner = 'mapred',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf',
content = Template('yarn.conf.j2'),
mode = 0644,
http://git-wip-us.apache.org/repos/asf/ambari/blob/8449cd1d/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py
index 8e24f7a..8f694fc 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py
@@ -238,18 +238,10 @@ class TestNodeManager(RMFTestCase):
conf_dir = '/etc/hadoop/conf',
configurations = self.getConfig()['configurations']['capacity-scheduler'],
)
- self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log',
- owner = 'yarn',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
owner = 'yarn',
group = 'hadoop',
)
- self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log',
- owner = 'mapred',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf',
content = Template('yarn.conf.j2'),
mode = 0644,
@@ -386,18 +378,10 @@ class TestNodeManager(RMFTestCase):
conf_dir = '/etc/hadoop/conf',
configurations = self.getConfig()['configurations']['capacity-scheduler'],
)
- self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log',
- owner = 'yarn',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
owner = 'yarn',
group = 'hadoop',
)
- self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log',
- owner = 'mapred',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf',
content = Template('yarn.conf.j2'),
mode = 0644,
http://git-wip-us.apache.org/repos/asf/ambari/blob/8449cd1d/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
index ca870c0..f8cd4d3 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
@@ -186,10 +186,6 @@ class TestResourceManager(RMFTestCase):
owner = 'yarn',
group = 'hadoop',
)
- self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log',
- owner = 'mapred',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf',
content = Template('yarn.conf.j2'),
mode = 0644,
@@ -275,10 +271,6 @@ class TestResourceManager(RMFTestCase):
owner = 'yarn',
group = 'hadoop',
)
- self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log',
- owner = 'mapred',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf',
content = Template('yarn.conf.j2'),
mode = 0644,
http://git-wip-us.apache.org/repos/asf/ambari/blob/8449cd1d/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py
index f57f055..0b15967 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py
@@ -98,18 +98,10 @@ class TestYarnClient(RMFTestCase):
conf_dir = '/etc/hadoop/conf',
configurations = self.getConfig()['configurations']['capacity-scheduler'],
)
- self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log',
- owner = 'yarn',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
owner = 'yarn',
group = 'hadoop',
)
- self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log',
- owner = 'mapred',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf',
content = Template('yarn.conf.j2'),
mode = 0644,
@@ -193,18 +185,10 @@ class TestYarnClient(RMFTestCase):
conf_dir = '/etc/hadoop/conf',
configurations = self.getConfig()['configurations']['capacity-scheduler'],
)
- self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log',
- owner = 'yarn',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
owner = 'yarn',
group = 'hadoop',
)
- self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log',
- owner = 'mapred',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf',
content = Template('yarn.conf.j2'),
mode = 0644,
@@ -306,18 +290,10 @@ class TestYarnClient(RMFTestCase):
conf_dir = '/etc/hadoop/conf',
configurations = self.getConfig()['configurations']['capacity-scheduler'],
)
- self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log',
- owner = 'yarn',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
owner = 'yarn',
group = 'hadoop',
)
- self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log',
- owner = 'mapred',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf',
content = Template('yarn.conf.j2'),
mode = 0644,
http://git-wip-us.apache.org/repos/asf/ambari/blob/8449cd1d/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py b/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py
index 233c57d..2046f52 100644
--- a/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py
+++ b/ambari-server/src/test/python/stacks/2.1/YARN/test_apptimelineserver.py
@@ -132,18 +132,10 @@ class TestAppTimelineServer(RMFTestCase):
conf_dir = '/etc/hadoop/conf',
configurations = self.getConfig()['configurations']['capacity-scheduler'],
)
- self.assertResourceCalled('File', '/var/log/hadoop-yarn/yarn/hadoop-mapreduce.jobsummary.log',
- owner = 'yarn',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
owner = 'yarn',
group = 'hadoop',
)
- self.assertResourceCalled('File', '/var/log/hadoop-mapreduce/mapred/hadoop-mapreduce.jobsummary.log',
- owner = 'mapred',
- group = 'hadoop',
- )
self.assertResourceCalled('File', '/etc/security/limits.d/yarn.conf',
content = Template('yarn.conf.j2'),
mode = 0644,