You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2015/05/28 23:22:14 UTC
ambari git commit: AMBARI-11476. RU from Dal to Dal+ has failures in
First Oozie Service Check, then Oozie Server Start,
because yarn-site.xml missing from hadoop conf and conf folder is incorrect
(alejandro)
Repository: ambari
Updated Branches:
refs/heads/trunk db09f15f9 -> ce429e750
AMBARI-11476. RU from Dal to Dal+ has failures in First Oozie Service Check, then Oozie Server Start, because yarn-site.xml missing from hadoop conf and conf folder is incorrect (alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ce429e75
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ce429e75
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ce429e75
Branch: refs/heads/trunk
Commit: ce429e75014edfb08ea20dbff3826127ab3a509a
Parents: db09f15
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Wed May 27 19:49:11 2015 -0700
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Thu May 28 14:21:07 2015 -0700
----------------------------------------------------------------------
.../libraries/functions/copy_tarball.py | 24 ++++++++++----------
.../package/scripts/oozie_server_upgrade.py | 2 +-
.../4.0.0.2.0/package/scripts/service_check.py | 20 ++++++++++++++--
.../stacks/2.0.6/OOZIE/test_service_check.py | 21 +++++++++++++++--
.../stacks/2.0.6/YARN/test_resourcemanager.py | 4 ++--
5 files changed, 52 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/ce429e75/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
index acb1e9f..f8a2570 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
@@ -31,23 +31,23 @@ STACK_VERSION_PATTERN = "{{ stack_version }}"
TARBALL_MAP = {
"HDP": {
- "tez": ("/usr/hdp/%s/tez/lib/tez.tar.gz" % STACK_VERSION_PATTERN,
- "/hdp/apps/%s/tez/tez.tar.gz" % STACK_VERSION_PATTERN),
+ "tez": ("/usr/hdp/{0}/tez/lib/tez.tar.gz".format(STACK_VERSION_PATTERN),
+ "/hdp/apps/{0}/tez/tez.tar.gz".format(STACK_VERSION_PATTERN)),
- "hive": ("/usr/hdp/%s/hive/hive.tar.gz" % STACK_VERSION_PATTERN,
- "/hdp/apps/%s/hive/hive.tar.gz" % STACK_VERSION_PATTERN),
+ "hive": ("/usr/hdp/{0}/hive/hive.tar.gz".format(STACK_VERSION_PATTERN),
+ "/hdp/apps/{0}/hive/hive.tar.gz".format(STACK_VERSION_PATTERN)),
- "pig": ("/usr/hdp/%s/pig/pig.tar.gz" % STACK_VERSION_PATTERN,
- "/hdp/apps/%s/pig/pig.tar.gz" % STACK_VERSION_PATTERN),
+ "pig": ("/usr/hdp/{0}/pig/pig.tar.gz".format(STACK_VERSION_PATTERN),
+ "/hdp/apps/{0}/pig/pig.tar.gz".format(STACK_VERSION_PATTERN)),
- "hadoop_streaming": ("/usr/hdp/%s/hadoop/hadoop-streaming.jar" % STACK_VERSION_PATTERN,
- "/hdp/apps/%s/mapreduce/hadoop-streaming.jar" % STACK_VERSION_PATTERN),
+ "hadoop_streaming": ("/usr/hdp/{0}/hadoop-mapreduce/hadoop-streaming.jar".format(STACK_VERSION_PATTERN),
+ "/hdp/apps/{0}/mapreduce/hadoop-streaming.jar".format(STACK_VERSION_PATTERN)),
- "sqoop": ("/usr/hdp/%s/sqoop/sqoop.tar.gz" % STACK_VERSION_PATTERN,
- "/hdp/apps/%s/sqoop/sqoop.tar.gz" % STACK_VERSION_PATTERN),
+ "sqoop": ("/usr/hdp/{0}/sqoop/sqoop.tar.gz".format(STACK_VERSION_PATTERN),
+ "/hdp/apps/{0}/sqoop/sqoop.tar.gz".format(STACK_VERSION_PATTERN)),
- "mapreduce": ("/usr/hdp/%s/hadoop/mapreduce.tar.gz" % STACK_VERSION_PATTERN,
- "/hdp/apps/%s/mapreduce/mapreduce.tar.gz" % STACK_VERSION_PATTERN)
+ "mapreduce": ("/usr/hdp/{0}/hadoop/mapreduce.tar.gz".format(STACK_VERSION_PATTERN),
+ "/hdp/apps/{0}/mapreduce/mapreduce.tar.gz".format(STACK_VERSION_PATTERN))
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/ce429e75/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
index 47361c4..1afae6a 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
@@ -222,4 +222,4 @@ def _get_directory_mappings():
"""
import params
- return { params.conf_dir : BACKUP_CONF_ARCHIVE }
+ return { params.conf_dir + "/" : BACKUP_CONF_ARCHIVE } # the trailing "/" is important here so as to not include the "conf" folder itself
http://git-wip-us.apache.org/repos/asf/ambari/blob/ce429e75/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/service_check.py
index 11f4431..2438543 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/service_check.py
@@ -17,16 +17,21 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
+import os
+import glob
+
from resource_management.core.resources.system import Execute
from resource_management.core.resources import File
from resource_management.core.source import StaticFile
from resource_management.core.system import System
from resource_management.libraries.functions import format
from resource_management.libraries.script import Script
+from resource_management.libraries.resources.xml_config import XmlConfig
+from resource_management.core.exceptions import Fail
from ambari_commons.os_family_impl import OsFamilyImpl
from ambari_commons import OSConst
-import os
-import glob
+
+from resource_management.core.logger import Logger
class OozieServiceCheck(Script):
pass
@@ -42,6 +47,17 @@ class OozieServiceCheckDefault(OozieServiceCheck):
prepare_hdfs_file_name = 'prepareOozieHdfsDirectories.sh'
smoke_test_file_name = 'oozieSmoke2.sh'
+ if 'yarn-site' in params.config['configurations']:
+ XmlConfig("yarn-site.xml",
+ conf_dir=params.hadoop_conf_dir,
+ configurations=params.config['configurations']['yarn-site'],
+ owner=params.hdfs_user,
+ group=params.user_group,
+ mode=0644
+ )
+ else:
+ raise Fail("yarn-site.xml was not present in config parameters.")
+
OozieServiceCheckDefault.oozie_smoke_shell_file(smoke_test_file_name, prepare_hdfs_file_name)
@staticmethod
http://git-wip-us.apache.org/repos/asf/ambari/blob/ce429e75/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py
index 219c874..ffa47cb 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_service_check.py
@@ -35,11 +35,20 @@ class TestServiceCheck(RMFTestCase):
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
-
+
+ self.assertResourceCalled('XmlConfig', 'yarn-site.xml',
+ owner = 'hdfs',
+ group = 'hadoop',
+ conf_dir = '/etc/hadoop/conf',
+ mode = 0644,
+ configurations = self.getConfig()['configurations']['yarn-site'],
+ )
+
self.assert_service_check()
self.assertNoMoreResources()
def test_service_check_secured(self):
+ self.maxDiff = None
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/service_check.py",
classname="OozieServiceCheck",
command="service_check",
@@ -47,7 +56,15 @@ class TestServiceCheck(RMFTestCase):
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
-
+
+ self.assertResourceCalled('XmlConfig', 'yarn-site.xml',
+ owner = 'hdfs',
+ group = 'hadoop',
+ conf_dir = '/etc/hadoop/conf',
+ mode = 0644,
+ configurations = self.getConfig()['configurations']['yarn-site'],
+ )
+
self.assert_service_check()
self.assertNoMoreResources()
http://git-wip-us.apache.org/repos/asf/ambari/blob/ce429e75/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
index e7957fc..448fc01 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
@@ -185,7 +185,7 @@ class TestResourceManager(RMFTestCase):
content = Template('exclude_hosts_list.j2'),
group = 'hadoop',
)
- self.assertResourceCalled('Execute', '{rm_kinit_cmd} yarn --config {conf_dir} rmadmin -refreshNodes',
+ self.assertResourceCalled('Execute', ' yarn --config /etc/hadoop/conf rmadmin -refreshNodes',
environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"},
user = 'yarn',
)
@@ -204,7 +204,7 @@ class TestResourceManager(RMFTestCase):
content = Template('exclude_hosts_list.j2'),
group = 'hadoop',
)
- self.assertResourceCalled('Execute', '{rm_kinit_cmd} yarn --config {conf_dir} rmadmin -refreshNodes',
+ self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/rm.service.keytab rm/c6401.ambari.apache.org@EXAMPLE.COM; yarn --config /etc/hadoop/conf rmadmin -refreshNodes',
environment = {'PATH': "/bin:/usr/bin:/usr/lib/hadoop-yarn/bin"},
user = 'yarn',
)