You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2016/07/13 18:19:16 UTC
ambari git commit: AMBARI-17692. Upgrade from HDP 2.5 to 2.5 (later
version) fails at RESTART HDFS/JOURNALNODE phase (ncole)
Repository: ambari
Updated Branches:
refs/heads/trunk 2e0a543f0 -> 1f2f81baf
AMBARI-17692. Upgrade from HDP 2.5 to 2.5 (later version) fails at RESTART HDFS/JOURNALNODE phase (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1f2f81ba
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1f2f81ba
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1f2f81ba
Branch: refs/heads/trunk
Commit: 1f2f81baf82aa9b75005875c99227788622a6f6c
Parents: 2e0a543
Author: Nate Cole <nc...@hortonworks.com>
Authored: Wed Jul 13 12:48:08 2016 -0400
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Wed Jul 13 14:18:28 2016 -0400
----------------------------------------------------------------------
.../package/scripts/journalnode_upgrade.py | 24 +++++++++++-----
.../stacks/2.0.6/HDFS/test_journalnode.py | 30 ++++++++++++++++++++
2 files changed, 47 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/1f2f81ba/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode_upgrade.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode_upgrade.py
index 8f478ef..7585107 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode_upgrade.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode_upgrade.py
@@ -89,23 +89,33 @@ def hdfs_roll_edits():
Execute(command, user=params.hdfs_user, tries=1)
-def ensure_jns_have_new_txn(nodes, last_txn_id):
+def ensure_jns_have_new_txn(nodelist, last_txn_id):
"""
- :param nodes: List of Journalnodes
+ :param nodelist: List of Journalnodes
:param last_txn_id: Integer of last transaction id
:return: Return true on success, false otherwise
"""
import params
+ jn_uri = default("/configurations/hdfs-site/dfs.namenode.shared.edits.dir", None)
+
+ if jn_uri is None:
+ raise Fail("No JournalNode URI found at hdfs-site/dfs.namenode.shared.edits.dir")
+
+ nodes = []
+ for node in nodelist:
+ if node in jn_uri:
+ nodes.append(node)
+
num_of_jns = len(nodes)
actual_txn_ids = {}
jns_updated = 0
if params.journalnode_address is None:
- raise Fail("Could not retrieve Journal node address")
+ raise Fail("Could not retrieve JournalNode address")
if params.journalnode_port is None:
- raise Fail("Could not retrieve Journalnode port")
+ raise Fail("Could not retrieve JournalNode port")
time_out_secs = 3 * 60
step_time_secs = 10
@@ -113,7 +123,7 @@ def ensure_jns_have_new_txn(nodes, last_txn_id):
protocol = "https" if params.https_only else "http"
- Logger.info("Checking if all Journalnodes are updated.")
+ Logger.info("Checking if all JournalNodes are updated.")
for i in range(iterations):
Logger.info('Try %d out of %d' % (i+1, iterations))
for node in nodes:
@@ -131,10 +141,10 @@ def ensure_jns_have_new_txn(nodes, last_txn_id):
if data:
actual_txn_ids[node] = int(data)
if actual_txn_ids[node] >= last_txn_id:
- Logger.info("Journalnode %s has a higher transaction id: %s" % (node, str(data)))
+ Logger.info("JournalNode %s has a higher transaction id: %s" % (node, str(data)))
jns_updated += 1
else:
- Logger.info("Journalnode %s is still on transaction id: %s" % (node, str(data)))
+ Logger.info("JournalNode %s is still on transaction id: %s" % (node, str(data)))
Logger.info("Sleeping for %d secs" % step_time_secs)
time.sleep(step_time_secs)
http://git-wip-us.apache.org/repos/asf/ambari/blob/1f2f81ba/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py
index 523b9aa..4b63de4 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py
@@ -298,10 +298,40 @@ class TestJournalnode(RMFTestCase):
urlopen_mock.assert_called
urlopen_mock.assert_called_with("http://c6407.ambari.apache.org:8480/jmx")
+
+ # Check the case where there is a journalnode installed but NOT part of the quorum.
+ url_stream_mock = MagicMock()
+ url_stream_mock.read.side_effect = (num_journalnodes * [namenode_jmx, journalnode_jmx])
+ urlopen_mock = MagicMock(return_value = url_stream_mock)
+ curl_krb_request_mock = MagicMock(side_effect=(num_journalnodes * [(namenode_jmx, "", 1), (journalnode_jmx, "", 1)]))
+
+ config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/journalnode-upgrade.json"
+ with open(config_file, "r") as f:
+ json_content = json.load(f)
+
+ # add one more host that is not part of the hdfs-site/dfs.namenode.shared.edits.dir property
+ json_content['clusterHostInfo']['journalnode_hosts'] = ['c6406.ambari.apache.org',
+ 'c6407.ambari.apache.org', 'c6408.ambari.apache.org', 'c6409.ambari.apache.org' ]
+
+ with patch.object(utils, "curl_krb_request", curl_krb_request_mock):
+ with patch.object(urllib2, "urlopen", urlopen_mock):
+ with patch.object(NamenodeHAState, "get_address", get_address_mock):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/journalnode.py",
+ classname = "JournalNode", command = "post_upgrade_restart",
+ config_dict = json_content,
+ checked_call_mocks = [(0, str(namenode_status_active)), (0, str(namenode_status_standby))],
+ stack_version = self.UPGRADE_STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES )
+
+ # ensure that the mock was called with the http-style version of the URL
+ urlopen_mock.assert_called
+ urlopen_mock.assert_called_with("http://c6407.ambari.apache.org:8480/jmx")
+
url_stream_mock.reset_mock()
curl_krb_request_mock.reset_mock()
get_address_mock.reset_mock()
+
# now try with HDFS on SSL
with patch.object(utils, "curl_krb_request", curl_krb_request_mock):
with patch.object(urllib2, "urlopen", urlopen_mock):