You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ma...@apache.org on 2014/08/20 22:18:46 UTC
git commit: AMBARI-6956. Rebalance HDFS should fail if NameNode is
down. (mahadev)
Repository: ambari
Updated Branches:
refs/heads/trunk 4b91492d2 -> 22a6007e7
AMBARI-6956. Rebalance HDFS should fail if NameNode is down. (mahadev)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/22a6007e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/22a6007e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/22a6007e
Branch: refs/heads/trunk
Commit: 22a6007e7118a5b9b6cfc5bab22073e423941ece
Parents: 4b91492
Author: Mahadev Konar <ma...@apache.org>
Authored: Wed Aug 20 13:17:42 2014 -0700
Committer: Mahadev Konar <ma...@apache.org>
Committed: Wed Aug 20 13:17:42 2014 -0700
----------------------------------------------------------------------
.../test/python/ambari_agent/TestActionQueue.py | 9 +---
.../scripts/balancer-emulator/hdfs-command.py | 4 +-
.../services/HDFS/package/scripts/namenode.py | 18 +++++---
.../python/stacks/2.0.6/HDFS/test_namenode.py | 47 +++++++++++++++++---
.../2.0.6/configs/rebalancehdfs_default.json | 2 +-
5 files changed, 58 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/22a6007e/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py b/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py
index f582a68..ff165b7 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py
@@ -595,15 +595,10 @@ class TestActionQueue(TestCase):
report = actionQueue.result()
self.assertEqual(len(report['reports']),1)
-
-
- @patch.object(StackVersionsFileHandler, "read_stack_version")
@patch.object(CustomServiceOrchestrator, "resolve_script_path")
- @patch.object(FileCache, "__init__")
- def test_execute_python_executor(self, read_stack_version_mock, FileCache_mock, resolve_script_path_mock):
- FileCache_mock.return_value = None
-
+ @patch.object(StackVersionsFileHandler, "read_stack_version")
+ def test_execute_python_executor(self, read_stack_version_mock, resolve_script_path_mock):
dummy_controller = MagicMock()
cfg = AmbariConfig().getConfig()
http://git-wip-us.apache.org/repos/asf/ambari/blob/22a6007e/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/balancer-emulator/hdfs-command.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/balancer-emulator/hdfs-command.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/balancer-emulator/hdfs-command.py
index df173fe..0cce48c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/balancer-emulator/hdfs-command.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/balancer-emulator/hdfs-command.py
@@ -29,10 +29,10 @@ def write_function(path, handle, interval):
handle.flush()
time.sleep(interval)
-thread = Thread(target = write_function, args = ('balancer.log', sys.stdout, 1))
+thread = Thread(target = write_function, args = ('balancer.log', sys.stdout, 1.5))
thread.start()
-threaderr = Thread(target = write_function, args = ('balancer-err.log', sys.stderr, 0.3))
+threaderr = Thread(target = write_function, args = ('balancer-err.log', sys.stderr, 1.5 * 0.023))
threaderr.start()
thread.join()
http://git-wip-us.apache.org/repos/asf/ambari/blob/22a6007e/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/namenode.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/namenode.py
index f401122..8dae3eb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/namenode.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/namenode.py
@@ -101,11 +101,13 @@ class NameNode(Script):
_print("Executing command %s\n" % command)
parser = hdfs_rebalance.HdfsParser()
- proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
- shell=False,
- close_fds=True,
- cwd=basedir
- )
+ proc = subprocess.Popen(
+ command,
+ stdout=subprocess.PIPE,
+ shell=False,
+ close_fds=True,
+ cwd=basedir
+ )
for line in iter(proc.stdout.readline, ''):
_print('[balancer] %s %s' % (str(datetime.now()), line ))
pl = parser.parseLine(line)
@@ -118,7 +120,11 @@ class NameNode(Script):
_print('[balancer] %s %s' % (str(datetime.now()), 'Process is finished' ))
self.put_structured_out({'completePercent' : 1})
break
-
+
+ proc.stdout.close()
+ proc.wait()
+ if proc.returncode != None and proc.returncode != 0:
+ raise Fail('Hdfs rebalance process exited with error. See the log output')
def _print(line):
sys.stdout.write(line)
http://git-wip-us.apache.org/repos/asf/ambari/blob/22a6007e/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
index d268f9d..1878d87 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
@@ -21,6 +21,9 @@ from ambari_commons import OSCheck
from ambari_commons import OSCheck
from mock.mock import MagicMock, patch
from stacks.utils.RMFTestCase import *
+import resource_management
+
+import subprocess
@patch.object(OSCheck,"get_os_type", new = MagicMock(return_value='suse'))
@@ -451,10 +454,42 @@ class TestNamenode(RMFTestCase):
recursive = True,
mode = 0755,
)
-# def test_rebalance_hdfs(self): ##Does not work because of exectuteScript Framework does not works with strcuturedoutput
-# self.executeScript("2.0.6/services/HDFS/package/scripts/namenode.py",
-# classname = "NameNode",
-# command = "rebalancehdfs",
-# config_file="rebalancehdfs_default.json"
-# )
+
+ @patch("resource_management.libraries.script.Script.put_structured_out")
+ def test_rebalance_hdfs(self, pso):
+ Popen_Mock.return_value = 1
+ with patch("subprocess.Popen", new_callable=Popen_Mock):
+ ll = subprocess.Popen()
+ self.assertTrue(isinstance(ll.stdout.readline(),str))
+ try:
+ self.executeScript("2.0.6/services/HDFS/package/scripts/namenode.py",
+ classname = "NameNode",
+ command = "rebalancehdfs",
+ config_file="rebalancehdfs_default.json"
+ )
+ self.fail("Exception was not thrown")
+ except resource_management.core.exceptions.Fail:
+ pass ##expected
+
+ pso.reset_mock()
+ Popen_Mock.return_value = 0
+ ll = subprocess.Popen()
+ self.assertTrue(isinstance(ll.stdout.readline(),str))
+ self.executeScript("2.0.6/services/HDFS/package/scripts/namenode.py",
+ classname = "NameNode",
+ command = "rebalancehdfs",
+ config_file="rebalancehdfs_default.json"
+ )
+ self.assertEqual(pso.call_count, 2, "Output was not parsed properly")
+class Popen_Mock:
+ return_value = 1
+ lines = ['Time Stamp Iteration# Bytes Already Moved Bytes Left To Move Bytes Being Moved\n',
+ 'Jul 28, 2014 5:01:49 PM 0 0 B 5.74 GB 9.79 GB\n',
+ 'Jul 28, 2014 5:03:00 PM 1 0 B 5.58 GB 9.79 GB\n',
+ '']
+ def __call__(self, *args,**kwargs):
+ popen = MagicMock()
+ popen.returncode = Popen_Mock.return_value
+ popen.stdout.readline = MagicMock(side_effect = Popen_Mock.lines)
+ return popen
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/22a6007e/ambari-server/src/test/python/stacks/2.0.6/configs/rebalancehdfs_default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/rebalancehdfs_default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/rebalancehdfs_default.json
index 95b8335..42b28f9 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/rebalancehdfs_default.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/rebalancehdfs_default.json
@@ -30,7 +30,7 @@
"script": "scripts/namenode.py",
"hooks_folder": "HDP/2.0.6/hooks",
"command_timeout": "600",
- "namenode": "{\"threshold\":\"DEBUG\"}",
+ "namenode": "{\"threshold\":\"-1\"}",
"script_type": "PYTHON"
},
"taskId": 104,