You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2014/10/24 11:52:19 UTC
[1/6] git commit: AMBARI-7828. TestActionScheduler fails occasionally
on builds.a.o stating expected: but was: (dlysnichenko)
Repository: ambari
Updated Branches:
refs/heads/branch-1.7.0 3a9d5c5ec -> 78f76f02c
refs/heads/trunk 2d4f7d4b8 -> 06d61c1ad
AMBARI-7828. TestActionScheduler fails occasionally on builds.a.o stating expected:<ABORTED> but was:<PENDING> (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f0887efb
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f0887efb
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f0887efb
Branch: refs/heads/branch-1.7.0
Commit: f0887efb78ac69aadb700107d1ea4380949e038d
Parents: 3a9d5c5
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Fri Oct 24 12:48:47 2014 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Fri Oct 24 12:48:47 2014 +0300
----------------------------------------------------------------------
.../actionmanager/TestActionScheduler.java | 95 ++++++++++----------
1 file changed, 46 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f0887efb/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java
index b9a38a4..7224924 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java
@@ -39,6 +39,7 @@ import java.util.concurrent.TimeUnit;
import com.google.common.reflect.TypeToken;
import com.google.inject.persist.UnitOfWork;
import junit.framework.Assert;
+import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.Role;
import org.apache.ambari.server.RoleCommand;
import org.apache.ambari.server.ServiceComponentHostNotFoundException;
@@ -86,6 +87,7 @@ public class TestActionScheduler {
+ " c6402.ambari.apache.org], slave_hosts=[c6401.ambari.apache.org,"
+ " c6402.ambari.apache.org]}";
private final String hostname = "ahost.ambari.apache.org";
+ private final int MAX_CYCLE_ITERATIONS = 100;
/**
@@ -138,19 +140,17 @@ public class TestActionScheduler {
//Keep large number of attempts so that the task is not expired finally
//Small action timeout to test rescheduling
- ActionScheduler scheduler = new ActionScheduler(100, 100, db, aq, fsm,
+ ActionScheduler scheduler = new ActionScheduler(100, 5, db, aq, fsm,
10000, new HostsMap((String) null), null, unitOfWork, conf);
scheduler.setTaskTimeoutAdjustment(false);
- // Start the thread
- scheduler.start();
- List<AgentCommand> ac = waitForQueueSize(hostname, aq, 1);
+ List<AgentCommand> ac = waitForQueueSize(hostname, aq, 1, scheduler);
assertTrue(ac.get(0) instanceof ExecutionCommand);
assertEquals("1-977", ((ExecutionCommand) (ac.get(0))).getCommandId());
assertEquals(clusterHostInfo, ((ExecutionCommand) (ac.get(0))).getClusterHostInfo());
//The action status has not changed, it should be queued again.
- ac = waitForQueueSize(hostname, aq, 1);
+ ac = waitForQueueSize(hostname, aq, 1, scheduler);
assertTrue(ac.get(0) instanceof ExecutionCommand);
assertEquals("1-977", ((ExecutionCommand) (ac.get(0))).getCommandId());
assertEquals(clusterHostInfo, ((ExecutionCommand) (ac.get(0))).getClusterHostInfo());
@@ -160,13 +160,13 @@ public class TestActionScheduler {
ac = aq.dequeueAll(hostname);
//Wait for sometime, it shouldn't be scheduled this time.
- ac = waitForQueueSize(hostname, aq, 0);
- scheduler.stop();
+ ac = waitForQueueSize(hostname, aq, 0, scheduler);
}
private List<AgentCommand> waitForQueueSize(String hostname, ActionQueue aq,
- int expectedQueueSize) throws InterruptedException {
- while (true) {
+ int expectedQueueSize, ActionScheduler scheduler) {
+ int cycleCount = 0;
+ while (cycleCount++ <= MAX_CYCLE_ITERATIONS) {
List<AgentCommand> ac = aq.dequeueAll(hostname);
if (ac != null) {
if (ac.size() == expectedQueueSize) {
@@ -176,16 +176,19 @@ public class TestActionScheduler {
+ ac.size());
}
}
- Thread.sleep(100);
+ try {
+ scheduler.doWork();
+ } catch (AmbariException e) {
+ Assert.fail("Ambari exception : " + e.getMessage() + e.getStackTrace());
+ }
}
+ return null;
}
/**
* Test whether scheduler times out an action
*/
@Test
- // BUG-15597
- @Ignore
public void testActionTimeout() throws Exception {
ActionQueue aq = new ActionQueue();
Properties properties = new Properties();
@@ -218,6 +221,10 @@ public class TestActionScheduler {
ActionDBAccessor db = mock(ActionDBAccessor.class);
when(db.getStagesInProgress()).thenReturn(stages);
+ Request request = mock(Request.class);
+ when(request.isExclusive()).thenReturn(false);
+ when(db.getRequest(anyLong())).thenReturn(request);
+
doAnswer(new Answer() {
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
@@ -231,15 +238,15 @@ public class TestActionScheduler {
//Small action timeout to test rescheduling
- ActionScheduler scheduler = new ActionScheduler(100, 50, db, aq, fsm, 3,
+ ActionScheduler scheduler = new ActionScheduler(100, 0, db, aq, fsm, 3,
new HostsMap((String) null), null, unitOfWork, conf);
scheduler.setTaskTimeoutAdjustment(false);
// Start the thread
- scheduler.start();
+ int cycleCount = 0;
while (!stages.get(0).getHostRoleStatus(hostname, "NAMENODE")
- .equals(HostRoleStatus.TIMEDOUT)) {
- Thread.sleep(100);
+ .equals(HostRoleStatus.TIMEDOUT) && cycleCount++ <= MAX_CYCLE_ITERATIONS) {
+ scheduler.doWork();
}
assertEquals(stages.get(0).getHostRoleStatus(hostname, "NAMENODE"),
HostRoleStatus.TIMEDOUT);
@@ -247,7 +254,6 @@ public class TestActionScheduler {
verify(db, times(1)).startRequest(eq(1L));
verify(db, times(1)).abortOperation(1L);
- scheduler.stop();
}
@Test
@@ -307,24 +313,23 @@ public class TestActionScheduler {
new HostsMap((String) null), sam, unitOfWork, conf).
addMockedMethod("cancelHostRoleCommands").
createMock();
+ scheduler.cancelHostRoleCommands((Collection<HostRoleCommand>)EasyMock.anyObject(),EasyMock.anyObject(String.class));
+ EasyMock.expectLastCall();
EasyMock.replay(scheduler);
scheduler.setTaskTimeoutAdjustment(false);
- // Start the thread
- scheduler.start();
+ int cycleCount=0;
while (!stages.get(0).getHostRoleStatus(hostname, "NAMENODE")
- .equals(HostRoleStatus.TIMEDOUT)) {
- Thread.sleep(100L);
+ .equals(HostRoleStatus.TIMEDOUT) && cycleCount++ <= MAX_CYCLE_ITERATIONS) {
+ scheduler.doWork();
}
- scheduler.stop();
+ Assert.assertEquals(HostRoleStatus.TIMEDOUT,stages.get(0).getHostRoleStatus(hostname, "NAMENODE"));
- //Check that cancelHostRoleCommands() was not called
EasyMock.verify(scheduler);
}
@Test
- @Ignore // This is temporary disabled as discussed here https://reviews.apache.org/r/26510/
public void testOpFailedEventRaisedForAbortedHostRole() throws Exception {
ActionQueue aq = new ActionQueue();
Properties properties = new Properties();
@@ -362,7 +367,7 @@ public class TestActionScheduler {
final List<Stage> stages = new ArrayList<Stage>();
Stage stage = new Stage(1, "/tmp", "cluster1", 1L, "stageWith2Tasks",
- CLUSTER_HOST_INFO, "", "");
+ CLUSTER_HOST_INFO, "{\"command_param\":\"param_value\"}", "{\"host_param\":\"param_value\"}");
addInstallTaskToStage(stage, hostname1, "cluster1", Role.DATANODE,
RoleCommand.INSTALL, Service.Type.HDFS, 1);
addInstallTaskToStage(stage, hostname2, "cluster1", Role.NAMENODE,
@@ -422,14 +427,12 @@ public class TestActionScheduler {
ActionScheduler scheduler = new ActionScheduler(100, 50000, db, aq, fsm, 3,
new HostsMap((String) null), null, unitOfWork, conf);
scheduler.setTaskTimeoutAdjustment(false);
- // Start the thread
- scheduler.start();
-
- while (!stages.get(0).getHostRoleStatus(hostname1, "DATANODE")
- .equals(HostRoleStatus.TIMEDOUT) && !stages.get(0).getHostRoleStatus
- (hostname2, "NAMENODE").equals(HostRoleStatus.ABORTED)) {
- Thread.sleep(100L);
+ int cycleCount=0;
+ while (!(stages.get(0).getHostRoleStatus(hostname1, "DATANODE")
+ .equals(HostRoleStatus.TIMEDOUT) && stages.get(0).getHostRoleStatus
+ (hostname2, "NAMENODE").equals(HostRoleStatus.ABORTED)) && cycleCount++ <= MAX_CYCLE_ITERATIONS) {
+ scheduler.doWork();
}
Assert.assertEquals(HostRoleStatus.TIMEDOUT,
@@ -466,7 +469,6 @@ public class TestActionScheduler {
Assert.assertNotNull("Namenode should be in Install failed state.",
namenodeFailedEvent);
- scheduler.stop();
}
/**
@@ -524,14 +526,13 @@ public class TestActionScheduler {
ActionScheduler scheduler = new ActionScheduler(100, 50, db, aq, fsm, 3,
new HostsMap((String) null), new ServerActionManagerImpl(fsm),
unitOfWork, conf);
- scheduler.start();
+ int cycleCount=0;
while (!stages.get(0).getHostRoleStatus(hostname, "AMBARI_SERVER_ACTION")
- .equals(HostRoleStatus.COMPLETED)) {
- Thread.sleep(100);
+ .equals(HostRoleStatus.COMPLETED) && cycleCount++ <= MAX_CYCLE_ITERATIONS) {
+ scheduler.doWork();
}
- scheduler.stop();
assertEquals(stages.get(0).getHostRoleStatus(hostname, "AMBARI_SERVER_ACTION"),
HostRoleStatus.COMPLETED);
@@ -588,13 +589,12 @@ public class TestActionScheduler {
ActionScheduler scheduler = new ActionScheduler(100, 50, db, aq, fsm, 3,
new HostsMap((String) null), new ServerActionManagerImpl(fsm), unitOfWork, conf);
- scheduler.start();
+ int cycleCount = 0;
while (!stages.get(0).getHostRoleStatus(hostname, "AMBARI_SERVER_ACTION")
- .equals(HostRoleStatus.FAILED)) {
- Thread.sleep(100);
+ .equals(HostRoleStatus.FAILED) && cycleCount++ <= MAX_CYCLE_ITERATIONS) {
+ scheduler.doWork();
}
- scheduler.stop();
assertEquals(stages.get(0).getHostRoleStatus(hostname, "AMBARI_SERVER_ACTION"),
HostRoleStatus.FAILED);
assertEquals("test", stages.get(0).getRequestContext());
@@ -1499,10 +1499,8 @@ public class TestActionScheduler {
ActionScheduler scheduler = new ActionScheduler(100, 100, db, aq, fsm,
10000, new HostsMap((String) null), null, unitOfWork, conf);
scheduler.setTaskTimeoutAdjustment(false);
- // Start the thread
- scheduler.start();
- List<AgentCommand> ac = waitForQueueSize(hostname, aq, 1);
+ List<AgentCommand> ac = waitForQueueSize(hostname, aq, 1, scheduler);
assertTrue(ac.get(0) instanceof ExecutionCommand);
assertEquals(String.valueOf(requestId1) + "-" + stageId, ((ExecutionCommand) (ac.get(0))).getCommandId());
@@ -1513,7 +1511,7 @@ public class TestActionScheduler {
when(db.getStagesInProgress()).thenReturn(Collections.singletonList(s2));
//Verify that ActionSheduler does not return cached value of cluster host info for new requestId
- ac = waitForQueueSize(hostname, aq, 1);
+ ac = waitForQueueSize(hostname, aq, 1, scheduler);
assertTrue(ac.get(0) instanceof ExecutionCommand);
assertEquals(String.valueOf(requestId2) + "-" + stageId, ((ExecutionCommand) (ac.get(0))).getCommandId());
assertEquals(clusterHostInfo2, ((ExecutionCommand) (ac.get(0))).getClusterHostInfo());
@@ -1675,14 +1673,13 @@ public class TestActionScheduler {
ActionScheduler scheduler = new ActionScheduler(100, 50, db, aq, fsm, 3,
new HostsMap((String) null), new ServerActionManagerImpl(fsm),
unitOfWork, conf);
- scheduler.start();
+ int cycleCount = 0;
while (!stages.get(0).getHostRoleStatus(hostname, "AMBARI_SERVER_ACTION")
- .equals(HostRoleStatus.COMPLETED)) {
- Thread.sleep(100);
+ .equals(HostRoleStatus.COMPLETED) && cycleCount <= MAX_CYCLE_ITERATIONS) {
+ scheduler.doWork();
}
- scheduler.stop();
assertEquals(stages.get(0).getHostRoleStatus(hostname, "AMBARI_SERVER_ACTION"),
HostRoleStatus.COMPLETED);
}
[3/6] git commit: AMBARI-7931. difference between hadoop-env.sh from
host and from UI (dlysnichenko)
Posted by dm...@apache.org.
AMBARI-7931. difference between hadoop-env.sh from host and from UI (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/78f76f02
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/78f76f02
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/78f76f02
Branch: refs/heads/branch-1.7.0
Commit: 78f76f02cef50b2d6daef184acd55138e33fb617
Parents: 9a2b140
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Fri Oct 24 12:50:34 2014 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Fri Oct 24 12:50:34 2014 +0300
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/78f76f02/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py
index 20950b9..d9fa00a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py
@@ -34,11 +34,13 @@ if stack_is_hdp22_or_further:
hadoop_libexec_dir = "/usr/hdp/current/hadoop-client/libexec"
hadoop_bin = "/usr/hdp/current/hadoop-client/sbin"
hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
+ hadoop_home = "/usr/hdp/current/hadoop-client"
else:
mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
hadoop_libexec_dir = "/usr/lib/hadoop/libexec"
hadoop_bin = "/usr/lib/hadoop/sbin"
hadoop_bin_dir = "/usr/bin"
+ hadoop_home = "/usr/lib/hadoop"
hadoop_conf_dir = "/etc/hadoop/conf"
hadoop_conf_empty_dir = "/etc/hadoop/conf.empty"
[6/6] git commit: AMBARI-7925. Namenode can not leave safe mode on
hdp-2.2 stack (dlysnichenko)
Posted by dm...@apache.org.
AMBARI-7925. Namenode can not leave safe mode on hdp-2.2 stack (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6dcab610
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6dcab610
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6dcab610
Branch: refs/heads/trunk
Commit: 6dcab6103d5894e0af4e2726f6f2b1afeeee3336
Parents: 65971b6
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Fri Oct 24 12:49:39 2014 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Fri Oct 24 12:51:59 2014 +0300
----------------------------------------------------------------------
.../HDP/1.3.2/hooks/before-START/files/checkForFormat.sh | 1 +
.../HDP/1.3.2/services/HDFS/package/files/checkForFormat.sh | 1 +
.../HDP/2.0.6/hooks/before-START/files/checkForFormat.sh | 3 ++-
.../HDP/2.0.6/services/HDFS/package/files/checkForFormat.sh | 3 ++-
.../HDP/2.0.6/services/HDFS/package/scripts/hdfs_namenode.py | 2 +-
.../HDP/2.0.6/services/HDFS/package/scripts/namenode.py | 2 +-
.../src/test/python/stacks/2.0.6/HDFS/test_namenode.py | 8 ++++----
7 files changed, 12 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6dcab610/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/files/checkForFormat.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/files/checkForFormat.sh b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/files/checkForFormat.sh
index eb48858..773510e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/files/checkForFormat.sh
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/files/checkForFormat.sh
@@ -51,6 +51,7 @@ if [[ ! -d $mark_dir ]] ; then
if [[ $EXIT_CODE == 0 ]] ; then
su -s /bin/bash - ${hdfs_user} -c "yes Y | hadoop --config ${conf_dir} ${command}"
+ (( EXIT_CODE = $EXIT_CODE | $? ))
else
echo "ERROR: Namenode directory(s) is non empty. Will not format the namenode. List of non-empty namenode dirs ${list_of_non_empty_dirs}"
fi
http://git-wip-us.apache.org/repos/asf/ambari/blob/6dcab610/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/files/checkForFormat.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/files/checkForFormat.sh b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/files/checkForFormat.sh
index eb48858..773510e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/files/checkForFormat.sh
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/files/checkForFormat.sh
@@ -51,6 +51,7 @@ if [[ ! -d $mark_dir ]] ; then
if [[ $EXIT_CODE == 0 ]] ; then
su -s /bin/bash - ${hdfs_user} -c "yes Y | hadoop --config ${conf_dir} ${command}"
+ (( EXIT_CODE = $EXIT_CODE | $? ))
else
echo "ERROR: Namenode directory(s) is non empty. Will not format the namenode. List of non-empty namenode dirs ${list_of_non_empty_dirs}"
fi
http://git-wip-us.apache.org/repos/asf/ambari/blob/6dcab610/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/checkForFormat.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/checkForFormat.sh b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/checkForFormat.sh
index 82dbda1..814cbc3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/checkForFormat.sh
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/checkForFormat.sh
@@ -53,7 +53,8 @@ if [[ ! -d $mark_dir ]] ; then
if [[ $EXIT_CODE == 0 ]] ; then
export PATH=$PATH:$bin_dir
- su -s /bin/bash - ${hdfs_user} -c "yes Y | hadoop --config ${conf_dir} ${command}"
+ su -s /bin/bash - ${hdfs_user} -c "yes Y | hdfs --config ${conf_dir} ${command}"
+ (( EXIT_CODE = $EXIT_CODE | $? ))
else
echo "ERROR: Namenode directory(s) is non empty. Will not format the namenode. List of non-empty namenode dirs ${list_of_non_empty_dirs}"
fi
http://git-wip-us.apache.org/repos/asf/ambari/blob/6dcab610/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/files/checkForFormat.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/files/checkForFormat.sh b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/files/checkForFormat.sh
index be8c75f..e6fd87d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/files/checkForFormat.sh
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/files/checkForFormat.sh
@@ -58,7 +58,8 @@ if [[ ! -d $mark_dir ]] ; then
done
if [[ $EXIT_CODE == 0 ]] ; then
- su -s /bin/bash - ${hdfs_user} -c "export PATH=$PATH:${bin_dir} ; yes Y | hadoop --config ${conf_dir} ${command}"
+ su -s /bin/bash - ${hdfs_user} -c "export PATH=$PATH:${bin_dir} ; yes Y | hdfs --config ${conf_dir} ${command}"
+ (( EXIT_CODE = $EXIT_CODE | $? ))
else
echo "ERROR: Namenode directory(s) is non empty. Will not format the namenode. List of non-empty namenode dirs ${list_of_non_empty_dirs}"
fi
http://git-wip-us.apache.org/repos/asf/ambari/blob/6dcab610/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs_namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs_namenode.py
index 31fc2f1..9a48f7b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs_namenode.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs_namenode.py
@@ -49,7 +49,7 @@ def namenode(action=None, do_format=True):
else:
dfs_check_nn_status_cmd = None
- namenode_safe_mode_off = format("su -s /bin/bash - {hdfs_user} -c 'export PATH=$PATH:{hadoop_bin_dir} ; hadoop --config {hadoop_conf_dir} dfsadmin -safemode get' | grep 'Safe mode is OFF'")
+ namenode_safe_mode_off = format("su -s /bin/bash - {hdfs_user} -c 'export PATH=$PATH:{hadoop_bin_dir} ; hdfs --config {hadoop_conf_dir} dfsadmin -safemode get' | grep 'Safe mode is OFF'")
if params.security_enabled:
Execute(format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"),
http://git-wip-us.apache.org/repos/asf/ambari/blob/6dcab610/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/namenode.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/namenode.py
index a0b07aa..ee2bb1d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/namenode.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/namenode.py
@@ -88,7 +88,7 @@ class NameNode(Script):
def startRebalancingProcess(threshold):
- rebalanceCommand = format('export PATH=$PATH:{hadoop_bin_dir} ; hadoop --config {hadoop_conf_dir} balancer -threshold {threshold}')
+ rebalanceCommand = format('export PATH=$PATH:{hadoop_bin_dir} ; hdfs --config {hadoop_conf_dir} balancer -threshold {threshold}')
return ['su','-',params.hdfs_user,'-c', rebalanceCommand]
command = startRebalancingProcess(threshold)
http://git-wip-us.apache.org/repos/asf/ambari/blob/6dcab610/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
index 2033a53..c436646 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
@@ -75,7 +75,7 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('Execute', 'ulimit -c unlimited; su -s /bin/bash - hdfs -c \'export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec && /usr/lib/hadoop/sbin/hadoop-daemon.sh --config /etc/hadoop/conf start namenode\'',
not_if = 'ls /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid >/dev/null 2>&1 && ps `cat /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid` >/dev/null 2>&1',
)
- self.assertResourceCalled('Execute', "su -s /bin/bash - hdfs -c 'export PATH=$PATH:/usr/bin ; hadoop --config /etc/hadoop/conf dfsadmin -safemode get' | grep 'Safe mode is OFF'",
+ self.assertResourceCalled('Execute', "su -s /bin/bash - hdfs -c 'export PATH=$PATH:/usr/bin ; hdfs --config /etc/hadoop/conf dfsadmin -safemode get' | grep 'Safe mode is OFF'",
tries = 40,
only_if = None,
try_sleep = 10,
@@ -182,7 +182,7 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/hdfs.headless.keytab hdfs',
user = 'hdfs',
)
- self.assertResourceCalled('Execute', "su -s /bin/bash - hdfs -c 'export PATH=$PATH:/usr/bin ; hadoop --config /etc/hadoop/conf dfsadmin -safemode get' | grep 'Safe mode is OFF'",
+ self.assertResourceCalled('Execute', "su -s /bin/bash - hdfs -c 'export PATH=$PATH:/usr/bin ; hdfs --config /etc/hadoop/conf dfsadmin -safemode get' | grep 'Safe mode is OFF'",
tries = 40,
only_if = None,
try_sleep = 10,
@@ -266,7 +266,7 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('Execute', 'ulimit -c unlimited; su -s /bin/bash - hdfs -c \'export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec && /usr/lib/hadoop/sbin/hadoop-daemon.sh --config /etc/hadoop/conf start namenode\'',
not_if = 'ls /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid >/dev/null 2>&1 && ps `cat /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid` >/dev/null 2>&1',
)
- self.assertResourceCalled('Execute', "su -s /bin/bash - hdfs -c 'export PATH=$PATH:/usr/bin ; hadoop --config /etc/hadoop/conf dfsadmin -safemode get' | grep 'Safe mode is OFF'",
+ self.assertResourceCalled('Execute', "su -s /bin/bash - hdfs -c 'export PATH=$PATH:/usr/bin ; hdfs --config /etc/hadoop/conf dfsadmin -safemode get' | grep 'Safe mode is OFF'",
tries = 40,
only_if = "su -s /bin/bash - hdfs -c 'export PATH=$PATH:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState nn1 | grep active > /dev/null'",
try_sleep = 10,
@@ -335,7 +335,7 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/hdfs.headless.keytab hdfs',
user = 'hdfs',
)
- self.assertResourceCalled('Execute', "su -s /bin/bash - hdfs -c 'export PATH=$PATH:/usr/bin ; hadoop --config /etc/hadoop/conf dfsadmin -safemode get' | grep 'Safe mode is OFF'",
+ self.assertResourceCalled('Execute', "su -s /bin/bash - hdfs -c 'export PATH=$PATH:/usr/bin ; hdfs --config /etc/hadoop/conf dfsadmin -safemode get' | grep 'Safe mode is OFF'",
tries = 40,
only_if = "su -s /bin/bash - hdfs -c 'export PATH=$PATH:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState nn1 | grep active > /dev/null'",
try_sleep = 10,
[5/6] git commit: AMBARI-7931. difference between hadoop-env.sh from
host and from UI (dlysnichenko)
Posted by dm...@apache.org.
AMBARI-7931. difference between hadoop-env.sh from host and from UI (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/06d61c1a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/06d61c1a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/06d61c1a
Branch: refs/heads/trunk
Commit: 06d61c1adcbda275d03080113dbce50b7c466cf2
Parents: 6dcab61
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Fri Oct 24 12:50:34 2014 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Fri Oct 24 12:51:59 2014 +0300
----------------------------------------------------------------------
.../stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/06d61c1a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py
index 20950b9..d9fa00a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py
@@ -34,11 +34,13 @@ if stack_is_hdp22_or_further:
hadoop_libexec_dir = "/usr/hdp/current/hadoop-client/libexec"
hadoop_bin = "/usr/hdp/current/hadoop-client/sbin"
hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
+ hadoop_home = "/usr/hdp/current/hadoop-client"
else:
mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
hadoop_libexec_dir = "/usr/lib/hadoop/libexec"
hadoop_bin = "/usr/lib/hadoop/sbin"
hadoop_bin_dir = "/usr/bin"
+ hadoop_home = "/usr/lib/hadoop"
hadoop_conf_dir = "/etc/hadoop/conf"
hadoop_conf_empty_dir = "/etc/hadoop/conf.empty"
[2/6] git commit: AMBARI-7925. Namenode can not leave safe mode on
hdp-2.2 stack (dlysnichenko)
Posted by dm...@apache.org.
AMBARI-7925. Namenode can not leave safe mode on hdp-2.2 stack (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9a2b1402
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9a2b1402
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9a2b1402
Branch: refs/heads/branch-1.7.0
Commit: 9a2b14028f0f61e24280526f65619f477d9dcab2
Parents: f0887ef
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Fri Oct 24 12:49:39 2014 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Fri Oct 24 12:49:39 2014 +0300
----------------------------------------------------------------------
.../HDP/1.3.2/hooks/before-START/files/checkForFormat.sh | 1 +
.../HDP/1.3.2/services/HDFS/package/files/checkForFormat.sh | 1 +
.../HDP/2.0.6/hooks/before-START/files/checkForFormat.sh | 3 ++-
.../HDP/2.0.6/services/HDFS/package/files/checkForFormat.sh | 3 ++-
.../HDP/2.0.6/services/HDFS/package/scripts/hdfs_namenode.py | 2 +-
.../HDP/2.0.6/services/HDFS/package/scripts/namenode.py | 2 +-
.../src/test/python/stacks/2.0.6/HDFS/test_namenode.py | 8 ++++----
7 files changed, 12 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9a2b1402/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/files/checkForFormat.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/files/checkForFormat.sh b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/files/checkForFormat.sh
index eb48858..773510e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/files/checkForFormat.sh
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/files/checkForFormat.sh
@@ -51,6 +51,7 @@ if [[ ! -d $mark_dir ]] ; then
if [[ $EXIT_CODE == 0 ]] ; then
su -s /bin/bash - ${hdfs_user} -c "yes Y | hadoop --config ${conf_dir} ${command}"
+ (( EXIT_CODE = $EXIT_CODE | $? ))
else
echo "ERROR: Namenode directory(s) is non empty. Will not format the namenode. List of non-empty namenode dirs ${list_of_non_empty_dirs}"
fi
http://git-wip-us.apache.org/repos/asf/ambari/blob/9a2b1402/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/files/checkForFormat.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/files/checkForFormat.sh b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/files/checkForFormat.sh
index eb48858..773510e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/files/checkForFormat.sh
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/files/checkForFormat.sh
@@ -51,6 +51,7 @@ if [[ ! -d $mark_dir ]] ; then
if [[ $EXIT_CODE == 0 ]] ; then
su -s /bin/bash - ${hdfs_user} -c "yes Y | hadoop --config ${conf_dir} ${command}"
+ (( EXIT_CODE = $EXIT_CODE | $? ))
else
echo "ERROR: Namenode directory(s) is non empty. Will not format the namenode. List of non-empty namenode dirs ${list_of_non_empty_dirs}"
fi
http://git-wip-us.apache.org/repos/asf/ambari/blob/9a2b1402/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/checkForFormat.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/checkForFormat.sh b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/checkForFormat.sh
index 82dbda1..814cbc3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/checkForFormat.sh
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/files/checkForFormat.sh
@@ -53,7 +53,8 @@ if [[ ! -d $mark_dir ]] ; then
if [[ $EXIT_CODE == 0 ]] ; then
export PATH=$PATH:$bin_dir
- su -s /bin/bash - ${hdfs_user} -c "yes Y | hadoop --config ${conf_dir} ${command}"
+ su -s /bin/bash - ${hdfs_user} -c "yes Y | hdfs --config ${conf_dir} ${command}"
+ (( EXIT_CODE = $EXIT_CODE | $? ))
else
echo "ERROR: Namenode directory(s) is non empty. Will not format the namenode. List of non-empty namenode dirs ${list_of_non_empty_dirs}"
fi
http://git-wip-us.apache.org/repos/asf/ambari/blob/9a2b1402/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/files/checkForFormat.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/files/checkForFormat.sh b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/files/checkForFormat.sh
index be8c75f..e6fd87d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/files/checkForFormat.sh
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/files/checkForFormat.sh
@@ -58,7 +58,8 @@ if [[ ! -d $mark_dir ]] ; then
done
if [[ $EXIT_CODE == 0 ]] ; then
- su -s /bin/bash - ${hdfs_user} -c "export PATH=$PATH:${bin_dir} ; yes Y | hadoop --config ${conf_dir} ${command}"
+ su -s /bin/bash - ${hdfs_user} -c "export PATH=$PATH:${bin_dir} ; yes Y | hdfs --config ${conf_dir} ${command}"
+ (( EXIT_CODE = $EXIT_CODE | $? ))
else
echo "ERROR: Namenode directory(s) is non empty. Will not format the namenode. List of non-empty namenode dirs ${list_of_non_empty_dirs}"
fi
http://git-wip-us.apache.org/repos/asf/ambari/blob/9a2b1402/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs_namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs_namenode.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs_namenode.py
index 31fc2f1..9a48f7b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs_namenode.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs_namenode.py
@@ -49,7 +49,7 @@ def namenode(action=None, do_format=True):
else:
dfs_check_nn_status_cmd = None
- namenode_safe_mode_off = format("su -s /bin/bash - {hdfs_user} -c 'export PATH=$PATH:{hadoop_bin_dir} ; hadoop --config {hadoop_conf_dir} dfsadmin -safemode get' | grep 'Safe mode is OFF'")
+ namenode_safe_mode_off = format("su -s /bin/bash - {hdfs_user} -c 'export PATH=$PATH:{hadoop_bin_dir} ; hdfs --config {hadoop_conf_dir} dfsadmin -safemode get' | grep 'Safe mode is OFF'")
if params.security_enabled:
Execute(format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"),
http://git-wip-us.apache.org/repos/asf/ambari/blob/9a2b1402/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/namenode.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/namenode.py
index a0b07aa..ee2bb1d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/namenode.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/namenode.py
@@ -88,7 +88,7 @@ class NameNode(Script):
def startRebalancingProcess(threshold):
- rebalanceCommand = format('export PATH=$PATH:{hadoop_bin_dir} ; hadoop --config {hadoop_conf_dir} balancer -threshold {threshold}')
+ rebalanceCommand = format('export PATH=$PATH:{hadoop_bin_dir} ; hdfs --config {hadoop_conf_dir} balancer -threshold {threshold}')
return ['su','-',params.hdfs_user,'-c', rebalanceCommand]
command = startRebalancingProcess(threshold)
http://git-wip-us.apache.org/repos/asf/ambari/blob/9a2b1402/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
index 2033a53..c436646 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
@@ -75,7 +75,7 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('Execute', 'ulimit -c unlimited; su -s /bin/bash - hdfs -c \'export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec && /usr/lib/hadoop/sbin/hadoop-daemon.sh --config /etc/hadoop/conf start namenode\'',
not_if = 'ls /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid >/dev/null 2>&1 && ps `cat /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid` >/dev/null 2>&1',
)
- self.assertResourceCalled('Execute', "su -s /bin/bash - hdfs -c 'export PATH=$PATH:/usr/bin ; hadoop --config /etc/hadoop/conf dfsadmin -safemode get' | grep 'Safe mode is OFF'",
+ self.assertResourceCalled('Execute', "su -s /bin/bash - hdfs -c 'export PATH=$PATH:/usr/bin ; hdfs --config /etc/hadoop/conf dfsadmin -safemode get' | grep 'Safe mode is OFF'",
tries = 40,
only_if = None,
try_sleep = 10,
@@ -182,7 +182,7 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/hdfs.headless.keytab hdfs',
user = 'hdfs',
)
- self.assertResourceCalled('Execute', "su -s /bin/bash - hdfs -c 'export PATH=$PATH:/usr/bin ; hadoop --config /etc/hadoop/conf dfsadmin -safemode get' | grep 'Safe mode is OFF'",
+ self.assertResourceCalled('Execute', "su -s /bin/bash - hdfs -c 'export PATH=$PATH:/usr/bin ; hdfs --config /etc/hadoop/conf dfsadmin -safemode get' | grep 'Safe mode is OFF'",
tries = 40,
only_if = None,
try_sleep = 10,
@@ -266,7 +266,7 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('Execute', 'ulimit -c unlimited; su -s /bin/bash - hdfs -c \'export HADOOP_LIBEXEC_DIR=/usr/lib/hadoop/libexec && /usr/lib/hadoop/sbin/hadoop-daemon.sh --config /etc/hadoop/conf start namenode\'',
not_if = 'ls /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid >/dev/null 2>&1 && ps `cat /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid` >/dev/null 2>&1',
)
- self.assertResourceCalled('Execute', "su -s /bin/bash - hdfs -c 'export PATH=$PATH:/usr/bin ; hadoop --config /etc/hadoop/conf dfsadmin -safemode get' | grep 'Safe mode is OFF'",
+ self.assertResourceCalled('Execute', "su -s /bin/bash - hdfs -c 'export PATH=$PATH:/usr/bin ; hdfs --config /etc/hadoop/conf dfsadmin -safemode get' | grep 'Safe mode is OFF'",
tries = 40,
only_if = "su -s /bin/bash - hdfs -c 'export PATH=$PATH:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState nn1 | grep active > /dev/null'",
try_sleep = 10,
@@ -335,7 +335,7 @@ class TestNamenode(RMFTestCase):
self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/hdfs.headless.keytab hdfs',
user = 'hdfs',
)
- self.assertResourceCalled('Execute', "su -s /bin/bash - hdfs -c 'export PATH=$PATH:/usr/bin ; hadoop --config /etc/hadoop/conf dfsadmin -safemode get' | grep 'Safe mode is OFF'",
+ self.assertResourceCalled('Execute', "su -s /bin/bash - hdfs -c 'export PATH=$PATH:/usr/bin ; hdfs --config /etc/hadoop/conf dfsadmin -safemode get' | grep 'Safe mode is OFF'",
tries = 40,
only_if = "su -s /bin/bash - hdfs -c 'export PATH=$PATH:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState nn1 | grep active > /dev/null'",
try_sleep = 10,
[4/6] git commit: AMBARI-7828. TestActionScheduler fails occasionally
on builds.a.o stating expected: but was: (dlysnichenko)
Posted by dm...@apache.org.
AMBARI-7828. TestActionScheduler fails occasionally on builds.a.o stating expected:<ABORTED> but was:<PENDING> (dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/65971b66
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/65971b66
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/65971b66
Branch: refs/heads/trunk
Commit: 65971b663a06984bb22886649caccd92a9d99515
Parents: 2d4f7d4
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Fri Oct 24 12:48:47 2014 +0300
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Fri Oct 24 12:51:59 2014 +0300
----------------------------------------------------------------------
.../actionmanager/TestActionScheduler.java | 95 ++++++++++----------
1 file changed, 46 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/65971b66/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java
index b9a38a4..7224924 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/actionmanager/TestActionScheduler.java
@@ -39,6 +39,7 @@ import java.util.concurrent.TimeUnit;
import com.google.common.reflect.TypeToken;
import com.google.inject.persist.UnitOfWork;
import junit.framework.Assert;
+import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.Role;
import org.apache.ambari.server.RoleCommand;
import org.apache.ambari.server.ServiceComponentHostNotFoundException;
@@ -86,6 +87,7 @@ public class TestActionScheduler {
+ " c6402.ambari.apache.org], slave_hosts=[c6401.ambari.apache.org,"
+ " c6402.ambari.apache.org]}";
private final String hostname = "ahost.ambari.apache.org";
+ private final int MAX_CYCLE_ITERATIONS = 100;
/**
@@ -138,19 +140,17 @@ public class TestActionScheduler {
//Keep large number of attempts so that the task is not expired finally
//Small action timeout to test rescheduling
- ActionScheduler scheduler = new ActionScheduler(100, 100, db, aq, fsm,
+ ActionScheduler scheduler = new ActionScheduler(100, 5, db, aq, fsm,
10000, new HostsMap((String) null), null, unitOfWork, conf);
scheduler.setTaskTimeoutAdjustment(false);
- // Start the thread
- scheduler.start();
- List<AgentCommand> ac = waitForQueueSize(hostname, aq, 1);
+ List<AgentCommand> ac = waitForQueueSize(hostname, aq, 1, scheduler);
assertTrue(ac.get(0) instanceof ExecutionCommand);
assertEquals("1-977", ((ExecutionCommand) (ac.get(0))).getCommandId());
assertEquals(clusterHostInfo, ((ExecutionCommand) (ac.get(0))).getClusterHostInfo());
//The action status has not changed, it should be queued again.
- ac = waitForQueueSize(hostname, aq, 1);
+ ac = waitForQueueSize(hostname, aq, 1, scheduler);
assertTrue(ac.get(0) instanceof ExecutionCommand);
assertEquals("1-977", ((ExecutionCommand) (ac.get(0))).getCommandId());
assertEquals(clusterHostInfo, ((ExecutionCommand) (ac.get(0))).getClusterHostInfo());
@@ -160,13 +160,13 @@ public class TestActionScheduler {
ac = aq.dequeueAll(hostname);
//Wait for sometime, it shouldn't be scheduled this time.
- ac = waitForQueueSize(hostname, aq, 0);
- scheduler.stop();
+ ac = waitForQueueSize(hostname, aq, 0, scheduler);
}
private List<AgentCommand> waitForQueueSize(String hostname, ActionQueue aq,
- int expectedQueueSize) throws InterruptedException {
- while (true) {
+ int expectedQueueSize, ActionScheduler scheduler) {
+ int cycleCount = 0;
+ while (cycleCount++ <= MAX_CYCLE_ITERATIONS) {
List<AgentCommand> ac = aq.dequeueAll(hostname);
if (ac != null) {
if (ac.size() == expectedQueueSize) {
@@ -176,16 +176,19 @@ public class TestActionScheduler {
+ ac.size());
}
}
- Thread.sleep(100);
+ try {
+ scheduler.doWork();
+ } catch (AmbariException e) {
+ Assert.fail("Ambari exception : " + e.getMessage() + e.getStackTrace());
+ }
}
+ return null;
}
/**
* Test whether scheduler times out an action
*/
@Test
- // BUG-15597
- @Ignore
public void testActionTimeout() throws Exception {
ActionQueue aq = new ActionQueue();
Properties properties = new Properties();
@@ -218,6 +221,10 @@ public class TestActionScheduler {
ActionDBAccessor db = mock(ActionDBAccessor.class);
when(db.getStagesInProgress()).thenReturn(stages);
+ Request request = mock(Request.class);
+ when(request.isExclusive()).thenReturn(false);
+ when(db.getRequest(anyLong())).thenReturn(request);
+
doAnswer(new Answer() {
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
@@ -231,15 +238,15 @@ public class TestActionScheduler {
//Small action timeout to test rescheduling
- ActionScheduler scheduler = new ActionScheduler(100, 50, db, aq, fsm, 3,
+ ActionScheduler scheduler = new ActionScheduler(100, 0, db, aq, fsm, 3,
new HostsMap((String) null), null, unitOfWork, conf);
scheduler.setTaskTimeoutAdjustment(false);
// Start the thread
- scheduler.start();
+ int cycleCount = 0;
while (!stages.get(0).getHostRoleStatus(hostname, "NAMENODE")
- .equals(HostRoleStatus.TIMEDOUT)) {
- Thread.sleep(100);
+ .equals(HostRoleStatus.TIMEDOUT) && cycleCount++ <= MAX_CYCLE_ITERATIONS) {
+ scheduler.doWork();
}
assertEquals(stages.get(0).getHostRoleStatus(hostname, "NAMENODE"),
HostRoleStatus.TIMEDOUT);
@@ -247,7 +254,6 @@ public class TestActionScheduler {
verify(db, times(1)).startRequest(eq(1L));
verify(db, times(1)).abortOperation(1L);
- scheduler.stop();
}
@Test
@@ -307,24 +313,23 @@ public class TestActionScheduler {
new HostsMap((String) null), sam, unitOfWork, conf).
addMockedMethod("cancelHostRoleCommands").
createMock();
+ scheduler.cancelHostRoleCommands((Collection<HostRoleCommand>)EasyMock.anyObject(),EasyMock.anyObject(String.class));
+ EasyMock.expectLastCall();
EasyMock.replay(scheduler);
scheduler.setTaskTimeoutAdjustment(false);
- // Start the thread
- scheduler.start();
+ int cycleCount=0;
while (!stages.get(0).getHostRoleStatus(hostname, "NAMENODE")
- .equals(HostRoleStatus.TIMEDOUT)) {
- Thread.sleep(100L);
+ .equals(HostRoleStatus.TIMEDOUT) && cycleCount++ <= MAX_CYCLE_ITERATIONS) {
+ scheduler.doWork();
}
- scheduler.stop();
+ Assert.assertEquals(HostRoleStatus.TIMEDOUT,stages.get(0).getHostRoleStatus(hostname, "NAMENODE"));
- //Check that cancelHostRoleCommands() was not called
EasyMock.verify(scheduler);
}
@Test
- @Ignore // This is temporary disabled as discussed here https://reviews.apache.org/r/26510/
public void testOpFailedEventRaisedForAbortedHostRole() throws Exception {
ActionQueue aq = new ActionQueue();
Properties properties = new Properties();
@@ -362,7 +367,7 @@ public class TestActionScheduler {
final List<Stage> stages = new ArrayList<Stage>();
Stage stage = new Stage(1, "/tmp", "cluster1", 1L, "stageWith2Tasks",
- CLUSTER_HOST_INFO, "", "");
+ CLUSTER_HOST_INFO, "{\"command_param\":\"param_value\"}", "{\"host_param\":\"param_value\"}");
addInstallTaskToStage(stage, hostname1, "cluster1", Role.DATANODE,
RoleCommand.INSTALL, Service.Type.HDFS, 1);
addInstallTaskToStage(stage, hostname2, "cluster1", Role.NAMENODE,
@@ -422,14 +427,12 @@ public class TestActionScheduler {
ActionScheduler scheduler = new ActionScheduler(100, 50000, db, aq, fsm, 3,
new HostsMap((String) null), null, unitOfWork, conf);
scheduler.setTaskTimeoutAdjustment(false);
- // Start the thread
- scheduler.start();
-
- while (!stages.get(0).getHostRoleStatus(hostname1, "DATANODE")
- .equals(HostRoleStatus.TIMEDOUT) && !stages.get(0).getHostRoleStatus
- (hostname2, "NAMENODE").equals(HostRoleStatus.ABORTED)) {
- Thread.sleep(100L);
+ int cycleCount=0;
+ while (!(stages.get(0).getHostRoleStatus(hostname1, "DATANODE")
+ .equals(HostRoleStatus.TIMEDOUT) && stages.get(0).getHostRoleStatus
+ (hostname2, "NAMENODE").equals(HostRoleStatus.ABORTED)) && cycleCount++ <= MAX_CYCLE_ITERATIONS) {
+ scheduler.doWork();
}
Assert.assertEquals(HostRoleStatus.TIMEDOUT,
@@ -466,7 +469,6 @@ public class TestActionScheduler {
Assert.assertNotNull("Namenode should be in Install failed state.",
namenodeFailedEvent);
- scheduler.stop();
}
/**
@@ -524,14 +526,13 @@ public class TestActionScheduler {
ActionScheduler scheduler = new ActionScheduler(100, 50, db, aq, fsm, 3,
new HostsMap((String) null), new ServerActionManagerImpl(fsm),
unitOfWork, conf);
- scheduler.start();
+ int cycleCount=0;
while (!stages.get(0).getHostRoleStatus(hostname, "AMBARI_SERVER_ACTION")
- .equals(HostRoleStatus.COMPLETED)) {
- Thread.sleep(100);
+ .equals(HostRoleStatus.COMPLETED) && cycleCount++ <= MAX_CYCLE_ITERATIONS) {
+ scheduler.doWork();
}
- scheduler.stop();
assertEquals(stages.get(0).getHostRoleStatus(hostname, "AMBARI_SERVER_ACTION"),
HostRoleStatus.COMPLETED);
@@ -588,13 +589,12 @@ public class TestActionScheduler {
ActionScheduler scheduler = new ActionScheduler(100, 50, db, aq, fsm, 3,
new HostsMap((String) null), new ServerActionManagerImpl(fsm), unitOfWork, conf);
- scheduler.start();
+ int cycleCount = 0;
while (!stages.get(0).getHostRoleStatus(hostname, "AMBARI_SERVER_ACTION")
- .equals(HostRoleStatus.FAILED)) {
- Thread.sleep(100);
+ .equals(HostRoleStatus.FAILED) && cycleCount++ <= MAX_CYCLE_ITERATIONS) {
+ scheduler.doWork();
}
- scheduler.stop();
assertEquals(stages.get(0).getHostRoleStatus(hostname, "AMBARI_SERVER_ACTION"),
HostRoleStatus.FAILED);
assertEquals("test", stages.get(0).getRequestContext());
@@ -1499,10 +1499,8 @@ public class TestActionScheduler {
ActionScheduler scheduler = new ActionScheduler(100, 100, db, aq, fsm,
10000, new HostsMap((String) null), null, unitOfWork, conf);
scheduler.setTaskTimeoutAdjustment(false);
- // Start the thread
- scheduler.start();
- List<AgentCommand> ac = waitForQueueSize(hostname, aq, 1);
+ List<AgentCommand> ac = waitForQueueSize(hostname, aq, 1, scheduler);
assertTrue(ac.get(0) instanceof ExecutionCommand);
assertEquals(String.valueOf(requestId1) + "-" + stageId, ((ExecutionCommand) (ac.get(0))).getCommandId());
@@ -1513,7 +1511,7 @@ public class TestActionScheduler {
when(db.getStagesInProgress()).thenReturn(Collections.singletonList(s2));
//Verify that ActionSheduler does not return cached value of cluster host info for new requestId
- ac = waitForQueueSize(hostname, aq, 1);
+ ac = waitForQueueSize(hostname, aq, 1, scheduler);
assertTrue(ac.get(0) instanceof ExecutionCommand);
assertEquals(String.valueOf(requestId2) + "-" + stageId, ((ExecutionCommand) (ac.get(0))).getCommandId());
assertEquals(clusterHostInfo2, ((ExecutionCommand) (ac.get(0))).getClusterHostInfo());
@@ -1675,14 +1673,13 @@ public class TestActionScheduler {
ActionScheduler scheduler = new ActionScheduler(100, 50, db, aq, fsm, 3,
new HostsMap((String) null), new ServerActionManagerImpl(fsm),
unitOfWork, conf);
- scheduler.start();
+ int cycleCount = 0;
while (!stages.get(0).getHostRoleStatus(hostname, "AMBARI_SERVER_ACTION")
- .equals(HostRoleStatus.COMPLETED)) {
- Thread.sleep(100);
+ .equals(HostRoleStatus.COMPLETED) && cycleCount <= MAX_CYCLE_ITERATIONS) {
+ scheduler.doWork();
}
- scheduler.stop();
assertEquals(stages.get(0).getHostRoleStatus(hostname, "AMBARI_SERVER_ACTION"),
HostRoleStatus.COMPLETED);
}