You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2017/06/12 15:04:38 UTC
ambari git commit: AMBARI-21204 Yarn stopped by itself after start.
HA run (dsen)
Repository: ambari
Updated Branches:
refs/heads/trunk d0eaaae8e -> 8d129d414
AMBARI-21204 Yarn stopped by itself after start. HA run (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8d129d41
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8d129d41
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8d129d41
Branch: refs/heads/trunk
Commit: 8d129d414a9e33975620d5b0c430a7fba6d914b3
Parents: d0eaaae
Author: Dmytro Sen <ds...@apache.org>
Authored: Mon Jun 12 18:04:27 2017 +0300
Committer: Dmytro Sen <ds...@apache.org>
Committed: Mon Jun 12 18:04:27 2017 +0300
----------------------------------------------------------------------
.../org/apache/ambari/tools/zk/ZkMigrator.java | 44 +++++++++++++-
.../apache/ambari/tools/zk/ZkMigratorTest.java | 60 +++++++++++++++++++-
.../core/resources/zkmigrator.py | 17 +++++-
.../package/scripts/resourcemanager.py | 2 +-
.../package/scripts/resourcemanager.py | 2 +-
5 files changed, 117 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/8d129d41/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkMigrator.java
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkMigrator.java b/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkMigrator.java
index c100b85..e4b33eb 100644
--- a/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkMigrator.java
+++ b/ambari-agent/src/main/java/org/apache/ambari/tools/zk/ZkMigrator.java
@@ -27,9 +27,12 @@ import org.apache.commons.cli.Options;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooKeeper;
+import static org.apache.ambari.tools.zk.ZkAcl.append;
+
/**
* I'm a command line utility that provides functionality that the official zookeeper-client does not support.
* E.g. I can set ACLs recursively on a znode.
+ * Also I can remove znode recursively.
*/
public class ZkMigrator {
private static final int SESSION_TIMEOUT_MILLIS = 5000;
@@ -37,13 +40,42 @@ public class ZkMigrator {
public static void main(String[] args) throws Exception {
CommandLine cli = new DefaultParser().parse(options(), args);
- if (cli.hasOption("connection-string") && cli.hasOption("acl") && cli.hasOption("znode")) {
- setAcls(cli.getOptionValue("connection-string"), cli.getOptionValue("znode"), ZkAcl.parse(cli.getOptionValue("acl")));
+ if (cli.hasOption("connection-string") && cli.hasOption("znode")) {
+ if (cli.hasOption("acl") && !cli.hasOption("delete")) {
+ setAcls(cli.getOptionValue("connection-string"), cli.getOptionValue("znode"), ZkAcl.parse(cli.getOptionValue("acl")));
+ } else if (cli.hasOption("delete") && !cli.hasOption("acl")) {
+ deleteZnodeRecursively(cli.getOptionValue("connection-string"), cli.getOptionValue("znode"));
+ } else {
+ printHelp();
+ }
} else {
printHelp();
}
}
+ private static void deleteZnodeRecursively(String connectionString, String znode) throws IOException, InterruptedException, KeeperException {
+ ZooKeeper client = ZkConnection.open(connectionString, SESSION_TIMEOUT_MILLIS, CONNECTION_TIMEOUT_MILLIS);
+ try {
+ ZkPathPattern paths = ZkPathPattern.fromString(znode);
+ for (String path : paths.findMatchingPaths(client, "/")) {
+ System.out.println("Recursively deleting znodes with matching path " + path);
+ deleteZnodeRecursively(client, path);
+ }
+ } catch (KeeperException.NoNodeException e) {
+ System.out.println("Could not delete " + znode + ". Reason: " + e.getMessage());
+ } finally {
+ client.close();
+ }
+ }
+
+ private static void deleteZnodeRecursively(ZooKeeper zkClient, String baseNode) throws KeeperException, InterruptedException {
+ for (String child : zkClient.getChildren(baseNode, null)) {
+ deleteZnodeRecursively(zkClient, append(baseNode, child));
+ }
+ System.out.println("Deleting znode " + baseNode);
+ zkClient.delete(baseNode, -1);
+ }
+
private static Options options() {
return new Options()
.addOption(Option.builder("h")
@@ -67,6 +99,11 @@ public class ZkMigrator {
.desc("znode path")
.hasArg()
.argName("znode")
+ .build())
+ .addOption(Option.builder("d")
+ .longOpt("delete")
+ .desc("delete specified znode and all it's children recursively")
+ .argName("delete")
.build());
}
@@ -82,7 +119,8 @@ public class ZkMigrator {
}
private static void printHelp() {
- System.out.println("Usage zkmigrator -connection-string <host:port> -acl <scheme:id:permission> -znode /path/to/znode");
+ System.out.println("Usage zkmigrator -connection-string <host:port> -acl <scheme:id:permission> -znode /path/to/znode\n" +
+ " OR -connection-string <host:port> -znode /path/to/znode -delete");
System.exit(1);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8d129d41/ambari-agent/src/test/java/org/apache/ambari/tools/zk/ZkMigratorTest.java
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/java/org/apache/ambari/tools/zk/ZkMigratorTest.java b/ambari-agent/src/test/java/org/apache/ambari/tools/zk/ZkMigratorTest.java
index b7e9c0c..be9727f 100644
--- a/ambari-agent/src/test/java/org/apache/ambari/tools/zk/ZkMigratorTest.java
+++ b/ambari-agent/src/test/java/org/apache/ambari/tools/zk/ZkMigratorTest.java
@@ -23,6 +23,7 @@ import static org.apache.zookeeper.ZooDefs.Perms.READ;
import static org.apache.zookeeper.ZooDefs.Perms.WRITE;
import static org.apache.zookeeper.ZooDefs.Perms.ALL;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.net.ServerSocket;
@@ -32,6 +33,7 @@ import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.RetryOneTime;
import org.apache.curator.test.TestingServer;
+import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Id;
import org.junit.After;
@@ -69,6 +71,38 @@ public class ZkMigratorTest {
}
@Test
+ public void testDeleteRecursive() throws Exception {
+ // Given
+ path("/parent");
+ path("/parent/a");
+ path("/parent/b");
+ path("/parent/b/q");
+ // When
+ deleteZnode("/parent");
+ // Then
+ assertRemoved("/parent");
+ assertRemoved("/parent/a");
+ assertRemoved("/parent/b");
+ assertRemoved("/parent/b/q");
+ }
+
+ @Test
+ public void testDeleteRecursiveWildcard() throws Exception {
+ // Given
+ path("/parent");
+ path("/parent/a");
+ path("/parent/b");
+ path("/parent/b/q");
+ // When
+ deleteZnode("/parent/*");
+ // Then
+ assertHasNode("/parent");
+ assertRemoved("/parent/a");
+ assertRemoved("/parent/b");
+ assertRemoved("/parent/b/q");
+ }
+
+ @Test
public void testSetAclsRecursively() throws Exception {
// Given
path("/parent");
@@ -179,6 +213,14 @@ public class ZkMigratorTest {
});
}
+ private void deleteZnode(String path) throws Exception {
+ ZkMigrator.main(new String[] {
+ "-connection-string", zkTestServer.getConnectString(),
+ "-znode", path,
+ "-delete"
+ });
+ }
+
private void assertHasAcl(String path, String scheme, String id, int permission) throws Exception {
List<ACL> acls = cli.getACL().forPath(path);
assertEquals("expected 1 acl on " + path, 1, acls.size());
@@ -186,6 +228,22 @@ public class ZkMigratorTest {
assertEquals(permission, acls.get(0).getPerms());
}
+ private void assertRemoved(String path) throws Exception {
+ try {
+ cli.getACL().forPath(path);
+ assertTrue(false);
+ } catch (KeeperException.NoNodeException e) {
+ //expected
+ }
+ }
+ private void assertHasNode(String path) throws Exception {
+ try {
+ cli.getACL().forPath(path);
+ } catch (KeeperException.NoNodeException e) {
+ assertTrue(false);
+ }
+ }
+
static class Port {
public static int free() throws IOException {
ServerSocket socket = null;
@@ -199,4 +257,4 @@ public class ZkMigratorTest {
}
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8d129d41/ambari-common/src/main/python/resource_management/core/resources/zkmigrator.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/resources/zkmigrator.py b/ambari-common/src/main/python/resource_management/core/resources/zkmigrator.py
index 344b9a4..49fb5e8 100644
--- a/ambari-common/src/main/python/resource_management/core/resources/zkmigrator.py
+++ b/ambari-common/src/main/python/resource_management/core/resources/zkmigrator.py
@@ -36,12 +36,25 @@ class ZkMigrator:
def set_acls(self, znode, acl, tries=3):
Logger.info(format("Setting ACL on znode {znode} to {acl}"))
Execute(
- self._command(znode, acl), \
+ self._acl_command(znode, acl), \
user=self.user, \
environment={ 'JAVA_HOME': self.java_home }, \
logoutput=True, \
tries=tries)
- def _command(self, znode, acl):
+ def delete_node(self, znode, tries=3):
+ Logger.info(format("Removing znode {znode}"))
+ Execute(
+ self._delete_command(znode), \
+ user=self.user, \
+ environment={ 'JAVA_HOME': self.java_home }, \
+ logoutput=True, \
+ tries=tries)
+
+ def _acl_command(self, znode, acl):
return "{0} -Djava.security.auth.login.config={1} -jar {2} -connection-string {3} -znode {4} -acl {5}".format( \
self.java_exec, self.jaas_file, self.zkmigrator_jar, self.zk_host, znode, acl)
+
+ def _delete_command(self, znode):
+ return "{0} -Djava.security.auth.login.config={1} -jar {2} -connection-string {3} -znode {4} -delete".format( \
+ self.java_exec, self.jaas_file, self.zkmigrator_jar, self.zk_host, znode)
http://git-wip-us.apache.org/repos/asf/ambari/blob/8d129d41/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
index 81b99e6..7d024b1 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
@@ -181,8 +181,8 @@ class ResourcemanagerDefault(Resourcemanager):
params.yarn_jaas_file, \
params.yarn_user)
zkmigrator.set_acls(params.rm_zk_znode, 'world:anyone:crdwa')
- zkmigrator.set_acls(params.rm_zk_failover_znode, 'world:anyone:crdwa')
zkmigrator.set_acls(params.hadoop_registry_zk_root, 'world:anyone:crdwa')
+ zkmigrator.delete_node(params.rm_zk_failover_znode)
def wait_for_dfs_directories_created(self, *dirs):
import params
http://git-wip-us.apache.org/repos/asf/ambari/blob/8d129d41/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py
index 78675bf..f6d6315 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/resourcemanager.py
@@ -122,8 +122,8 @@ class ResourcemanagerDefault(Resourcemanager):
params.yarn_jaas_file, \
params.yarn_user)
zkmigrator.set_acls(params.rm_zk_znode, 'world:anyone:crdwa')
- zkmigrator.set_acls(params.rm_zk_failover_znode, 'world:anyone:crdwa')
zkmigrator.set_acls(params.hadoop_registry_zk_root, 'world:anyone:crdwa')
+ zkmigrator.delete_node(params.rm_zk_failover_znode)
def start(self, env, upgrade_type=None):
import params