You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sa...@apache.org on 2013/08/28 20:03:37 UTC
svn commit: r1518306 - in
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src:
main/java/org/apache/hadoop/ha/ main/java/org/apache/hadoop/util/
test/java/org/apache/hadoop/ha/ test/java/org/apache/hadoop/util/
Author: sandy
Date: Wed Aug 28 18:03:37 2013
New Revision: 1518306
URL: http://svn.apache.org/r1518306
Log:
Adding and removing files missed for HADOOP-9906
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestZKUtil.java
Removed:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAZKUtil.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHAZKUtil.java
Added: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java?rev=1518306&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java (added)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ZKUtil.java Wed Aug 28 18:03:37 2013
@@ -0,0 +1,206 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Id;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
+import com.google.common.io.Files;
+
+/**
+ * Utilities for working with ZooKeeper.
+ */
+@InterfaceAudience.Private
+public class ZKUtil {
+
+ /**
+ * Parse ACL permission string, partially borrowed from
+ * ZooKeeperMain private method
+ */
+ private static int getPermFromString(String permString) {
+ int perm = 0;
+ for (int i = 0; i < permString.length(); i++) {
+ char c = permString.charAt(i);
+ switch (c) {
+ case 'r':
+ perm |= ZooDefs.Perms.READ;
+ break;
+ case 'w':
+ perm |= ZooDefs.Perms.WRITE;
+ break;
+ case 'c':
+ perm |= ZooDefs.Perms.CREATE;
+ break;
+ case 'd':
+ perm |= ZooDefs.Perms.DELETE;
+ break;
+ case 'a':
+ perm |= ZooDefs.Perms.ADMIN;
+ break;
+ default:
+ throw new BadAclFormatException(
+ "Invalid permission '" + c + "' in permission string '" +
+ permString + "'");
+ }
+ }
+ return perm;
+ }
+
+ /**
+ * Parse comma separated list of ACL entries to secure generated nodes, e.g.
+ * <code>sasl:hdfs/host1@MY.DOMAIN:cdrwa,sasl:hdfs/host2@MY.DOMAIN:cdrwa</code>
+ *
+ * @return ACL list
+ * @throws {@link BadAclFormatException} if an ACL is invalid
+ */
+ public static List<ACL> parseACLs(String aclString) throws
+ BadAclFormatException {
+ List<ACL> acl = Lists.newArrayList();
+ if (aclString == null) {
+ return acl;
+ }
+
+ List<String> aclComps = Lists.newArrayList(
+ Splitter.on(',').omitEmptyStrings().trimResults()
+ .split(aclString));
+ for (String a : aclComps) {
+ // from ZooKeeperMain private method
+ int firstColon = a.indexOf(':');
+ int lastColon = a.lastIndexOf(':');
+ if (firstColon == -1 || lastColon == -1 || firstColon == lastColon) {
+ throw new BadAclFormatException(
+ "ACL '" + a + "' not of expected form scheme:id:perm");
+ }
+
+ ACL newAcl = new ACL();
+ newAcl.setId(new Id(a.substring(0, firstColon), a.substring(
+ firstColon + 1, lastColon)));
+ newAcl.setPerms(getPermFromString(a.substring(lastColon + 1)));
+ acl.add(newAcl);
+ }
+
+ return acl;
+ }
+
+ /**
+ * Parse a comma-separated list of authentication mechanisms. Each
+ * such mechanism should be of the form 'scheme:auth' -- the same
+ * syntax used for the 'addAuth' command in the ZK CLI.
+ *
+ * @param authString the comma-separated auth mechanisms
+ * @return a list of parsed authentications
+ * @throws {@link BadAuthFormatException} if the auth format is invalid
+ */
+ public static List<ZKAuthInfo> parseAuth(String authString) throws
+ BadAuthFormatException{
+ List<ZKAuthInfo> ret = Lists.newArrayList();
+ if (authString == null) {
+ return ret;
+ }
+
+ List<String> authComps = Lists.newArrayList(
+ Splitter.on(',').omitEmptyStrings().trimResults()
+ .split(authString));
+
+ for (String comp : authComps) {
+ String parts[] = comp.split(":", 2);
+ if (parts.length != 2) {
+ throw new BadAuthFormatException(
+ "Auth '" + comp + "' not of expected form scheme:auth");
+ }
+ ret.add(new ZKAuthInfo(parts[0],
+ parts[1].getBytes(Charsets.UTF_8)));
+ }
+ return ret;
+ }
+
+ /**
+ * Because ZK ACLs and authentication information may be secret,
+ * allow the configuration values to be indirected through a file
+ * by specifying the configuration as "@/path/to/file". If this
+ * syntax is used, this function will return the contents of the file
+ * as a String.
+ *
+ * @param valInConf the value from the Configuration
+ * @return either the same value, or the contents of the referenced
+ * file if the configured value starts with "@"
+ * @throws IOException if the file cannot be read
+ */
+ public static String resolveConfIndirection(String valInConf)
+ throws IOException {
+ if (valInConf == null) return null;
+ if (!valInConf.startsWith("@")) {
+ return valInConf;
+ }
+ String path = valInConf.substring(1).trim();
+ return Files.toString(new File(path), Charsets.UTF_8).trim();
+ }
+
+ /**
+ * An authentication token passed to ZooKeeper.addAuthInfo
+ */
+ @InterfaceAudience.Private
+ public static class ZKAuthInfo {
+ private final String scheme;
+ private final byte[] auth;
+
+ public ZKAuthInfo(String scheme, byte[] auth) {
+ super();
+ this.scheme = scheme;
+ this.auth = auth;
+ }
+
+ public String getScheme() {
+ return scheme;
+ }
+
+ public byte[] getAuth() {
+ return auth;
+ }
+ }
+
+ @InterfaceAudience.Private
+ public static class BadAclFormatException extends
+ HadoopIllegalArgumentException {
+ private static final long serialVersionUID = 1L;
+
+ public BadAclFormatException(String message) {
+ super(message);
+ }
+ }
+
+ @InterfaceAudience.Private
+ public static class BadAuthFormatException extends
+ HadoopIllegalArgumentException {
+ private static final long serialVersionUID = 1L;
+
+ public BadAuthFormatException(String message) {
+ super(message);
+ }
+ }
+}
Added: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestZKUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestZKUtil.java?rev=1518306&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestZKUtil.java (added)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestZKUtil.java Wed Aug 28 18:03:37 2013
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.util;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.util.ZKUtil;
+import org.apache.hadoop.util.ZKUtil.BadAclFormatException;
+import org.apache.hadoop.util.ZKUtil.ZKAuthInfo;
+import org.apache.zookeeper.ZooDefs.Perms;
+import org.apache.zookeeper.data.ACL;
+import org.junit.Test;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+
+public class TestZKUtil {
+ private static final String TEST_ROOT_DIR = System.getProperty(
+ "test.build.data", "/tmp") + "/TestZKUtil";
+ private static final File TEST_FILE = new File(TEST_ROOT_DIR,
+ "test-file");
+
+ /** A path which is expected not to exist */
+ private static final String BOGUS_FILE =
+ new File("/xxxx-this-does-not-exist").getPath();
+
+ @Test
+ public void testEmptyACL() {
+ List<ACL> result = ZKUtil.parseACLs("");
+ assertTrue(result.isEmpty());
+ }
+
+ @Test
+ public void testNullACL() {
+ List<ACL> result = ZKUtil.parseACLs(null);
+ assertTrue(result.isEmpty());
+ }
+
+ @Test
+ public void testInvalidACLs() {
+ badAcl("a:b",
+ "ACL 'a:b' not of expected form scheme:id:perm"); // not enough parts
+ badAcl("a",
+ "ACL 'a' not of expected form scheme:id:perm"); // not enough parts
+ badAcl("password:foo:rx",
+ "Invalid permission 'x' in permission string 'rx'");
+ }
+
+ private static void badAcl(String acls, String expectedErr) {
+ try {
+ ZKUtil.parseACLs(acls);
+ fail("Should have failed to parse '" + acls + "'");
+ } catch (BadAclFormatException e) {
+ assertEquals(expectedErr, e.getMessage());
+ }
+ }
+
+ @Test
+ public void testGoodACLs() {
+ List<ACL> result = ZKUtil.parseACLs(
+ "sasl:hdfs/host1@MY.DOMAIN:cdrwa, sasl:hdfs/host2@MY.DOMAIN:ca");
+ ACL acl0 = result.get(0);
+ assertEquals(Perms.CREATE | Perms.DELETE | Perms.READ |
+ Perms.WRITE | Perms.ADMIN, acl0.getPerms());
+ assertEquals("sasl", acl0.getId().getScheme());
+ assertEquals("hdfs/host1@MY.DOMAIN", acl0.getId().getId());
+
+ ACL acl1 = result.get(1);
+ assertEquals(Perms.CREATE | Perms.ADMIN, acl1.getPerms());
+ assertEquals("sasl", acl1.getId().getScheme());
+ assertEquals("hdfs/host2@MY.DOMAIN", acl1.getId().getId());
+ }
+
+ @Test
+ public void testEmptyAuth() {
+ List<ZKAuthInfo> result = ZKUtil.parseAuth("");
+ assertTrue(result.isEmpty());
+ }
+
+ @Test
+ public void testNullAuth() {
+ List<ZKAuthInfo> result = ZKUtil.parseAuth(null);
+ assertTrue(result.isEmpty());
+ }
+
+ @Test
+ public void testGoodAuths() {
+ List<ZKAuthInfo> result = ZKUtil.parseAuth(
+ "scheme:data,\n scheme2:user:pass");
+ assertEquals(2, result.size());
+ ZKAuthInfo auth0 = result.get(0);
+ assertEquals("scheme", auth0.getScheme());
+ assertEquals("data", new String(auth0.getAuth()));
+
+ ZKAuthInfo auth1 = result.get(1);
+ assertEquals("scheme2", auth1.getScheme());
+ assertEquals("user:pass", new String(auth1.getAuth()));
+ }
+
+ @Test
+ public void testConfIndirection() throws IOException {
+ assertNull(ZKUtil.resolveConfIndirection(null));
+ assertEquals("x", ZKUtil.resolveConfIndirection("x"));
+
+ TEST_FILE.getParentFile().mkdirs();
+ Files.write("hello world", TEST_FILE, Charsets.UTF_8);
+ assertEquals("hello world", ZKUtil.resolveConfIndirection(
+ "@" + TEST_FILE.getAbsolutePath()));
+
+ try {
+ ZKUtil.resolveConfIndirection("@" + BOGUS_FILE);
+ fail("Did not throw for non-existent file reference");
+ } catch (FileNotFoundException fnfe) {
+ assertTrue(fnfe.getMessage().startsWith(BOGUS_FILE));
+ }
+ }
+}