You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2014/10/08 21:54:49 UTC
[2/6] YARN-913 service registry: YARN-2652 add hadoop-yarn-registry
package under hadoop-yarn
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestMarshalling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestMarshalling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestMarshalling.java
new file mode 100644
index 0000000..14e3b1f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestMarshalling.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.registry.client.binding;
+
+import org.apache.hadoop.registry.RegistryTestHelper;
+import org.apache.hadoop.registry.client.exceptions.NoRecordException;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.ServiceRecordHeader;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+
+/**
+ * Test record marshalling
+ */
+public class TestMarshalling extends RegistryTestHelper {
+ private static final Logger
+ LOG = LoggerFactory.getLogger(TestMarshalling.class);
+
+ @Rule
+ public final Timeout testTimeout = new Timeout(10000);
+ @Rule
+ public TestName methodName = new TestName();
+ private static RegistryUtils.ServiceRecordMarshal marshal;
+
+ @BeforeClass
+ public static void setupClass() {
+ marshal = new RegistryUtils.ServiceRecordMarshal();
+ }
+
+ @Test
+ public void testRoundTrip() throws Throwable {
+ String persistence = PersistencePolicies.PERMANENT;
+ ServiceRecord record = createRecord(persistence);
+ record.set("customkey","customvalue");
+ record.set("customkey2","customvalue2");
+ LOG.info(marshal.toJson(record));
+ byte[] bytes = marshal.toBytes(record);
+ ServiceRecord r2 = marshal.fromBytes("", bytes, 0);
+ assertMatches(record, r2);
+ }
+
+ @Test
+ public void testRoundTripHeaders() throws Throwable {
+ ServiceRecord record = createRecord(PersistencePolicies.CONTAINER);
+ byte[] bytes = marshal.toByteswithHeader(record);
+ ServiceRecord r2 = marshal.fromBytesWithHeader("", bytes);
+ assertMatches(record, r2);
+
+ }
+
+ @Test(expected = NoRecordException.class)
+ public void testRoundTripBadHeaders() throws Throwable {
+ ServiceRecord record = createRecord(PersistencePolicies.APPLICATION);
+ byte[] bytes = marshal.toByteswithHeader(record);
+ bytes[1] = 0x01;
+ marshal.fromBytesWithHeader("src", bytes);
+ }
+
+ @Test(expected = NoRecordException.class)
+ public void testUnmarshallHeaderTooShort() throws Throwable {
+ marshal.fromBytesWithHeader("src", new byte[]{'a'});
+ }
+
+ @Test(expected = EOFException.class)
+ public void testUnmarshallNoBody() throws Throwable {
+ byte[] bytes = ServiceRecordHeader.getData();
+ marshal.fromBytesWithHeader("src", bytes);
+ }
+
+
+ @Test
+ public void testUnknownFieldsRoundTrip() throws Throwable {
+ ServiceRecord record =
+ createRecord(PersistencePolicies.APPLICATION_ATTEMPT);
+ record.set("key", "value");
+ record.set("intval", "2");
+ assertEquals("value", record.get("key"));
+ assertEquals("2", record.get("intval"));
+ assertNull(record.get("null"));
+ assertEquals("defval", record.get("null", "defval"));
+ byte[] bytes = marshal.toByteswithHeader(record);
+ ServiceRecord r2 = marshal.fromBytesWithHeader("", bytes);
+ assertEquals("value", r2.get("key"));
+ assertEquals("2", r2.get("intval"));
+ }
+
+ @Test
+ public void testFieldPropagationInCopy() throws Throwable {
+ ServiceRecord record =
+ createRecord(PersistencePolicies.APPLICATION_ATTEMPT);
+ record.set("key", "value");
+ record.set("intval", "2");
+ ServiceRecord that = new ServiceRecord(record);
+ assertMatches(record, that);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryOperationUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryOperationUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryOperationUtils.java
new file mode 100644
index 0000000..b86e3fe
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryOperationUtils.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.registry.client.binding;
+
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests for the {@link RegistryUtils} class
+ */
+public class TestRegistryOperationUtils extends Assert {
+
+ @Test
+ public void testUsernameExtractionEnvVarOverrride() throws Throwable {
+ String whoami = RegistryUtils.getCurrentUsernameUnencoded("drwho");
+ assertEquals("drwho", whoami);
+
+ }
+
+ @Test
+ public void testUsernameExtractionCurrentuser() throws Throwable {
+ String whoami = RegistryUtils.getCurrentUsernameUnencoded("");
+ String ugiUser = UserGroupInformation.getCurrentUser().getShortUserName();
+
+ assertEquals(ugiUser, whoami);
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java
new file mode 100644
index 0000000..9a24f1c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.registry.client.binding;
+
+import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.*;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.registry.client.exceptions.InvalidPathnameException;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestRegistryPathUtils extends Assert {
+
+
+ public static final String EURO = "\u20AC";
+
+ @Test
+ public void testFormatAscii() throws Throwable {
+
+ String in = "hostname01101101-1";
+ assertConverted(in, in);
+ }
+
+ /*
+ * Euro symbol
+ */
+ @Test
+ public void testFormatEuroSymbol() throws Throwable {
+ assertConverted("xn--lzg", EURO);
+ }
+
+ @Test
+ public void testFormatIdempotent() throws Throwable {
+ assertConverted("xn--lzg", RegistryPathUtils.encodeForRegistry(EURO));
+ }
+
+ @Test
+ public void testFormatCyrillicSpaced() throws Throwable {
+ assertConverted("xn--pa 3-k4di", "\u0413PA\u0414 3");
+ }
+
+ protected void assertConverted(String expected, String in) {
+ String out = RegistryPathUtils.encodeForRegistry(in);
+ assertEquals("Conversion of " + in, expected, out);
+ }
+
+ @Test
+ public void testPaths() throws Throwable {
+ assertCreatedPathEquals("/", "/", "");
+ assertCreatedPathEquals("/", "", "");
+ assertCreatedPathEquals("/", "", "/");
+ assertCreatedPathEquals("/", "/", "/");
+
+ assertCreatedPathEquals("/a", "/a", "");
+ assertCreatedPathEquals("/a", "/", "a");
+ assertCreatedPathEquals("/a/b", "/a", "b");
+ assertCreatedPathEquals("/a/b", "/a/", "b");
+ assertCreatedPathEquals("/a/b", "/a", "/b");
+ assertCreatedPathEquals("/a/b", "/a", "/b/");
+ assertCreatedPathEquals("/a", "/a", "/");
+ assertCreatedPathEquals("/alice", "/", "/alice");
+ assertCreatedPathEquals("/alice", "/alice", "/");
+ }
+
+
+
+
+ @Test
+ public void testComplexPaths() throws Throwable {
+ assertCreatedPathEquals("/", "", "");
+ assertCreatedPathEquals("/yarn/registry/users/hadoop/org-apache-hadoop",
+ "/yarn/registry",
+ "users/hadoop/org-apache-hadoop/");
+ }
+
+
+ private static void assertCreatedPathEquals(String expected, String base,
+ String path) throws IOException {
+ String fullPath = createFullPath(base, path);
+ assertEquals("\"" + base + "\" + \"" + path + "\" =\"" + fullPath + "\"",
+ expected, fullPath);
+ }
+
+ @Test
+ public void testSplittingEmpty() throws Throwable {
+ assertEquals(0, split("").size());
+ assertEquals(0, split("/").size());
+ assertEquals(0, split("///").size());
+ }
+
+
+ @Test
+ public void testSplitting() throws Throwable {
+ assertEquals(1, split("/a").size());
+ assertEquals(0, split("/").size());
+ assertEquals(3, split("/a/b/c").size());
+ assertEquals(3, split("/a/b/c/").size());
+ assertEquals(3, split("a/b/c").size());
+ assertEquals(3, split("/a/b//c").size());
+ assertEquals(3, split("//a/b/c/").size());
+ List<String> split = split("//a/b/c/");
+ assertEquals("a", split.get(0));
+ assertEquals("b", split.get(1));
+ assertEquals("c", split.get(2));
+ }
+
+ @Test
+ public void testParentOf() throws Throwable {
+ assertEquals("/", parentOf("/a"));
+ assertEquals("/", parentOf("/a/"));
+ assertEquals("/a", parentOf("/a/b"));
+ assertEquals("/a/b", parentOf("/a/b/c"));
+ }
+
+ @Test
+ public void testLastPathEntry() throws Throwable {
+ assertEquals("",lastPathEntry("/"));
+ assertEquals("",lastPathEntry("//"));
+ assertEquals("c",lastPathEntry("/a/b/c"));
+ assertEquals("c",lastPathEntry("/a/b/c/"));
+ }
+
+ @Test(expected = PathNotFoundException.class)
+ public void testParentOfRoot() throws Throwable {
+ parentOf("/");
+ }
+
+ @Test
+ public void testValidPaths() throws Throwable {
+ assertValidPath("/");
+ assertValidPath("/a/b/c");
+ assertValidPath("/users/drwho/org-apache-hadoop/registry/appid-55-55");
+ assertValidPath("/a50");
+ }
+
+ @Test
+ public void testInvalidPaths() throws Throwable {
+ assertInvalidPath("/a_b");
+ assertInvalidPath("/UpperAndLowerCase");
+ assertInvalidPath("/space in string");
+// Is this valid? assertInvalidPath("/50");
+ }
+
+
+ private void assertValidPath(String path) throws InvalidPathnameException {
+ validateZKPath(path);
+ }
+
+
+ private void assertInvalidPath(String path) throws InvalidPathnameException {
+ try {
+ validateElementsAsDNS(path);
+ fail("path considered valid: " + path);
+ } catch (InvalidPathnameException expected) {
+ // expected
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/CuratorEventCatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/CuratorEventCatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/CuratorEventCatcher.java
new file mode 100644
index 0000000..254ab79
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/CuratorEventCatcher.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.registry.client.impl;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.api.BackgroundCallback;
+import org.apache.curator.framework.api.CuratorEvent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * This is a little event catcher for curator asynchronous
+ * operations.
+ */
+public class CuratorEventCatcher implements BackgroundCallback {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(CuratorEventCatcher.class);
+
+ public final BlockingQueue<CuratorEvent>
+ events = new LinkedBlockingQueue<CuratorEvent>(1);
+
+ private final AtomicInteger eventCounter = new AtomicInteger(0);
+
+
+ @Override
+ public void processResult(CuratorFramework client,
+ CuratorEvent event) throws
+ Exception {
+ LOG.info("received {}", event);
+ eventCounter.incrementAndGet();
+ events.put(event);
+ }
+
+
+ public int getCount() {
+ return eventCounter.get();
+ }
+
+ /**
+ * Blocking operation to take the first event off the queue
+ * @return the first event on the queue, when it arrives
+ * @throws InterruptedException if interrupted
+ */
+ public CuratorEvent take() throws InterruptedException {
+ return events.take();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/TestCuratorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/TestCuratorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/TestCuratorService.java
new file mode 100644
index 0000000..3c8b1d1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/TestCuratorService.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.registry.client.impl;
+
+import org.apache.curator.framework.api.CuratorEvent;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.service.ServiceOperations;
+import org.apache.hadoop.registry.AbstractZKRegistryTest;
+import org.apache.hadoop.registry.client.impl.zk.CuratorService;
+import org.apache.hadoop.registry.client.impl.zk.RegistrySecurity;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.data.ACL;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Test the curator service
+ */
+public class TestCuratorService extends AbstractZKRegistryTest {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestCuratorService.class);
+
+
+ protected CuratorService curatorService;
+
+ public static final String MISSING = "/missing";
+ private List<ACL> rootACL;
+
+ @Before
+ public void startCurator() throws IOException {
+ createCuratorService();
+ }
+
+ @After
+ public void stopCurator() {
+ ServiceOperations.stop(curatorService);
+ }
+
+ /**
+ * Create an instance
+ */
+ protected void createCuratorService() throws IOException {
+ curatorService = new CuratorService("curatorService");
+ curatorService.init(createRegistryConfiguration());
+ curatorService.start();
+ rootACL = RegistrySecurity.WorldReadWriteACL;
+ curatorService.maybeCreate("", CreateMode.PERSISTENT, rootACL, true);
+ }
+
+ @Test
+ public void testLs() throws Throwable {
+ curatorService.zkList("/");
+ }
+
+ @Test(expected = PathNotFoundException.class)
+ public void testLsNotFound() throws Throwable {
+ List<String> ls = curatorService.zkList(MISSING);
+ }
+
+ @Test
+ public void testExists() throws Throwable {
+ assertTrue(curatorService.zkPathExists("/"));
+ }
+
+ @Test
+ public void testExistsMissing() throws Throwable {
+ assertFalse(curatorService.zkPathExists(MISSING));
+ }
+
+ @Test
+ public void testVerifyExists() throws Throwable {
+ pathMustExist("/");
+ }
+
+ @Test(expected = PathNotFoundException.class)
+ public void testVerifyExistsMissing() throws Throwable {
+ pathMustExist("/file-not-found");
+ }
+
+ @Test
+ public void testMkdirs() throws Throwable {
+ mkPath("/p1", CreateMode.PERSISTENT);
+ pathMustExist("/p1");
+ mkPath("/p1/p2", CreateMode.EPHEMERAL);
+ pathMustExist("/p1/p2");
+ }
+
+ private void mkPath(String path, CreateMode mode) throws IOException {
+ curatorService.zkMkPath(path, mode, false,
+ RegistrySecurity.WorldReadWriteACL);
+ }
+
+ public void pathMustExist(String path) throws IOException {
+ curatorService.zkPathMustExist(path);
+ }
+
+ @Test(expected = PathNotFoundException.class)
+ public void testMkdirChild() throws Throwable {
+ mkPath("/testMkdirChild/child", CreateMode.PERSISTENT);
+ }
+
+ @Test
+ public void testMaybeCreate() throws Throwable {
+ assertTrue(curatorService.maybeCreate("/p3", CreateMode.PERSISTENT,
+ RegistrySecurity.WorldReadWriteACL, false));
+ assertFalse(curatorService.maybeCreate("/p3", CreateMode.PERSISTENT,
+ RegistrySecurity.WorldReadWriteACL, false));
+ }
+
+ @Test
+ public void testRM() throws Throwable {
+ mkPath("/rm", CreateMode.PERSISTENT);
+ curatorService.zkDelete("/rm", false, null);
+ verifyNotExists("/rm");
+ curatorService.zkDelete("/rm", false, null);
+ }
+
+ @Test
+ public void testRMNonRf() throws Throwable {
+ mkPath("/rm", CreateMode.PERSISTENT);
+ mkPath("/rm/child", CreateMode.PERSISTENT);
+ try {
+ curatorService.zkDelete("/rm", false, null);
+ fail("expected a failure");
+ } catch (PathIsNotEmptyDirectoryException expected) {
+
+ }
+ }
+
+ @Test
+ public void testRMRf() throws Throwable {
+ mkPath("/rm", CreateMode.PERSISTENT);
+ mkPath("/rm/child", CreateMode.PERSISTENT);
+ curatorService.zkDelete("/rm", true, null);
+ verifyNotExists("/rm");
+ curatorService.zkDelete("/rm", true, null);
+ }
+
+
+ @Test
+ public void testBackgroundDelete() throws Throwable {
+ mkPath("/rm", CreateMode.PERSISTENT);
+ mkPath("/rm/child", CreateMode.PERSISTENT);
+ CuratorEventCatcher events = new CuratorEventCatcher();
+ curatorService.zkDelete("/rm", true, events);
+ CuratorEvent taken = events.take();
+ LOG.info("took {}", taken);
+ assertEquals(1, events.getCount());
+ }
+
+ @Test
+ public void testCreate() throws Throwable {
+
+ curatorService.zkCreate("/testcreate",
+ CreateMode.PERSISTENT, getTestBuffer(),
+ rootACL
+ );
+ pathMustExist("/testcreate");
+ }
+
+ @Test
+ public void testCreateTwice() throws Throwable {
+ byte[] buffer = getTestBuffer();
+ curatorService.zkCreate("/testcreatetwice",
+ CreateMode.PERSISTENT, buffer,
+ rootACL);
+ try {
+ curatorService.zkCreate("/testcreatetwice",
+ CreateMode.PERSISTENT, buffer,
+ rootACL);
+ fail();
+ } catch (FileAlreadyExistsException e) {
+
+ }
+ }
+
+ @Test
+ public void testCreateUpdate() throws Throwable {
+ byte[] buffer = getTestBuffer();
+ curatorService.zkCreate("/testcreateupdate",
+ CreateMode.PERSISTENT, buffer,
+ rootACL
+ );
+ curatorService.zkUpdate("/testcreateupdate", buffer);
+ }
+
+ @Test(expected = PathNotFoundException.class)
+ public void testUpdateMissing() throws Throwable {
+ curatorService.zkUpdate("/testupdatemissing", getTestBuffer());
+ }
+
+ @Test
+ public void testUpdateDirectory() throws Throwable {
+ mkPath("/testupdatedirectory", CreateMode.PERSISTENT);
+ curatorService.zkUpdate("/testupdatedirectory", getTestBuffer());
+ }
+
+ @Test
+ public void testUpdateDirectorywithChild() throws Throwable {
+ mkPath("/testupdatedirectorywithchild", CreateMode.PERSISTENT);
+ mkPath("/testupdatedirectorywithchild/child", CreateMode.PERSISTENT);
+ curatorService.zkUpdate("/testupdatedirectorywithchild", getTestBuffer());
+ }
+
+ @Test
+ public void testUseZKServiceForBinding() throws Throwable {
+ CuratorService cs2 = new CuratorService("curator", zookeeper);
+ cs2.init(new Configuration());
+ cs2.start();
+ }
+
+ protected byte[] getTestBuffer() {
+ byte[] buffer = new byte[1];
+ buffer[0] = '0';
+ return buffer;
+ }
+
+
+ public void verifyNotExists(String path) throws IOException {
+ if (curatorService.zkPathExists(path)) {
+ fail("Path should not exist: " + path);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/TestMicroZookeeperService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/TestMicroZookeeperService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/TestMicroZookeeperService.java
new file mode 100644
index 0000000..4dfe453
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/TestMicroZookeeperService.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.registry.client.impl;
+
+import org.apache.hadoop.service.ServiceOperations;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.registry.server.services.MicroZookeeperService;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+
+/**
+ * Simple tests to look at the micro ZK service itself
+ */
+public class TestMicroZookeeperService extends Assert {
+
+ private MicroZookeeperService zookeeper;
+
+ @Rule
+ public final Timeout testTimeout = new Timeout(10000);
+ @Rule
+ public TestName methodName = new TestName();
+
+ @After
+ public void destroyZKServer() throws IOException {
+
+ ServiceOperations.stop(zookeeper);
+ }
+
+ @Test
+ public void testTempDirSupport() throws Throwable {
+ YarnConfiguration conf = new YarnConfiguration();
+ zookeeper = new MicroZookeeperService("t1");
+ zookeeper.init(conf);
+ zookeeper.start();
+ zookeeper.stop();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestRegistryRMOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestRegistryRMOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestRegistryRMOperations.java
new file mode 100644
index 0000000..451a69b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestRegistryRMOperations.java
@@ -0,0 +1,369 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.registry.integration;
+
+import org.apache.curator.framework.api.BackgroundCallback;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.registry.AbstractRegistryTest;
+import org.apache.hadoop.registry.client.api.BindFlags;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.impl.zk.ZKPathDumper;
+import org.apache.hadoop.registry.client.impl.CuratorEventCatcher;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
+import org.apache.hadoop.registry.client.types.RegistryPathStatus;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.registry.server.services.DeleteCompletionCallback;
+import org.apache.hadoop.registry.server.services.RegistryAdminService;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
+import static org.apache.hadoop.registry.client.binding.RegistryTypeUtils.inetAddrEndpoint;
+import static org.apache.hadoop.registry.client.binding.RegistryTypeUtils.restEndpoint;
+
+public class TestRegistryRMOperations extends AbstractRegistryTest {
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(TestRegistryRMOperations.class);
+
+ /**
+ * trigger a purge operation
+ * @param path path
+ * @param id yarn ID
+ * @param policyMatch policy to match ID on
+ * @param purgePolicy policy when there are children under a match
+ * @return the number purged
+ * @throws IOException
+ */
+ public int purge(String path,
+ String id,
+ String policyMatch,
+ RegistryAdminService.PurgePolicy purgePolicy) throws
+ IOException,
+ ExecutionException,
+ InterruptedException {
+ return purge(path, id, policyMatch, purgePolicy, null);
+ }
+
+ /**
+ *
+ * trigger a purge operation
+ * @param path pathn
+ * @param id yarn ID
+ * @param policyMatch policy to match ID on
+ * @param purgePolicy policy when there are children under a match
+ * @param callback optional callback
+ * @return the number purged
+ * @throws IOException
+ */
+ public int purge(String path,
+ String id,
+ String policyMatch,
+ RegistryAdminService.PurgePolicy purgePolicy,
+ BackgroundCallback callback) throws
+ IOException,
+ ExecutionException,
+ InterruptedException {
+
+ Future<Integer> future = registry.purgeRecordsAsync(path,
+ id, policyMatch, purgePolicy, callback);
+ try {
+ return future.get();
+ } catch (ExecutionException e) {
+ if (e.getCause() instanceof IOException) {
+ throw (IOException) e.getCause();
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ @Test
+ public void testPurgeEntryCuratorCallback() throws Throwable {
+
+ String path = "/users/example/hbase/hbase1/";
+ ServiceRecord written = buildExampleServiceEntry(
+ PersistencePolicies.APPLICATION_ATTEMPT);
+ written.set(YarnRegistryAttributes.YARN_ID,
+ "testAsyncPurgeEntry_attempt_001");
+
+ operations.mknode(RegistryPathUtils.parentOf(path), true);
+ operations.bind(path, written, 0);
+
+ ZKPathDumper dump = registry.dumpPath(false);
+ CuratorEventCatcher events = new CuratorEventCatcher();
+
+ LOG.info("Initial state {}", dump);
+
+ // container query
+ String id = written.get(YarnRegistryAttributes.YARN_ID, "");
+ int opcount = purge("/",
+ id,
+ PersistencePolicies.CONTAINER,
+ RegistryAdminService.PurgePolicy.PurgeAll,
+ events);
+ assertPathExists(path);
+ assertEquals(0, opcount);
+ assertEquals("Event counter", 0, events.getCount());
+
+ // now the application attempt
+ opcount = purge("/",
+ id,
+ PersistencePolicies.APPLICATION_ATTEMPT,
+ RegistryAdminService.PurgePolicy.PurgeAll,
+ events);
+
+ LOG.info("Final state {}", dump);
+
+ assertPathNotFound(path);
+ assertEquals("wrong no of delete operations in " + dump, 1, opcount);
+ // and validate the callback event
+ assertEquals("Event counter", 1, events.getCount());
+ }
+
+ @Test
+ public void testAsyncPurgeEntry() throws Throwable {
+
+ String path = "/users/example/hbase/hbase1/";
+ ServiceRecord written = buildExampleServiceEntry(
+ PersistencePolicies.APPLICATION_ATTEMPT);
+ written.set(YarnRegistryAttributes.YARN_ID,
+ "testAsyncPurgeEntry_attempt_001");
+
+ operations.mknode(RegistryPathUtils.parentOf(path), true);
+ operations.bind(path, written, 0);
+
+ ZKPathDumper dump = registry.dumpPath(false);
+
+ LOG.info("Initial state {}", dump);
+
+ DeleteCompletionCallback deletions = new DeleteCompletionCallback();
+ int opcount = purge("/",
+ written.get(YarnRegistryAttributes.YARN_ID, ""),
+ PersistencePolicies.CONTAINER,
+ RegistryAdminService.PurgePolicy.PurgeAll,
+ deletions);
+ assertPathExists(path);
+
+ dump = registry.dumpPath(false);
+
+ assertEquals("wrong no of delete operations in " + dump, 0,
+ deletions.getEventCount());
+ assertEquals("wrong no of delete operations in " + dump, 0, opcount);
+
+
+ // now app attempt
+ deletions = new DeleteCompletionCallback();
+ opcount = purge("/",
+ written.get(YarnRegistryAttributes.YARN_ID, ""),
+ PersistencePolicies.APPLICATION_ATTEMPT,
+ RegistryAdminService.PurgePolicy.PurgeAll,
+ deletions);
+
+ dump = registry.dumpPath(false);
+ LOG.info("Final state {}", dump);
+
+ assertPathNotFound(path);
+ assertEquals("wrong no of delete operations in " + dump, 1,
+ deletions.getEventCount());
+ assertEquals("wrong no of delete operations in " + dump, 1, opcount);
+ // and validate the callback event
+
+ }
+
+ @Test
+ public void testPutGetContainerPersistenceServiceEntry() throws Throwable {
+
+ String path = ENTRY_PATH;
+ ServiceRecord written = buildExampleServiceEntry(
+ PersistencePolicies.CONTAINER);
+
+ operations.mknode(RegistryPathUtils.parentOf(path), true);
+ operations.bind(path, written, BindFlags.CREATE);
+ ServiceRecord resolved = operations.resolve(path);
+ validateEntry(resolved);
+ assertMatches(written, resolved);
+ }
+
+ /**
+ * Create a complex example app
+ * @throws Throwable
+ */
+ @Test
+ public void testCreateComplexApplication() throws Throwable {
+ String appId = "application_1408631738011_0001";
+ String cid = "container_1408631738011_0001_01_";
+ String cid1 = cid + "000001";
+ String cid2 = cid + "000002";
+ String appPath = USERPATH + "tomcat";
+
+ ServiceRecord webapp = createRecord(appId,
+ PersistencePolicies.APPLICATION, "tomcat-based web application",
+ null);
+ webapp.addExternalEndpoint(restEndpoint("www",
+ new URI("http", "//loadbalancer/", null)));
+
+ ServiceRecord comp1 = createRecord(cid1, PersistencePolicies.CONTAINER,
+ null,
+ null);
+ comp1.addExternalEndpoint(restEndpoint("www",
+ new URI("http", "//rack4server3:43572", null)));
+ comp1.addInternalEndpoint(
+ inetAddrEndpoint("jmx", "JMX", "rack4server3", 43573));
+
+ // Component 2 has a container lifespan
+ ServiceRecord comp2 = createRecord(cid2, PersistencePolicies.CONTAINER,
+ null,
+ null);
+ comp2.addExternalEndpoint(restEndpoint("www",
+ new URI("http", "//rack1server28:35881", null)));
+ comp2.addInternalEndpoint(
+ inetAddrEndpoint("jmx", "JMX", "rack1server28", 35882));
+
+ operations.mknode(USERPATH, false);
+ operations.bind(appPath, webapp, BindFlags.OVERWRITE);
+ String componentsPath = appPath + RegistryConstants.SUBPATH_COMPONENTS;
+ operations.mknode(componentsPath, false);
+ String dns1 = RegistryPathUtils.encodeYarnID(cid1);
+ String dns1path = componentsPath + dns1;
+ operations.bind(dns1path, comp1, BindFlags.CREATE);
+ String dns2 = RegistryPathUtils.encodeYarnID(cid2);
+ String dns2path = componentsPath + dns2;
+ operations.bind(dns2path, comp2, BindFlags.CREATE);
+
+ ZKPathDumper pathDumper = registry.dumpPath(false);
+ LOG.info(pathDumper.toString());
+
+ logRecord("tomcat", webapp);
+ logRecord(dns1, comp1);
+ logRecord(dns2, comp2);
+
+ ServiceRecord dns1resolved = operations.resolve(dns1path);
+ assertEquals("Persistence policies on resolved entry",
+ PersistencePolicies.CONTAINER,
+ dns1resolved.get(YarnRegistryAttributes.YARN_PERSISTENCE, ""));
+
+ Map<String, RegistryPathStatus> children =
+ RegistryUtils.statChildren(operations, componentsPath);
+ assertEquals(2, children.size());
+ Collection<RegistryPathStatus>
+ componentStats = children.values();
+ Map<String, ServiceRecord> records =
+ RegistryUtils.extractServiceRecords(operations,
+ componentsPath, componentStats);
+ assertEquals(2, records.size());
+ ServiceRecord retrieved1 = records.get(dns1path);
+ logRecord(retrieved1.get(YarnRegistryAttributes.YARN_ID, ""), retrieved1);
+ assertMatches(dns1resolved, retrieved1);
+ assertEquals(PersistencePolicies.CONTAINER,
+ retrieved1.get(YarnRegistryAttributes.YARN_PERSISTENCE, ""));
+
+ // create a listing under components/
+ operations.mknode(componentsPath + "subdir", false);
+
+ // this shows up in the listing of child entries
+ Map<String, RegistryPathStatus> childrenUpdated =
+ RegistryUtils.statChildren(operations, componentsPath);
+ assertEquals(3, childrenUpdated.size());
+
+ // the non-record child this is not picked up in the record listing
+ Map<String, ServiceRecord> recordsUpdated =
+
+ RegistryUtils.extractServiceRecords(operations,
+ componentsPath,
+ childrenUpdated);
+ assertEquals(2, recordsUpdated.size());
+
+ // now do some deletions.
+
+ // synchronous delete container ID 2
+
+ // fail if the app policy is chosen
+ assertEquals(0, purge("/", cid2, PersistencePolicies.APPLICATION,
+ RegistryAdminService.PurgePolicy.FailOnChildren));
+ // succeed for container
+ assertEquals(1, purge("/", cid2, PersistencePolicies.CONTAINER,
+ RegistryAdminService.PurgePolicy.FailOnChildren));
+ assertPathNotFound(dns2path);
+ assertPathExists(dns1path);
+
+ // expect a skip on children to skip
+ assertEquals(0,
+ purge("/", appId, PersistencePolicies.APPLICATION,
+ RegistryAdminService.PurgePolicy.SkipOnChildren));
+ assertPathExists(appPath);
+ assertPathExists(dns1path);
+
+ // attempt to delete app with policy of fail on children
+ try {
+ int p = purge("/",
+ appId,
+ PersistencePolicies.APPLICATION,
+ RegistryAdminService.PurgePolicy.FailOnChildren);
+ fail("expected a failure, got a purge count of " + p);
+ } catch (PathIsNotEmptyDirectoryException expected) {
+ // expected
+ }
+ assertPathExists(appPath);
+ assertPathExists(dns1path);
+
+
+ // now trigger recursive delete
+ assertEquals(1,
+ purge("/", appId, PersistencePolicies.APPLICATION,
+ RegistryAdminService.PurgePolicy.PurgeAll));
+ assertPathNotFound(appPath);
+ assertPathNotFound(dns1path);
+
+ }
+
+ @Test
+ public void testChildDeletion() throws Throwable {
+ ServiceRecord app = createRecord("app1",
+ PersistencePolicies.APPLICATION, "app",
+ null);
+ ServiceRecord container = createRecord("container1",
+ PersistencePolicies.CONTAINER, "container",
+ null);
+
+ operations.bind("/app", app, BindFlags.OVERWRITE);
+ operations.bind("/app/container", container, BindFlags.OVERWRITE);
+
+ try {
+ int p = purge("/",
+ "app1",
+ PersistencePolicies.APPLICATION,
+ RegistryAdminService.PurgePolicy.FailOnChildren);
+ fail("expected a failure, got a purge count of " + p);
+ } catch (PathIsNotEmptyDirectoryException expected) {
+ // expected
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestYarnPolicySelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestYarnPolicySelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestYarnPolicySelector.java
new file mode 100644
index 0000000..441b3d7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestYarnPolicySelector.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.registry.integration;
+
+import org.apache.hadoop.registry.RegistryTestHelper;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
+import org.apache.hadoop.registry.client.types.RegistryPathStatus;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.server.integration.SelectByYarnPersistence;
+import org.apache.hadoop.registry.server.services.RegistryAdminService;
+import org.junit.Test;
+
+public class TestYarnPolicySelector extends RegistryTestHelper {
+
+
+ private ServiceRecord record = createRecord("1",
+ PersistencePolicies.APPLICATION, "one",
+ null);
+ private RegistryPathStatus status = new RegistryPathStatus("/", 0, 0, 1);
+
+ public void assertSelected(boolean outcome,
+ RegistryAdminService.NodeSelector selector) {
+ boolean select = selector.shouldSelect("/", status, record);
+ assertEquals(selector.toString(), outcome, select);
+ }
+
+ @Test
+ public void testByContainer() throws Throwable {
+ assertSelected(false,
+ new SelectByYarnPersistence("1",
+ PersistencePolicies.CONTAINER));
+ }
+
+ @Test
+ public void testByApp() throws Throwable {
+ assertSelected(true,
+ new SelectByYarnPersistence("1",
+ PersistencePolicies.APPLICATION));
+ }
+
+
+ @Test
+ public void testByAppName() throws Throwable {
+ assertSelected(false,
+ new SelectByYarnPersistence("2",
+ PersistencePolicies.APPLICATION));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/operations/TestRegistryOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/operations/TestRegistryOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/operations/TestRegistryOperations.java
new file mode 100644
index 0000000..1cfb025
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/operations/TestRegistryOperations.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.registry.operations;
+
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.registry.AbstractRegistryTest;
+import org.apache.hadoop.registry.client.api.BindFlags;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.exceptions.NoRecordException;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
+import org.apache.hadoop.registry.client.types.RegistryPathStatus;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class TestRegistryOperations extends AbstractRegistryTest {
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(TestRegistryOperations.class);
+
+ @Test
+ public void testPutGetServiceEntry() throws Throwable {
+ ServiceRecord written = putExampleServiceEntry(ENTRY_PATH, 0,
+ PersistencePolicies.APPLICATION);
+ ServiceRecord resolved = operations.resolve(ENTRY_PATH);
+ validateEntry(resolved);
+ assertMatches(written, resolved);
+ }
+
+ @Test
+ public void testDeleteServiceEntry() throws Throwable {
+ putExampleServiceEntry(ENTRY_PATH, 0);
+ operations.delete(ENTRY_PATH, false);
+ }
+
+ @Test
+ public void testDeleteNonexistentEntry() throws Throwable {
+ operations.delete(ENTRY_PATH, false);
+ operations.delete(ENTRY_PATH, true);
+ }
+
+ @Test
+ public void testStat() throws Throwable {
+ putExampleServiceEntry(ENTRY_PATH, 0);
+ RegistryPathStatus stat = operations.stat(ENTRY_PATH);
+ assertTrue(stat.size > 0);
+ assertTrue(stat.time > 0);
+ assertEquals(NAME, stat.path);
+ }
+
+ @Test
+ public void testLsParent() throws Throwable {
+ ServiceRecord written = putExampleServiceEntry(ENTRY_PATH, 0);
+ RegistryPathStatus stat = operations.stat(ENTRY_PATH);
+
+ List<String> children = operations.list(PARENT_PATH);
+ assertEquals(1, children.size());
+ assertEquals(NAME, children.get(0));
+ Map<String, RegistryPathStatus> childStats =
+ RegistryUtils.statChildren(operations, PARENT_PATH);
+ assertEquals(1, childStats.size());
+ assertEquals(stat, childStats.get(NAME));
+
+ Map<String, ServiceRecord> records =
+ RegistryUtils.extractServiceRecords(operations,
+ PARENT_PATH,
+ childStats.values());
+ assertEquals(1, records.size());
+ ServiceRecord record = records.get(ENTRY_PATH);
+ assertNotNull(record);
+ record.validate();
+ assertMatches(written, record);
+
+ }
+
+ @Test
+ public void testDeleteNonEmpty() throws Throwable {
+ putExampleServiceEntry(ENTRY_PATH, 0);
+ try {
+ operations.delete(PARENT_PATH, false);
+ fail("Expected a failure");
+ } catch (PathIsNotEmptyDirectoryException expected) {
+ // expected; ignore
+ }
+ operations.delete(PARENT_PATH, true);
+ }
+
+ @Test(expected = PathNotFoundException.class)
+ public void testStatEmptyPath() throws Throwable {
+ operations.stat(ENTRY_PATH);
+ }
+
+ @Test(expected = PathNotFoundException.class)
+ public void testLsEmptyPath() throws Throwable {
+ operations.list(PARENT_PATH);
+ }
+
+ @Test(expected = PathNotFoundException.class)
+ public void testResolveEmptyPath() throws Throwable {
+ operations.resolve(ENTRY_PATH);
+ }
+
+ @Test
+ public void testMkdirNoParent() throws Throwable {
+ String path = ENTRY_PATH + "/missing";
+ try {
+ operations.mknode(path, false);
+ RegistryPathStatus stat = operations.stat(path);
+ fail("Got a status " + stat);
+ } catch (PathNotFoundException expected) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testDoubleMkdir() throws Throwable {
+ operations.mknode(USERPATH, false);
+ String path = USERPATH + "newentry";
+ assertTrue(operations.mknode(path, false));
+ operations.stat(path);
+ assertFalse(operations.mknode(path, false));
+ }
+
+ @Test
+ public void testPutNoParent() throws Throwable {
+ ServiceRecord record = new ServiceRecord();
+ record.set(YarnRegistryAttributes.YARN_ID, "testPutNoParent");
+ String path = "/path/without/parent";
+ try {
+ operations.bind(path, record, 0);
+ // didn't get a failure
+ // trouble
+ RegistryPathStatus stat = operations.stat(path);
+ fail("Got a status " + stat);
+ } catch (PathNotFoundException expected) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testPutMinimalRecord() throws Throwable {
+ String path = "/path/with/minimal";
+ operations.mknode(path, true);
+ ServiceRecord record = new ServiceRecord();
+ operations.bind(path, record, BindFlags.OVERWRITE);
+ ServiceRecord resolve = operations.resolve(path);
+ assertMatches(record, resolve);
+
+ }
+
+ @Test(expected = PathNotFoundException.class)
+ public void testPutNoParent2() throws Throwable {
+ ServiceRecord record = new ServiceRecord();
+ record.set(YarnRegistryAttributes.YARN_ID, "testPutNoParent");
+ String path = "/path/without/parent";
+ operations.bind(path, record, 0);
+ }
+
+ @Test
+ public void testStatDirectory() throws Throwable {
+ String empty = "/empty";
+ operations.mknode(empty, false);
+ operations.stat(empty);
+ }
+
+ @Test
+ public void testStatRootPath() throws Throwable {
+ operations.mknode("/", false);
+ operations.stat("/");
+ operations.list("/");
+ operations.list("/");
+ }
+
+ @Test
+ public void testStatOneLevelDown() throws Throwable {
+ operations.mknode("/subdir", true);
+ operations.stat("/subdir");
+ }
+
+ @Test
+ public void testLsRootPath() throws Throwable {
+ String empty = "/";
+ operations.mknode(empty, false);
+ operations.stat(empty);
+ }
+
+ @Test
+ public void testResolvePathThatHasNoEntry() throws Throwable {
+ String empty = "/empty2";
+ operations.mknode(empty, false);
+ try {
+ ServiceRecord record = operations.resolve(empty);
+ fail("expected an exception, got " + record);
+ } catch (NoRecordException expected) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testOverwrite() throws Throwable {
+ ServiceRecord written = putExampleServiceEntry(ENTRY_PATH, 0);
+ ServiceRecord resolved1 = operations.resolve(ENTRY_PATH);
+ resolved1.description = "resolved1";
+ try {
+ operations.bind(ENTRY_PATH, resolved1, 0);
+ fail("overwrite succeeded when it should have failed");
+ } catch (FileAlreadyExistsException expected) {
+ // expected
+ }
+
+ // verify there's no changed
+ ServiceRecord resolved2 = operations.resolve(ENTRY_PATH);
+ assertMatches(written, resolved2);
+ operations.bind(ENTRY_PATH, resolved1, BindFlags.OVERWRITE);
+ ServiceRecord resolved3 = operations.resolve(ENTRY_PATH);
+ assertMatches(resolved1, resolved3);
+ }
+
+ @Test
+ public void testPutGetContainerPersistenceServiceEntry() throws Throwable {
+
+ String path = ENTRY_PATH;
+ ServiceRecord written = buildExampleServiceEntry(
+ PersistencePolicies.CONTAINER);
+
+ operations.mknode(RegistryPathUtils.parentOf(path), true);
+ operations.bind(path, written, BindFlags.CREATE);
+ ServiceRecord resolved = operations.resolve(path);
+ validateEntry(resolved);
+ assertMatches(written, resolved);
+ }
+
+ @Test
+ public void testAddingWriteAccessIsNoOpEntry() throws Throwable {
+
+ assertFalse(operations.addWriteAccessor("id","pass"));
+ operations.clearWriteAccessors();
+ }
+
+ @Test
+ public void testListListFully() throws Throwable {
+ ServiceRecord r1 = new ServiceRecord();
+ ServiceRecord r2 = createRecord("i",
+ PersistencePolicies.PERMANENT, "r2");
+
+ String path = USERPATH + SC_HADOOP + "/listing" ;
+ operations.mknode(path, true);
+ String r1path = path + "/r1";
+ operations.bind(r1path, r1, 0);
+ String r2path = path + "/r2";
+ operations.bind(r2path, r2, 0);
+
+ RegistryPathStatus r1stat = operations.stat(r1path);
+ assertEquals("r1", r1stat.path);
+ RegistryPathStatus r2stat = operations.stat(r2path);
+ assertEquals("r2", r2stat.path);
+ assertNotEquals(r1stat, r2stat);
+
+ // listings now
+ List<String> list = operations.list(path);
+ assertEquals("Wrong no. of children", 2, list.size());
+ // there's no order here, so create one
+ Map<String, String> names = new HashMap<String, String>();
+ String entries = "";
+ for (String child : list) {
+ names.put(child, child);
+ entries += child + " ";
+ }
+ assertTrue("No 'r1' in " + entries,
+ names.containsKey("r1"));
+ assertTrue("No 'r2' in " + entries,
+ names.containsKey("r2"));
+
+ Map<String, RegistryPathStatus> stats =
+ RegistryUtils.statChildren(operations, path);
+ assertEquals("Wrong no. of children", 2, stats.size());
+ assertEquals(r1stat, stats.get("r1"));
+ assertEquals(r2stat, stats.get("r2"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java
new file mode 100644
index 0000000..ca3f9c9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java
@@ -0,0 +1,356 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.registry.secure;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.util.KerberosName;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.service.ServiceOperations;
+import org.apache.hadoop.registry.RegistryTestHelper;
+import org.apache.hadoop.registry.client.impl.zk.RegistrySecurity;
+import org.apache.hadoop.registry.client.impl.zk.ZookeeperConfigOptions;
+import org.apache.hadoop.registry.server.services.AddingCompositeService;
+import org.apache.hadoop.registry.server.services.MicroZookeeperService;
+import org.apache.hadoop.registry.server.services.MicroZookeeperServiceKeys;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.rules.TestName;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.security.auth.Subject;
+import javax.security.auth.kerberos.KerberosPrincipal;
+import javax.security.auth.login.LoginContext;
+import javax.security.auth.login.LoginException;
+import java.io.File;
+import java.io.IOException;
+import java.security.Principal;
+import java.util.HashSet;
+import java.util.Properties;
+import java.util.Set;
+
+/**
+ * Add kerberos tests. This is based on the (JUnit3) KerberosSecurityTestcase
+ * and its test case, <code>TestMiniKdc</code>
+ */
+public class AbstractSecureRegistryTest extends RegistryTestHelper {
+ public static final String REALM = "EXAMPLE.COM";
+ public static final String ZOOKEEPER = "zookeeper";
+ public static final String ZOOKEEPER_LOCALHOST = "zookeeper/localhost";
+ public static final String ZOOKEEPER_REALM = "zookeeper@" + REALM;
+ public static final String ZOOKEEPER_CLIENT_CONTEXT = ZOOKEEPER;
+ public static final String ZOOKEEPER_SERVER_CONTEXT = "ZOOKEEPER_SERVER";
+ ;
+ public static final String ZOOKEEPER_LOCALHOST_REALM =
+ ZOOKEEPER_LOCALHOST + "@" + REALM;
+ public static final String ALICE = "alice";
+ public static final String ALICE_CLIENT_CONTEXT = "alice";
+ public static final String ALICE_LOCALHOST = "alice/localhost";
+ public static final String BOB = "bob";
+ public static final String BOB_CLIENT_CONTEXT = "bob";
+ public static final String BOB_LOCALHOST = "bob/localhost";
+
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AbstractSecureRegistryTest.class);
+
+ public static final Configuration CONF;
+
+ static {
+ CONF = new Configuration();
+ CONF.set("hadoop.security.authentication", "kerberos");
+ CONF.setBoolean("hadoop.security.authorization", true);
+ }
+
+ private static final AddingCompositeService classTeardown =
+ new AddingCompositeService("classTeardown");
+
+ // static initializer guarantees it is always started
+ // ahead of any @BeforeClass methods
+ static {
+ classTeardown.init(CONF);
+ classTeardown.start();
+ }
+
+ public static final String SUN_SECURITY_KRB5_DEBUG =
+ "sun.security.krb5.debug";
+
+ private final AddingCompositeService teardown =
+ new AddingCompositeService("teardown");
+
+ protected static MiniKdc kdc;
+ protected static File keytab_zk;
+ protected static File keytab_bob;
+ protected static File keytab_alice;
+ protected static File kdcWorkDir;
+ protected static Properties kdcConf;
+ protected static RegistrySecurity registrySecurity;
+
+ @Rule
+ public final Timeout testTimeout = new Timeout(900000);
+
+ @Rule
+ public TestName methodName = new TestName();
+ protected MicroZookeeperService secureZK;
+ protected static File jaasFile;
+ private LoginContext zookeeperLogin;
+
+ /**
+ * All class initialization for this test class
+ * @throws Exception
+ */
+ @BeforeClass
+ public static void beforeSecureRegistryTestClass() throws Exception {
+ registrySecurity = new RegistrySecurity("registrySecurity");
+ registrySecurity.init(CONF);
+ setupKDCAndPrincipals();
+ RegistrySecurity.clearJaasSystemProperties();
+ RegistrySecurity.bindJVMtoJAASFile(jaasFile);
+ initHadoopSecurity();
+ }
+
+ @AfterClass
+ public static void afterSecureRegistryTestClass() throws
+ Exception {
+ describe(LOG, "teardown of class");
+ classTeardown.close();
+ teardownKDC();
+ }
+
+ /**
+ * give our thread a name
+ */
+ @Before
+ public void nameThread() {
+ Thread.currentThread().setName("JUnit");
+ }
+
+ /**
+ * For unknown reasons, the before-class setting of the JVM properties were
+ * not being picked up. This method addresses that by setting them
+ * before every test case
+ */
+ @Before
+ public void beforeSecureRegistryTest() {
+
+ }
+
+ @After
+ public void afterSecureRegistryTest() throws IOException {
+ describe(LOG, "teardown of instance");
+ teardown.close();
+ stopSecureZK();
+ }
+
+ protected static void addToClassTeardown(Service svc) {
+ classTeardown.addService(svc);
+ }
+
+ protected void addToTeardown(Service svc) {
+ teardown.addService(svc);
+ }
+
+
+ public static void teardownKDC() throws Exception {
+ if (kdc != null) {
+ kdc.stop();
+ kdc = null;
+ }
+ }
+
+ /**
+ * Sets up the KDC and a set of principals in the JAAS file
+ *
+ * @throws Exception
+ */
+ public static void setupKDCAndPrincipals() throws Exception {
+ // set up the KDC
+ File target = new File(System.getProperty("test.dir", "target"));
+ kdcWorkDir = new File(target, "kdc");
+ kdcWorkDir.mkdirs();
+ if (!kdcWorkDir.mkdirs()) {
+ assertTrue(kdcWorkDir.isDirectory());
+ }
+ kdcConf = MiniKdc.createConf();
+ kdcConf.setProperty(MiniKdc.DEBUG, "true");
+ kdc = new MiniKdc(kdcConf, kdcWorkDir);
+ kdc.start();
+
+ keytab_zk = createKeytab(ZOOKEEPER, "zookeeper.keytab");
+ keytab_alice = createKeytab(ALICE, "alice.keytab");
+ keytab_bob = createKeytab(BOB, "bob.keytab");
+
+ StringBuilder jaas = new StringBuilder(1024);
+ jaas.append(registrySecurity.createJAASEntry(ZOOKEEPER_CLIENT_CONTEXT,
+ ZOOKEEPER, keytab_zk));
+ jaas.append(registrySecurity.createJAASEntry(ZOOKEEPER_SERVER_CONTEXT,
+ ZOOKEEPER_LOCALHOST, keytab_zk));
+ jaas.append(registrySecurity.createJAASEntry(ALICE_CLIENT_CONTEXT,
+ ALICE_LOCALHOST , keytab_alice));
+ jaas.append(registrySecurity.createJAASEntry(BOB_CLIENT_CONTEXT,
+ BOB_LOCALHOST, keytab_bob));
+
+ jaasFile = new File(kdcWorkDir, "jaas.txt");
+ FileUtils.write(jaasFile, jaas.toString());
+ LOG.info("\n"+ jaas);
+ RegistrySecurity.bindJVMtoJAASFile(jaasFile);
+ }
+
+
+ //
+ protected static final String kerberosRule =
+ "RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT";
+
+ /**
+ * Init hadoop security by setting up the UGI config
+ */
+ public static void initHadoopSecurity() {
+
+ UserGroupInformation.setConfiguration(CONF);
+
+ KerberosName.setRules(kerberosRule);
+ }
+
+ /**
+ * Stop the secure ZK and log out the ZK account
+ */
+ public synchronized void stopSecureZK() {
+ ServiceOperations.stop(secureZK);
+ secureZK = null;
+ logout(zookeeperLogin);
+ zookeeperLogin = null;
+ }
+
+
+ public static MiniKdc getKdc() {
+ return kdc;
+ }
+
+ public static File getKdcWorkDir() {
+ return kdcWorkDir;
+ }
+
+ public static Properties getKdcConf() {
+ return kdcConf;
+ }
+
+ /**
+ * Create a secure instance
+ * @param name instance name
+ * @return the instance
+ * @throws Exception
+ */
+ protected static MicroZookeeperService createSecureZKInstance(String name)
+ throws Exception {
+ String context = ZOOKEEPER_SERVER_CONTEXT;
+ Configuration conf = new Configuration();
+
+ File testdir = new File(System.getProperty("test.dir", "target"));
+ File workDir = new File(testdir, name);
+ if (!workDir.mkdirs()) {
+ assertTrue(workDir.isDirectory());
+ }
+ System.setProperty(
+ ZookeeperConfigOptions.PROP_ZK_SERVER_MAINTAIN_CONNECTION_DESPITE_SASL_FAILURE,
+ "false");
+ RegistrySecurity.validateContext(context);
+ conf.set(MicroZookeeperServiceKeys.KEY_REGISTRY_ZKSERVICE_JAAS_CONTEXT,
+ context);
+ MicroZookeeperService secureZK = new MicroZookeeperService(name);
+ secureZK.init(conf);
+ LOG.info(secureZK.getDiagnostics());
+ return secureZK;
+ }
+
+ /**
+ * Create the keytabl for the given principal, includes
+ * raw principal and $principal/localhost
+ * @param principal principal short name
+ * @param filename filename of keytab
+ * @return file of keytab
+ * @throws Exception
+ */
+ public static File createKeytab(String principal,
+ String filename) throws Exception {
+ assertNotEmpty("empty principal", principal);
+ assertNotEmpty("empty host", filename);
+ assertNotNull("Null KDC", kdc);
+ File keytab = new File(kdcWorkDir, filename);
+ kdc.createPrincipal(keytab, principal, principal +"/localhost");
+ return keytab;
+ }
+
+ public static String getPrincipalAndRealm(String principal) {
+ return principal + "@" + getRealm();
+ }
+
+ protected static String getRealm() {
+ return kdc.getRealm();
+ }
+
+
+ /**
+ * Log in, defaulting to the client context
+ * @param principal principal
+ * @param context context
+ * @param keytab keytab
+ * @return the logged in context
+ * @throws LoginException failure to log in
+ */
+ protected LoginContext login(String principal,
+ String context, File keytab) throws LoginException {
+ LOG.info("Logging in as {} in context {} with keytab {}",
+ principal, context, keytab);
+ Set<Principal> principals = new HashSet<Principal>();
+ principals.add(new KerberosPrincipal(principal));
+ Subject subject = new Subject(false, principals, new HashSet<Object>(),
+ new HashSet<Object>());
+ LoginContext login;
+ login = new LoginContext(context, subject, null,
+ KerberosConfiguration.createClientConfig(principal, keytab));
+ login.login();
+ return login;
+ }
+
+
+ /**
+ * Start the secure ZK instance using the test method name as the path.
+ * As the entry is saved to the {@link #secureZK} field, it
+ * is automatically stopped after the test case.
+ * @throws Exception on any failure
+ */
+ protected synchronized void startSecureZK() throws Exception {
+ assertNull("Zookeeper is already running", secureZK);
+
+ zookeeperLogin = login(ZOOKEEPER_LOCALHOST,
+ ZOOKEEPER_SERVER_CONTEXT,
+ keytab_zk);
+ secureZK = createSecureZKInstance("test-" + methodName.getMethodName());
+ secureZK.start();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/KerberosConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/KerberosConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/KerberosConfiguration.java
new file mode 100644
index 0000000..f511bf6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/KerberosConfiguration.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.registry.secure;
+
+import org.apache.hadoop.security.authentication.util.KerberosUtil;
+
+import javax.security.auth.login.AppConfigurationEntry;
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+
+class KerberosConfiguration extends javax.security.auth.login.Configuration {
+ private String principal;
+ private String keytab;
+ private boolean isInitiator;
+
+ KerberosConfiguration(String principal, File keytab,
+ boolean client) {
+ this.principal = principal;
+ this.keytab = keytab.getAbsolutePath();
+ this.isInitiator = client;
+ }
+
+ public static javax.security.auth.login.Configuration createClientConfig(
+ String principal,
+ File keytab) {
+ return new KerberosConfiguration(principal, keytab, true);
+ }
+
+ public static javax.security.auth.login.Configuration createServerConfig(
+ String principal,
+ File keytab) {
+ return new KerberosConfiguration(principal, keytab, false);
+ }
+
+ @Override
+ public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
+ Map<String, String> options = new HashMap<String, String>();
+ options.put("keyTab", keytab);
+ options.put("principal", principal);
+ options.put("useKeyTab", "true");
+ options.put("storeKey", "true");
+ options.put("doNotPrompt", "true");
+ options.put("useTicketCache", "true");
+ options.put("renewTGT", "true");
+ options.put("refreshKrb5Config", "true");
+ options.put("isInitiator", Boolean.toString(isInitiator));
+ String ticketCache = System.getenv("KRB5CCNAME");
+ if (ticketCache != null) {
+ options.put("ticketCache", ticketCache);
+ }
+ options.put("debug", "true");
+
+ return new AppConfigurationEntry[]{
+ new AppConfigurationEntry(KerberosUtil.getKrb5LoginModuleName(),
+ AppConfigurationEntry.LoginModuleControlFlag.REQUIRED,
+ options)
+ };
+ }
+
+ @Override
+ public String toString() {
+ return "KerberosConfiguration with principal " + principal;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestRegistrySecurityHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestRegistrySecurityHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestRegistrySecurityHelper.java
new file mode 100644
index 0000000..8d0dc6a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestRegistrySecurityHelper.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.registry.secure;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.impl.zk.RegistrySecurity;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.data.ACL;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.hadoop.registry.client.api.RegistryConstants.*;
+
+/**
+ * Test for registry security operations
+ */
+public class TestRegistrySecurityHelper extends Assert {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestRegistrySecurityHelper.class);
+
+ public static final String YARN_EXAMPLE_COM = "yarn@example.com";
+ public static final String SASL_YARN_EXAMPLE_COM =
+ "sasl:" + YARN_EXAMPLE_COM;
+ public static final String MAPRED_EXAMPLE_COM = "mapred@example.com";
+ public static final String SASL_MAPRED_EXAMPLE_COM =
+ "sasl:" + MAPRED_EXAMPLE_COM;
+ public static final String SASL_MAPRED_APACHE = "sasl:mapred@APACHE";
+ public static final String DIGEST_F0AF = "digest:f0afbeeb00baa";
+ public static final String SASL_YARN_SHORT = "sasl:yarn@";
+ public static final String SASL_MAPRED_SHORT = "sasl:mapred@";
+ public static final String REALM_EXAMPLE_COM = "example.com";
+ private static RegistrySecurity registrySecurity;
+
+ @BeforeClass
+ public static void setupTestRegistrySecurityHelper() throws IOException {
+ Configuration conf = new Configuration();
+ conf.setBoolean(KEY_REGISTRY_SECURE, true);
+ conf.set(KEY_REGISTRY_KERBEROS_REALM, "KERBEROS");
+ registrySecurity = new RegistrySecurity("");
+ // init the ACLs OUTSIDE A KERBEROS CLUSTER
+ registrySecurity.init(conf);
+ }
+
+ @Test
+ public void testACLSplitRealmed() throws Throwable {
+ List<String> pairs =
+ registrySecurity.splitAclPairs(
+ SASL_YARN_EXAMPLE_COM +
+ ", " +
+ SASL_MAPRED_EXAMPLE_COM,
+ "");
+
+ assertEquals(SASL_YARN_EXAMPLE_COM, pairs.get(0));
+ assertEquals(SASL_MAPRED_EXAMPLE_COM, pairs.get(1));
+ }
+
+
+ @Test
+ public void testBuildAclsRealmed() throws Throwable {
+ List<ACL> acls = registrySecurity.buildACLs(
+ SASL_YARN_EXAMPLE_COM +
+ ", " +
+ SASL_MAPRED_EXAMPLE_COM,
+ "",
+ ZooDefs.Perms.ALL);
+ assertEquals(YARN_EXAMPLE_COM, acls.get(0).getId().getId());
+ assertEquals(MAPRED_EXAMPLE_COM, acls.get(1).getId().getId());
+ }
+
+ @Test
+ public void testACLDefaultRealm() throws Throwable {
+ List<String> pairs =
+ registrySecurity.splitAclPairs(
+ SASL_YARN_SHORT +
+ ", " +
+ SASL_MAPRED_SHORT,
+ REALM_EXAMPLE_COM);
+
+ assertEquals(SASL_YARN_EXAMPLE_COM, pairs.get(0));
+ assertEquals(SASL_MAPRED_EXAMPLE_COM, pairs.get(1));
+ }
+
+ @Test
+ public void testBuildAclsDefaultRealm() throws Throwable {
+ List<ACL> acls = registrySecurity.buildACLs(
+ SASL_YARN_SHORT +
+ ", " +
+ SASL_MAPRED_SHORT,
+ REALM_EXAMPLE_COM, ZooDefs.Perms.ALL);
+
+ assertEquals(YARN_EXAMPLE_COM, acls.get(0).getId().getId());
+ assertEquals(MAPRED_EXAMPLE_COM, acls.get(1).getId().getId());
+ }
+
+ @Test
+ public void testACLSplitNullRealm() throws Throwable {
+ List<String> pairs =
+ registrySecurity.splitAclPairs(
+ SASL_YARN_SHORT +
+ ", " +
+ SASL_MAPRED_SHORT,
+ "");
+
+ assertEquals(SASL_YARN_SHORT, pairs.get(0));
+ assertEquals(SASL_MAPRED_SHORT, pairs.get(1));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testBuildAclsNullRealm() throws Throwable {
+ registrySecurity.buildACLs(
+ SASL_YARN_SHORT +
+ ", " +
+ SASL_MAPRED_SHORT,
+ "", ZooDefs.Perms.ALL);
+ fail("");
+
+ }
+
+ @Test
+ public void testACLDefaultRealmOnlySASL() throws Throwable {
+ List<String> pairs =
+ registrySecurity.splitAclPairs(
+ SASL_YARN_SHORT +
+ ", " +
+ DIGEST_F0AF,
+ REALM_EXAMPLE_COM);
+
+ assertEquals(SASL_YARN_EXAMPLE_COM, pairs.get(0));
+ assertEquals(DIGEST_F0AF, pairs.get(1));
+ }
+
+ @Test
+ public void testACLSplitMixed() throws Throwable {
+ List<String> pairs =
+ registrySecurity.splitAclPairs(
+ SASL_YARN_SHORT +
+ ", " +
+ SASL_MAPRED_APACHE +
+ ", ,," +
+ DIGEST_F0AF,
+ REALM_EXAMPLE_COM);
+
+ assertEquals(SASL_YARN_EXAMPLE_COM, pairs.get(0));
+ assertEquals(SASL_MAPRED_APACHE, pairs.get(1));
+ assertEquals(DIGEST_F0AF, pairs.get(2));
+ }
+
+ @Test
+ public void testDefaultAClsValid() throws Throwable {
+ registrySecurity.buildACLs(
+ RegistryConstants.DEFAULT_REGISTRY_SYSTEM_ACCOUNTS,
+ REALM_EXAMPLE_COM, ZooDefs.Perms.ALL);
+ }
+
+ @Test
+ public void testDefaultRealm() throws Throwable {
+ String realm = RegistrySecurity.getDefaultRealmInJVM();
+ LOG.info("Realm {}", realm);
+ }
+
+ @Test
+ public void testUGIProperties() throws Throwable {
+ UserGroupInformation user = UserGroupInformation.getCurrentUser();
+ ACL acl = registrySecurity.createACLForUser(user, ZooDefs.Perms.ALL);
+ assertFalse(RegistrySecurity.ALL_READWRITE_ACCESS.equals(acl));
+ LOG.info("User {} has ACL {}", user, acl);
+ }
+
+
+ @Test
+ public void testSecurityImpliesKerberos() throws Throwable {
+ Configuration conf = new Configuration();
+ conf.setBoolean("hadoop.security.authentication", true);
+ conf.setBoolean(KEY_REGISTRY_SECURE, true);
+ conf.set(KEY_REGISTRY_KERBEROS_REALM, "KERBEROS");
+ RegistrySecurity security = new RegistrySecurity("registry security");
+ try {
+ security.init(conf);
+ } catch (Exception e) {
+ assertTrue(
+ "did not find "+ RegistrySecurity.E_NO_KERBEROS + " in " + e,
+ e.toString().contains(RegistrySecurity.E_NO_KERBEROS));
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java
new file mode 100644
index 0000000..ab9d490
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.registry.secure;
+
+
+
+import com.sun.security.auth.module.Krb5LoginModule;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.security.HadoopKerberosName;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.util.KerberosName;
+import org.apache.hadoop.security.authentication.util.KerberosUtil;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.registry.client.impl.zk.RegistrySecurity;
+import org.apache.hadoop.registry.client.impl.zk.ZookeeperConfigOptions;
+import org.apache.zookeeper.Environment;
+import org.apache.zookeeper.data.ACL;
+import org.junit.Assume;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.security.auth.Subject;
+import javax.security.auth.kerberos.KerberosPrincipal;
+import javax.security.auth.login.LoginContext;
+import javax.security.auth.login.LoginException;
+import java.io.File;
+import java.io.IOException;
+import java.security.Principal;
+import java.security.PrivilegedExceptionAction;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Verify that logins work
+ */
+public class TestSecureLogins extends AbstractSecureRegistryTest {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestSecureLogins.class);
+
+ @Test
+ public void testZKinKeytab() throws Throwable {
+ Assume.assumeTrue(!Shell.WINDOWS);
+ try {
+ String listing = ktList(keytab_zk);
+ assertTrue("no " + ZOOKEEPER_LOCALHOST + " in " + listing,
+ listing.contains(ZOOKEEPER_LOCALHOST));
+ } catch (IOException e) {
+ LOG.debug(KTUTIL + " failure: {}", e, e);
+ Assume.assumeTrue("Failed to run "+ KTUTIL+": " + e, false );
+ }
+ }
+
+ @Test
+ public void testHasRealm() throws Throwable {
+ assertNotNull(getRealm());
+ LOG.info("ZK principal = {}", getPrincipalAndRealm(ZOOKEEPER_LOCALHOST));
+ }
+
+ @Test
+ public void testJaasFileSetup() throws Throwable {
+ // the JVM has seemed inconsistent on setting up here
+ assertNotNull("jaasFile", jaasFile);
+ String confFilename = System.getProperty(Environment.JAAS_CONF_KEY);
+ assertEquals(jaasFile.getAbsolutePath(), confFilename);
+ }
+
+ @Test
+ public void testJaasFileBinding() throws Throwable {
+ // the JVM has seemed inconsistent on setting up here
+ assertNotNull("jaasFile", jaasFile);
+ RegistrySecurity.bindJVMtoJAASFile(jaasFile);
+ String confFilename = System.getProperty(Environment.JAAS_CONF_KEY);
+ assertEquals(jaasFile.getAbsolutePath(), confFilename);
+ }
+
+
+ @Test
+ public void testClientLogin() throws Throwable {
+ LoginContext client = login(ALICE_LOCALHOST,
+ ALICE_CLIENT_CONTEXT,
+ keytab_alice);
+
+ logLoginDetails(ALICE_LOCALHOST, client);
+ String confFilename = System.getProperty(Environment.JAAS_CONF_KEY);
+ assertNotNull("Unset: "+ Environment.JAAS_CONF_KEY, confFilename);
+ String config = FileUtils.readFileToString(new File(confFilename));
+ LOG.info("{}=\n{}", confFilename, config);
+ RegistrySecurity.setZKSaslClientProperties(ALICE, ALICE_CLIENT_CONTEXT);
+ client.logout();
+ }
+
+
+ @Test
+ public void testServerLogin() throws Throwable {
+ LoginContext loginContext = createLoginContextZookeeperLocalhost();
+ loginContext.login();
+ loginContext.logout();
+ }
+
+ public LoginContext createLoginContextZookeeperLocalhost() throws
+ LoginException {
+ String principalAndRealm = getPrincipalAndRealm(ZOOKEEPER_LOCALHOST);
+ Set<Principal> principals = new HashSet<Principal>();
+ principals.add(new KerberosPrincipal(ZOOKEEPER_LOCALHOST));
+ Subject subject = new Subject(false, principals, new HashSet<Object>(),
+ new HashSet<Object>());
+ return new LoginContext("", subject, null,
+ KerberosConfiguration.createServerConfig(ZOOKEEPER_LOCALHOST, keytab_zk));
+ }
+
+
+ @Test
+ public void testKerberosAuth() throws Throwable {
+ File krb5conf = getKdc().getKrb5conf();
+ String krbConfig = FileUtils.readFileToString(krb5conf);
+ LOG.info("krb5.conf at {}:\n{}", krb5conf, krbConfig);
+ Subject subject = new Subject();
+
+ final Krb5LoginModule krb5LoginModule = new Krb5LoginModule();
+ final Map<String, String> options = new HashMap<String, String>();
+ options.put("keyTab", keytab_alice.getAbsolutePath());
+ options.put("principal", ALICE_LOCALHOST);
+ options.put("debug", "true");
+ options.put("doNotPrompt", "true");
+ options.put("isInitiator", "true");
+ options.put("refreshKrb5Config", "true");
+ options.put("renewTGT", "true");
+ options.put("storeKey", "true");
+ options.put("useKeyTab", "true");
+ options.put("useTicketCache", "true");
+
+ krb5LoginModule.initialize(subject, null,
+ new HashMap<String, String>(),
+ options);
+
+ boolean loginOk = krb5LoginModule.login();
+ assertTrue("Failed to login", loginOk);
+ boolean commitOk = krb5LoginModule.commit();
+ assertTrue("Failed to Commit", commitOk);
+ }
+
+ @Test
+ public void testDefaultRealmValid() throws Throwable {
+ String defaultRealm = KerberosUtil.getDefaultRealm();
+ assertNotEmpty("No default Kerberos Realm",
+ defaultRealm);
+ LOG.info("Default Realm '{}'", defaultRealm);
+ }
+
+ @Test
+ public void testKerberosRulesValid() throws Throwable {
+ assertTrue("!KerberosName.hasRulesBeenSet()",
+ KerberosName.hasRulesBeenSet());
+ String rules = KerberosName.getRules();
+ assertEquals(kerberosRule, rules);
+ LOG.info(rules);
+ }
+
+ @Test
+ public void testValidKerberosName() throws Throwable {
+
+ new HadoopKerberosName(ZOOKEEPER).getShortName();
+ new HadoopKerberosName(ZOOKEEPER_LOCALHOST).getShortName();
+ new HadoopKerberosName(ZOOKEEPER_REALM).getShortName();
+ // standard rules don't pick this up
+ // new HadoopKerberosName(ZOOKEEPER_LOCALHOST_REALM).getShortName();
+ }
+
+
+ @Test
+ public void testUGILogin() throws Throwable {
+
+ UserGroupInformation ugi = loginUGI(ZOOKEEPER, keytab_zk);
+ RegistrySecurity.UgiInfo ugiInfo =
+ new RegistrySecurity.UgiInfo(ugi);
+ LOG.info("logged in as: {}", ugiInfo);
+ assertTrue("security is not enabled: " + ugiInfo,
+ UserGroupInformation.isSecurityEnabled());
+ assertTrue("login is keytab based: " + ugiInfo,
+ ugi.isFromKeytab());
+
+ // now we are here, build a SASL ACL
+ ACL acl = ugi.doAs(new PrivilegedExceptionAction<ACL>() {
+ @Override
+ public ACL run() throws Exception {
+ return registrySecurity.createSaslACLFromCurrentUser(0);
+ }
+ });
+ assertEquals(ZOOKEEPER_REALM, acl.getId().getId());
+ assertEquals(ZookeeperConfigOptions.SCHEME_SASL, acl.getId().getScheme());
+ registrySecurity.addSystemACL(acl);
+
+ }
+
+}