You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/07/25 18:03:12 UTC
[30/50] [abbrv] hadoop git commit: YARN-6335. Port slider's groovy
unit tests to yarn native services. Contributed by Billie Rinaldi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java
index 59ab30b..6df4bf4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java
@@ -29,7 +29,7 @@ public enum ContainerOutcome {
Completed,
Failed,
Failed_limits_exceeded,
- Node_failure,
+ Disk_failure,
Preempted;
/**
@@ -48,7 +48,7 @@ public enum ContainerOutcome {
// could either be a release or node failure. Treat as completion
return Completed;
case ContainerExitStatus.DISKS_FAILED:
- return Node_failure;
+ return Disk_failure;
case ContainerExitStatus.PREEMPTED:
return Preempted;
case ContainerExitStatus.KILLED_EXCEEDED_PMEM:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
index eb8ff03..d57b6d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
@@ -222,7 +222,7 @@ public class NodeEntry implements Cloneable {
// general "any reason" app failure
case Failed:
// specific node failure
- case Node_failure:
+ case Disk_failure:
++failed;
++failedRecently;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index 8e8546b..5051aee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -21,7 +21,6 @@ package org.apache.slider.server.appmaster.state;
import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricSet;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.RoleStatistics;
@@ -35,8 +34,6 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
-import static org.apache.hadoop.metrics2.lib.Interns.info;
-
/**
* Models the ongoing status of all nodes in an application.
*
@@ -207,12 +204,16 @@ public final class RoleStatus implements MetricSet {
return componentMetrics.containersDesired.value();
}
- long getRunning() {
+ public void setDesired(int desired) {
+ componentMetrics.containersDesired.set(desired);
+ }
+
+ public long getRunning() {
return componentMetrics.containersRunning.value();
}
- public long getPending() {
- return componentMetrics.containersPending.value();
+ public long getRequested() {
+ return componentMetrics.containersRequested.value();
}
public long getAAPending() {
@@ -222,22 +223,35 @@ public final class RoleStatus implements MetricSet {
void decAAPending() {
componentMetrics.pendingAAContainers.decr();
}
+
void setAAPending(long n) {
componentMetrics.pendingAAContainers.set((int)n);
}
- long getFailedRecently() {
+ public long getLimitsExceeded() {
+ return componentMetrics.containersLimitsExceeded.value();
+ }
+
+ public long getPreempted() {
+ return componentMetrics.containersPreempted.value();
+ }
+
+ public long getDiskFailed() {
+ return componentMetrics.containersDiskFailure.value();
+ }
+
+ public long getFailedRecently() {
return componentMetrics.failedSinceLastThreshold.value();
}
- long resetFailedRecently() {
+ public long resetFailedRecently() {
long count =
componentMetrics.failedSinceLastThreshold.value();
componentMetrics.failedSinceLastThreshold.set(0);
return count;
}
- long getFailed() {
+ public long getFailed() {
return componentMetrics.containersFailed.value();
}
@@ -254,6 +268,8 @@ public final class RoleStatus implements MetricSet {
long inuse = getActualAndRequested();
long delta = getDesired() - inuse;
if (delta < 0) {
+ // TODO this doesn't do anything now that we're not tracking releasing
+ // containers -- maybe we need releasing
//if we are releasing, remove the number that are already released.
//but never switch to a positive
delta = Math.min(delta, 0);
@@ -262,11 +278,11 @@ public final class RoleStatus implements MetricSet {
}
/**
- * Get count of actual and requested containers. This includes pending ones
+ * Get count of actual and requested containers.
* @return the size of the application when outstanding requests are included.
*/
public long getActualAndRequested() {
- return getRunning() + getPending();
+ return getRunning() + getRequested();
}
/**
@@ -341,6 +357,14 @@ public final class RoleStatus implements MetricSet {
public synchronized RoleStatistics getStatistics() {
RoleStatistics stats = new RoleStatistics();
stats.activeAA = getOutstandingAARequestCount();
+ stats.actual = getRunning();
+ stats.desired = getDesired();
+ stats.failed = getFailed();
+ stats.limitsExceeded = getLimitsExceeded();
+ stats.nodeFailed = getDiskFailed();
+ stats.preempted = getPreempted();
+ stats.requested = getRequested();
+ stats.started = getRunning();
return stats;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
index 440094e..8dca4ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
@@ -156,7 +156,7 @@ public class IndexBlock extends SliderHamletBlock {
} else {
aatext = "";
}
- if (status.getPending() > 0) {
+ if (status.getRequested() > 0) {
roleWithOpenRequest ++;
}
}
@@ -165,7 +165,7 @@ public class IndexBlock extends SliderHamletBlock {
.td().a(nameUrl, roleName)._()
.td(String.format("%d", metrics.containersDesired.value()))
.td(String.format("%d", metrics.containersRunning.value()))
- .td(String.format("%d", metrics.containersPending.value()))
+ .td(String.format("%d", metrics.containersRequested.value()))
.td(String.format("%d", metrics.containersFailed.value()))
.td(aatext)
._();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
index bfcab23..691f861 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
@@ -80,12 +80,16 @@ message UpgradeContainersRequestProto {
message UpgradeContainersResponseProto {
}
-message FlexComponentRequestProto {
+message FlexComponentsRequestProto {
+ repeated ComponentCountProto components = 1;
+}
+
+message ComponentCountProto {
optional string name = 1;
- optional int32 numberOfContainers = 2;
+ optional int64 numberOfContainers = 2;
}
-message FlexComponentResponseProto {
+message FlexComponentsResponseProto {
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto
index f52d7a1..776ce28 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto
@@ -61,7 +61,7 @@ service SliderClusterProtocolPB {
rpc upgradeContainers(UpgradeContainersRequestProto)
returns(UpgradeContainersResponseProto);
- rpc flexComponent(FlexComponentRequestProto) returns (FlexComponentResponseProto);
+ rpc flexComponents(FlexComponentsRequestProto) returns (FlexComponentsResponseProto);
/**
* Get the current cluster status
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/api/TestRPCBinding.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/api/TestRPCBinding.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/api/TestRPCBinding.java
new file mode 100644
index 0000000..28483dc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/api/TestRPCBinding.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.slider.server.appmaster.rpc.RpcBinder;
+import org.apache.slider.server.appmaster.rpc.SliderClusterProtocolPB;
+import org.junit.Test;
+
+import java.net.InetSocketAddress;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests RPC work.
+ */
+public class TestRPCBinding {
+
+ @Test
+ public void testRegistration() throws Throwable {
+ Configuration conf = new Configuration();
+ RpcBinder.registerSliderAPI(conf);
+ assertTrue(RpcBinder.verifyBondedToProtobuf(conf,
+ SliderClusterProtocolPB.class));
+ }
+
+ @Test
+ public void testGetProxy() throws Throwable {
+ Configuration conf = new Configuration();
+ InetSocketAddress saddr = new InetSocketAddress("127.0.0.1", 9000);
+ SliderClusterProtocol proxy =
+ RpcBinder.connectToServer(saddr, null, conf, 1000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java
new file mode 100644
index 0000000..6299a9c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.slider.common.params.Arguments;
+import org.apache.slider.common.params.SliderActions;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.slider.core.exceptions.UsageException;
+import org.apache.slider.utils.SliderTestBase;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+
+/**
+ * Test the argument parsing/validation logic.
+ */
+public class TestClientBadArgs extends SliderTestBase {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestClientBadArgs.class);
+
+ @Test
+ public void testNoAction() throws Throwable {
+ launchExpectingException(SliderClient.class,
+ createTestConfig(),
+ "Usage: slider COMMAND",
+ EMPTY_LIST);
+
+ }
+
+ @Test
+ public void testUnknownAction() throws Throwable {
+ launchExpectingException(SliderClient.class,
+ createTestConfig(),
+ "not-a-known-action",
+ Arrays.asList("not-a-known-action"));
+ }
+
+ @Test
+ public void testActionWithoutOptions() throws Throwable {
+ launchExpectingException(SliderClient.class,
+ createTestConfig(),
+ "Usage: slider build <application>",
+ Arrays.asList(SliderActions.ACTION_BUILD));
+ }
+
+ @Test
+ public void testActionWithoutEnoughArgs() throws Throwable {
+ launchExpectingException(SliderClient.class,
+ createTestConfig(),
+ ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS,
+ Arrays.asList(SliderActions.ACTION_START));
+ }
+
+ @Test
+ public void testActionWithTooManyArgs() throws Throwable {
+ launchExpectingException(SliderClient.class,
+ createTestConfig(),
+ ErrorStrings.ERROR_TOO_MANY_ARGUMENTS,
+ Arrays.asList(SliderActions.ACTION_HELP,
+ "hello, world"));
+ }
+
+ @Test
+ public void testBadImageArg() throws Throwable {
+ launchExpectingException(SliderClient.class,
+ createTestConfig(),
+ "Unknown option: --image",
+ Arrays.asList(SliderActions.ACTION_HELP,
+ Arguments.ARG_IMAGE));
+ }
+
+ @Test
+ public void testRegistryUsage() throws Throwable {
+ Throwable exception = launchExpectingException(SliderClient.class,
+ createTestConfig(),
+ "org.apache.slider.core.exceptions.UsageException: Argument --name " +
+ "missing",
+ Arrays.asList(SliderActions.ACTION_REGISTRY));
+ assertTrue(exception instanceof UsageException);
+ LOG.info(exception.toString());
+ }
+
+ @Test
+ public void testRegistryExportBadUsage1() throws Throwable {
+ Throwable exception = launchExpectingException(SliderClient.class,
+ createTestConfig(),
+ "Expected a value after parameter --getexp",
+ Arrays.asList(SliderActions.ACTION_REGISTRY,
+ Arguments.ARG_NAME,
+ "cl1",
+ Arguments.ARG_GETEXP));
+ assertTrue(exception instanceof BadCommandArgumentsException);
+ LOG.info(exception.toString());
+ }
+
+ @Test
+ public void testRegistryExportBadUsage2() throws Throwable {
+ Throwable exception = launchExpectingException(SliderClient.class,
+ createTestConfig(),
+ "Expected a value after parameter --getexp",
+ Arrays.asList(SliderActions.ACTION_REGISTRY,
+ Arguments.ARG_NAME,
+ "cl1",
+ Arguments.ARG_LISTEXP,
+ Arguments.ARG_GETEXP));
+ assertTrue(exception instanceof BadCommandArgumentsException);
+ LOG.info(exception.toString());
+ }
+
+ @Test
+ public void testRegistryExportBadUsage3() throws Throwable {
+ Throwable exception = launchExpectingException(SliderClient.class,
+ createTestConfig(),
+ "Usage: registry",
+ Arrays.asList(SliderActions.ACTION_REGISTRY,
+ Arguments.ARG_NAME,
+ "cl1",
+ Arguments.ARG_LISTEXP,
+ Arguments.ARG_GETEXP,
+ "export1"));
+ assertTrue(exception instanceof UsageException);
+ LOG.info(exception.toString());
+ }
+
+ @Test
+ public void testUpgradeUsage() throws Throwable {
+ Throwable exception = launchExpectingException(SliderClient.class,
+ createTestConfig(),
+ "org.apache.slider.core.exceptions.BadCommandArgumentsException: Not " +
+ "enough arguments for action: upgrade Expected minimum 1 but got 0",
+ Arrays.asList(SliderActions.ACTION_UPGRADE));
+ assertTrue(exception instanceof BadCommandArgumentsException);
+ LOG.info(exception.toString());
+ }
+
+ public Configuration createTestConfig() {
+ Configuration configuration = new Configuration();
+ configuration.set(YarnConfiguration.RM_ADDRESS, "127.0.0.1:8032");
+ return configuration;
+ }
+
+ @Ignore
+ @Test
+ public void testUpgradeWithTemplateResourcesAndContainersOption() throws
+ Throwable {
+ //TODO test upgrade args
+ String appName = "test_hbase";
+ Throwable exception = launchExpectingException(SliderClient.class,
+ createTestConfig(),
+ "BadCommandArgumentsException: Option --containers cannot be "
+ + "specified with --appdef",
+ Arrays.asList(SliderActions.ACTION_UPGRADE,
+ appName,
+ Arguments.ARG_APPDEF,
+ "/tmp/app.json",
+ Arguments.ARG_CONTAINERS,
+ "container_1"
+ ));
+ assertTrue(exception instanceof BadCommandArgumentsException);
+ LOG.info(exception.toString());
+ }
+
+ @Ignore
+ @Test
+ public void testUpgradeWithTemplateResourcesAndComponentsOption() throws
+ Throwable {
+ //TODO test upgrade args
+ String appName = "test_hbase";
+ Throwable exception = launchExpectingException(SliderClient.class,
+ createTestConfig(),
+ "BadCommandArgumentsException: Option --components cannot be "
+ + "specified with --appdef",
+ Arrays.asList(SliderActions.ACTION_UPGRADE,
+ appName,
+ Arguments.ARG_APPDEF,
+ "/tmp/app.json",
+ Arguments.ARG_COMPONENTS,
+ "HBASE_MASTER"
+ ));
+ assertTrue(exception instanceof BadCommandArgumentsException);
+ LOG.info(exception.toString());
+ }
+
+ @Test
+ public void testNodesMissingFile() throws Throwable {
+ Throwable exception = launchExpectingException(SliderClient.class,
+ createTestConfig(),
+ "after parameter --out",
+ Arrays.asList(SliderActions.ACTION_NODES, Arguments.ARG_OUTPUT));
+ assertTrue(exception instanceof BadCommandArgumentsException);
+ }
+
+ @Test
+ public void testFlexWithNoComponents() throws Throwable {
+ Throwable exception = launchExpectingException(SliderClient.class,
+ new Configuration(),
+ "Usage: slider flex <application>",
+ Arrays.asList(
+ SliderActions.ACTION_FLEX,
+ "flex1",
+ Arguments.ARG_DEFINE,
+ YarnConfiguration.RM_ADDRESS + "=127.0.0.1:8032"
+ ));
+ assertTrue(exception instanceof UsageException);
+ LOG.info(exception.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBasicArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBasicArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBasicArgs.java
new file mode 100644
index 0000000..43c5163
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBasicArgs.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client;
+
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.slider.common.params.Arguments;
+import org.apache.slider.common.params.ClientArgs;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.main.ServiceLauncher;
+import org.apache.slider.utils.SliderTestBase;
+import org.junit.Test;
+
+import java.net.UnknownHostException;
+import java.util.Arrays;
+
+/**
+ * Test bad argument handling.
+ */
+public class TestClientBasicArgs extends SliderTestBase {
+
+ /**
+ * Help should print out help string and then succeed.
+ * @throws Throwable
+ */
+ @Test
+ public void testHelp() throws Throwable {
+ ServiceLauncher launcher = launch(SliderClient.class,
+ SliderUtils.createConfiguration(),
+ Arrays.asList(ClientArgs.ACTION_HELP));
+ assertEquals(0, launcher.getServiceExitCode());
+ }
+
+ @Test
+ public void testNoArgs() throws Throwable {
+ launchExpectingException(SliderClient.class,
+ SliderUtils.createConfiguration(),
+ "Usage: slider COMMAND",
+ EMPTY_LIST);
+ }
+
+ @Test
+ public void testListUnknownRM() throws Throwable {
+ try {
+ YarnConfiguration conf = SliderUtils.createConfiguration();
+ conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
+ 1000);
+ conf.setLong(YarnConfiguration
+ .RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 1000);
+ ServiceLauncher launcher = launch(SliderClient.class,
+ conf,
+ Arrays.asList(
+ ClientArgs.ACTION_LIST,
+ "cluster",
+ Arguments.ARG_MANAGER,
+ "badhost:8888"));
+ fail("expected an exception, got a launcher with exit code " +
+ launcher.getServiceExitCode());
+ } catch (UnknownHostException expected) {
+ //expected
+ }
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestCommonArgParsing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestCommonArgParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestCommonArgParsing.java
new file mode 100644
index 0000000..ec6dbb8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestCommonArgParsing.java
@@ -0,0 +1,522 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.RoleKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
+import org.apache.slider.common.params.ActionBuildArgs;
+import org.apache.slider.common.params.ActionCreateArgs;
+import org.apache.slider.common.params.ActionDestroyArgs;
+import org.apache.slider.common.params.ActionExistsArgs;
+import org.apache.slider.common.params.ActionFlexArgs;
+import org.apache.slider.common.params.ActionFreezeArgs;
+import org.apache.slider.common.params.ActionListArgs;
+import org.apache.slider.common.params.ActionStatusArgs;
+import org.apache.slider.common.params.ActionThawArgs;
+import org.apache.slider.common.params.ActionUpdateArgs;
+import org.apache.slider.common.params.ArgOps;
+import org.apache.slider.common.params.Arguments;
+import org.apache.slider.common.params.ClientArgs;
+import org.apache.slider.common.params.SliderActions;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.slider.core.exceptions.SliderException;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test handling of common arguments, specifically how things get split up.
+ */
+public class TestCommonArgParsing implements SliderActions, Arguments {
+
+
+ public static final String CLUSTERNAME = "clustername";
+
+ @Test
+ public void testCreateActionArgs() throws Throwable {
+ ClientArgs clientArgs = createClientArgs(Arrays.asList(ACTION_CREATE,
+ "cluster1"));
+ assertEquals("cluster1", clientArgs.getClusterName());
+ }
+
+ @Test
+ public void testCreateFailsNoClustername() throws Throwable {
+ assertParseFails(Arrays.asList(ACTION_CREATE));
+ }
+
+ @Test
+ public void testCreateFailsTwoClusternames() throws Throwable {
+ assertParseFails(Arrays.asList(
+ ACTION_CREATE,
+ "c1",
+ "c2"
+ ));
+ }
+
+ @Test
+ public void testHelp() throws Throwable {
+ ClientArgs clientArgs = createClientArgs(Arrays.asList(ACTION_HELP));
+ assertNull(clientArgs.getClusterName());
+ }
+
+ @Test
+ public void testSliderBasePath() throws Throwable {
+ ClientArgs clientArgs = createClientArgs(Arrays.asList(ACTION_LIST,
+ ARG_BASE_PATH, "/projects/slider/clusters"));
+ assertEquals(new Path("/projects/slider/clusters"),
+ clientArgs.getBasePath());
+ }
+
+ @Test
+ public void testNoSliderBasePath() throws Throwable {
+ ClientArgs clientArgs = createClientArgs(Arrays.asList(ACTION_LIST));
+ assertNull(clientArgs.getBasePath());
+ }
+
+ @Test
+ public void testListNoClusternames() throws Throwable {
+ ClientArgs clientArgs = createClientArgs(Arrays.asList(ACTION_LIST));
+ assertNull(clientArgs.getClusterName());
+ }
+
+ @Test
+ public void testListNoClusternamesDefinition() throws Throwable {
+ ClientArgs clientArgs = createClientArgs(Arrays.asList(
+ ACTION_LIST,
+ ARG_DEFINE,
+ "fs.default.FS=file://localhost"
+ ));
+ assertNull(clientArgs.getClusterName());
+ }
+
+ @Test
+ public void testList1Clustername() throws Throwable {
+ ClientArgs ca = createClientArgs(Arrays.asList(ACTION_LIST, "cluster1"));
+ assertEquals("cluster1", ca.getClusterName());
+ assertTrue(ca.getCoreAction() instanceof ActionListArgs);
+ }
+
+ @Test
+ public void testListFailsTwoClusternames() throws Throwable {
+ assertParseFails(Arrays.asList(
+ ACTION_LIST,
+ "c1",
+ "c2"
+ ));
+ }
+
+ @Test
+ public void testDefinitions() throws Throwable {
+ ClientArgs ca = createClientArgs(Arrays.asList(
+ ACTION_CREATE,
+ CLUSTERNAME,
+ "-D", "yarn.resourcemanager.principal=yarn/server@LOCAL",
+ "-D", "dfs.datanode.kerberos.principal=hdfs/server@LOCAL"
+ ));
+ Configuration conf = new Configuration(false);
+ ca.applyDefinitions(conf);
+ assertEquals(CLUSTERNAME, ca.getClusterName());
+ assertNull(conf.get(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH));
+ SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
+ SliderUtils.verifyPrincipalSet(
+ conf,
+ SliderXmlConfKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY);
+
+ }
+
+ @Test
+ public void testDefinitionsSettingBaseSliderDir() throws Throwable {
+ ClientArgs ca = createClientArgs(Arrays.asList(
+ ACTION_CREATE,
+ CLUSTERNAME,
+ "--basepath", "/projects/slider/clusters",
+ "-D", "yarn.resourcemanager.principal=yarn/server@LOCAL",
+ "-D", "dfs.datanode.kerberos.principal=hdfs/server@LOCAL"
+ ));
+ Configuration conf = new Configuration(false);
+ ca.applyDefinitions(conf);
+ assertEquals(CLUSTERNAME, ca.getClusterName());
+ assertEquals("/projects/slider/clusters", conf.get(SliderXmlConfKeys
+ .KEY_SLIDER_BASE_PATH));
+ SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
+ SliderUtils.verifyPrincipalSet(conf, SliderXmlConfKeys
+ .DFS_DATANODE_KERBEROS_PRINCIPAL_KEY);
+
+ }
+
+ /**
+ * Test a start command.
+ * @throws Throwable
+ */
+ @Test
+ public void testComplexThaw() throws Throwable {
+ ClientArgs ca = createClientArgs(Arrays.asList(
+ ACTION_START,
+ "--manager", "rhel:8032",
+ "--filesystem", "hdfs://rhel:9090",
+ "-S", "java.security.krb5.realm=LOCAL",
+ "-S", "java.security.krb5.kdc=rhel",
+ "-D", "yarn.resourcemanager.principal=yarn/rhel@LOCAL",
+ "-D", "namenode.resourcemanager.principal=hdfs/rhel@LOCAL",
+ "cl1"
+ ));
+ assertEquals("cl1", ca.getClusterName());
+ assertTrue(ca.getCoreAction() instanceof ActionThawArgs);
+ }
+
+ /**
+ * Test a force kill command where the app comes at the end of the line.
+ * @throws Throwable
+ *
+ */
+ @Test
+ public void testStatusSplit() throws Throwable {
+
+ String appId = "application_1381252124398_0013";
+ ClientArgs ca = createClientArgs(Arrays.asList(
+ ACTION_STATUS,
+ "--manager", "rhel:8032",
+ "--filesystem", "hdfs://rhel:9090",
+ "-S", "java.security.krb5.realm=LOCAL",
+ "-S", "java.security.krb5.kdc=rhel",
+ "-D", "yarn.resourcemanager.principal=yarn/rhel@LOCAL",
+ "-D", "namenode.resourcemanager.principal=hdfs/rhel@LOCAL",
+ appId
+ ));
+ assertEquals(appId, ca.getClusterName());
+ }
+
+ @Test
+ public void testFreezeFailsNoArg() throws Throwable {
+ assertParseFails(Arrays.asList(
+ ACTION_STOP
+ ));
+ }
+
+ @Test
+ public void testFreezeWorks1Arg() throws Throwable {
+ ClientArgs ca = createClientArgs(Arrays.asList(
+ ACTION_STOP,
+ CLUSTERNAME
+ ));
+ assertEquals(CLUSTERNAME, ca.getClusterName());
+ assertTrue(ca.getCoreAction() instanceof ActionFreezeArgs);
+ }
+
+ @Test
+ public void testFreezeFails2Arg() throws Throwable {
+ assertParseFails(Arrays.asList(
+ ACTION_STOP, "cluster", "cluster2"
+ ));
+ }
+
+ @Test
+ public void testFreezeForceWaitAndMessage() throws Throwable {
+ ClientArgs ca = createClientArgs(Arrays.asList(
+ ACTION_STOP, CLUSTERNAME,
+ ARG_FORCE,
+ ARG_WAIT, "0",
+ ARG_MESSAGE, "explanation"
+ ));
+ assertEquals(CLUSTERNAME, ca.getClusterName());
+ assertTrue(ca.getCoreAction() instanceof ActionFreezeArgs);
+ ActionFreezeArgs freezeArgs = (ActionFreezeArgs) ca.getCoreAction();
+ assertEquals("explanation", freezeArgs.message);
+ assertTrue(freezeArgs.force);
+ }
+
+ @Test
+ public void testGetStatusWorks1Arg() throws Throwable {
+ ClientArgs ca = createClientArgs(Arrays.asList(
+ ACTION_STATUS,
+ CLUSTERNAME
+ ));
+ assertEquals(CLUSTERNAME, ca.getClusterName());
+ assertTrue(ca.getCoreAction() instanceof ActionStatusArgs);
+ }
+
+ @Test
+ public void testExistsWorks1Arg() throws Throwable {
+ ClientArgs ca = createClientArgs(Arrays.asList(
+ ACTION_EXISTS,
+ CLUSTERNAME,
+ ARG_LIVE
+ ));
+ assertEquals(CLUSTERNAME, ca.getClusterName());
+ assertTrue(ca.getCoreAction() instanceof ActionExistsArgs);
+ assertTrue(ca.getActionExistsArgs().live);
+ }
+
+ @Test
+ public void testDestroy1Arg() throws Throwable {
+ ClientArgs ca = createClientArgs(Arrays.asList(
+ ACTION_DESTROY,
+ CLUSTERNAME
+ ));
+ assertEquals(CLUSTERNAME, ca.getClusterName());
+ assertTrue(ca.getCoreAction() instanceof ActionDestroyArgs);
+ }
+
+ /**
+ * Assert that a pass fails with a BadCommandArgumentsException.
+ * @param argsList
+ */
+
+ private void assertParseFails(List argsList) throws SliderException {
+ try {
+ ClientArgs clientArgs = createClientArgs(argsList);
+ Assert.fail("exected an exception, got " + clientArgs);
+ } catch (BadCommandArgumentsException ignored) {
+ //expected
+ }
+ }
+
+ /**
+ * Build and parse client args, after adding the base args list.
+ * @param argsList
+ */
+ public ClientArgs createClientArgs(List<String> argsList)
+ throws SliderException {
+ ClientArgs serviceArgs = new ClientArgs(argsList);
+ serviceArgs.parse();
+ return serviceArgs;
+ }
+
+ public ActionCreateArgs createAction(List<String> argsList)
+ throws SliderException {
+ ClientArgs ca = createClientArgs(argsList);
+ assertEquals(ACTION_CREATE, ca.getAction());
+ ActionCreateArgs args = ca.getActionCreateArgs();
+ assertNotNull(args);
+ return args;
+ }
+
+ @Test
+ public void testSingleRoleArg() throws Throwable {
+ ActionCreateArgs createArgs = createAction(Arrays.asList(
+ ACTION_CREATE, "cluster1",
+ ARG_COMPONENT, "master", "5"
+ ));
+ List<String> tuples = createArgs.getComponentTuples();
+ assertEquals(2, tuples.size());
+ Map<String, String> roleMap = ArgOps.convertTupleListToMap("roles", tuples);
+ assertEquals("5", roleMap.get("master"));
+ }
+
+ @Test
+ public void testNoRoleArg() throws Throwable {
+ ActionCreateArgs createArgs = createAction(Arrays.asList(
+ ACTION_CREATE, "cluster1"
+ ));
+ List<String> tuples = createArgs.getComponentTuples();
+ Map<String, String> roleMap = ArgOps.convertTupleListToMap("roles", tuples);
+ assertNull(roleMap.get("master"));
+ }
+
+
+ @Test
+ public void testMultiRoleArgBuild() throws Throwable {
+ ClientArgs ca = createClientArgs(Arrays.asList(
+ ACTION_BUILD, "cluster1",
+ ARG_COMPONENT, "master", "1",
+ ARG_COMPONENT, "worker", "2"
+ ));
+ assertEquals(ACTION_BUILD, ca.getAction());
+ assertTrue(ca.getCoreAction() instanceof ActionBuildArgs);
+ assertTrue(ca.getBuildingActionArgs() instanceof ActionBuildArgs);
+ AbstractClusterBuildingActionArgs args = ca.getActionBuildArgs();
+ List<String> tuples = args.getComponentTuples();
+ assertEquals(4, tuples.size());
+ Map<String, String> roleMap = ArgOps.convertTupleListToMap("roles", tuples);
+ assertEquals("1", roleMap.get("master"));
+ assertEquals("2", roleMap.get("worker"));
+ }
+
+ @Test
+ public void testArgUpdate() throws Throwable {
+ ClientArgs ca = createClientArgs(Arrays.asList(
+ ACTION_UPDATE, "cluster1",
+ ARG_APPDEF, "app.json"
+ ));
+ assertEquals(ACTION_UPDATE, ca.getAction());
+ assertTrue(ca.getCoreAction() instanceof ActionUpdateArgs);
+ assertTrue(ca.getActionUpdateArgs() instanceof ActionUpdateArgs);
+ AbstractClusterBuildingActionArgs args = ca.getActionUpdateArgs();
+ assertNotNull(args.appDef);
+ }
+
+ @Test
+ public void testFlexArgs() throws Throwable {
+ ClientArgs ca = createClientArgs(Arrays.asList(
+ ACTION_FLEX, "cluster1",
+ ARG_COMPONENT, "master", "1",
+ ARG_COMPONENT, "worker", "2"
+ ));
+ assertTrue(ca.getCoreAction() instanceof ActionFlexArgs);
+ List<String> tuples = ca.getActionFlexArgs().getComponentTuples();
+ assertEquals(4, tuples.size());
+ Map<String, String> roleMap = ArgOps.convertTupleListToMap("roles", tuples);
+ assertEquals("1", roleMap.get("master"));
+ assertEquals("2", roleMap.get("worker"));
+ }
+
+ @Test
+ public void testDuplicateRole() throws Throwable {
+ ActionCreateArgs createArgs = createAction(Arrays.asList(
+ ACTION_CREATE, "cluster1",
+ ARG_COMPONENT, "master", "1",
+ ARG_COMPONENT, "master", "2"
+ ));
+ List<String> tuples = createArgs.getComponentTuples();
+ assertEquals(4, tuples.size());
+ try {
+ Map<String, String> roleMap = ArgOps.convertTupleListToMap(
+ "roles",
+ tuples);
+ Assert.fail("got a role map $roleMap not a failure");
+ } catch (BadCommandArgumentsException expected) {
+ assertTrue(expected.getMessage().contains(ErrorStrings
+ .ERROR_DUPLICATE_ENTRY));
+ }
+ }
+
+ @Test
+ public void testOddRoleCount() throws Throwable {
+ ActionCreateArgs createArgs = createAction(Arrays.asList(
+ ACTION_CREATE, "cluster1",
+ ARG_COMPONENT, "master", "1",
+ ARG_COMPONENT, "master", "2"
+ ));
+ List<String> tuples = createArgs.getComponentTuples();
+ tuples.add("loggers");
+ assertEquals(5, tuples.size());
+ try {
+ Map<String, String> roleMap = ArgOps.convertTupleListToMap("roles",
+ tuples);
+ Assert.fail("got a role map " + roleMap + " not a failure");
+ } catch (BadCommandArgumentsException expected) {
+ assertTrue(expected.getMessage().contains(ErrorStrings
+ .ERROR_PARSE_FAILURE));
+ }
+ }
+
+ /**
+ * Create some role-opt client args, so that multiple tests can use it.
+ * @return the args
+ */
+ public ActionCreateArgs createRoleOptClientArgs() throws SliderException {
+ ActionCreateArgs createArgs = createAction(Arrays.asList(
+ ACTION_CREATE, "cluster1",
+ ARG_COMPONENT, "master", "1",
+ ARG_COMP_OPT, "master", "cheese", "swiss",
+ ARG_COMP_OPT, "master", "env.CHEESE", "cheddar",
+ ARG_COMP_OPT, "master", ResourceKeys.YARN_CORES, "3",
+
+ ARG_COMPONENT, "worker", "2",
+ ARG_COMP_OPT, "worker", ResourceKeys.YARN_CORES, "2",
+ ARG_COMP_OPT, "worker", RoleKeys.JVM_HEAP, "65536",
+ ARG_COMP_OPT, "worker", "env.CHEESE", "stilton"
+ ));
+ return createArgs;
+ }
+
+ @Test
+ public void testRoleOptionParse() throws Throwable {
+ ActionCreateArgs createArgs = createRoleOptClientArgs();
+ Map<String, Map<String, String>> tripleMaps = createArgs.getCompOptionMap();
+ Map<String, String> workerOpts = tripleMaps.get("worker");
+ assertEquals(3, workerOpts.size());
+ assertEquals("2", workerOpts.get(ResourceKeys.YARN_CORES));
+ assertEquals("65536", workerOpts.get(RoleKeys.JVM_HEAP));
+
+ Map<String, String> masterOpts = tripleMaps.get("master");
+ assertEquals(3, masterOpts.size());
+ assertEquals("3", masterOpts.get(ResourceKeys.YARN_CORES));
+
+ }
+
+ @Test
+ public void testRoleOptionsMerge() throws Throwable {
+ ActionCreateArgs createArgs = createRoleOptClientArgs();
+
+ Map<String, Map<String, String>> roleOpts = createArgs.getCompOptionMap();
+
+ Map<String, Map<String, String>> clusterRoleMap = createEnvMap();
+ SliderUtils.applyCommandLineRoleOptsToRoleMap(clusterRoleMap, roleOpts);
+
+ Map<String, String> masterOpts = clusterRoleMap.get("master");
+ assertEquals("swiss", masterOpts.get("cheese"));
+
+ Map<String, String> workerOpts = clusterRoleMap.get("worker");
+ assertEquals("stilton", workerOpts.get("env.CHEESE"));
+ }
+
+ @Test
+ public void testEnvVariableApply() throws Throwable {
+ ActionCreateArgs createArgs = createRoleOptClientArgs();
+
+
+ Map<String, Map<String, String>> roleOpts = createArgs.getCompOptionMap();
+
+ Map<String, Map<String, String>> clusterRoleMap = createEnvMap();
+ SliderUtils.applyCommandLineRoleOptsToRoleMap(clusterRoleMap, roleOpts);
+
+ Map<String, String> workerOpts = clusterRoleMap.get("worker");
+ assertEquals("stilton", workerOpts.get("env.CHEESE"));
+
+ Map<String, String> envmap = SliderUtils.buildEnvMap(workerOpts);
+ assertEquals("stilton", envmap.get("CHEESE"));
+
+ }
+
+ /**
+ * Static compiler complaining about matching LinkedHashMap with Map,
+ * so some explicit creation here.
+ * @return a map of maps
+ */
+ public Map<String, Map<String, String>> createEnvMap() {
+
+ Map<String, String> cheese = new HashMap<>();
+ cheese.put("cheese", "french");
+ Map<String, String> envCheese = new HashMap<>();
+ envCheese.put("env.CHEESE", "french");
+ Map<String, Map<String, String>> envMap = new HashMap<>();
+ envMap.put("master", cheese);
+ envMap.put("worker", envCheese);
+ return envMap;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java
new file mode 100644
index 0000000..07d8c10
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.log4j.AppenderSkeleton;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.LoggingEvent;
+import org.apache.slider.common.params.Arguments;
+import org.apache.slider.common.params.ClientArgs;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.main.ServiceLauncher;
+import org.apache.slider.utils.SliderTestBase;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.UUID;
+
+/**
+ * Test a keytab installation.
+ */
+public class TestKeytabCommandOptions extends SliderTestBase {
+
+ private static SliderFileSystem testFileSystem;
+
+ @Before
+ public void setupFilesystem() throws IOException {
+ org.apache.hadoop.fs.FileSystem fileSystem = new RawLocalFileSystem();
+ YarnConfiguration configuration = SliderUtils.createConfiguration();
+ fileSystem.setConf(configuration);
+ testFileSystem = new SliderFileSystem(fileSystem, configuration);
+ File testFolderDir = new File(testFileSystem
+ .buildKeytabInstallationDirPath("").toUri().getPath());
+ FileUtils.deleteDirectory(testFolderDir);
+ }
+
+ @Test
+ public void testInstallKeytab() throws Throwable {
+ // create a mock keytab file
+ File localKeytab =
+ FileUtil.createLocalTempFile(getTempLocation(), "test", true);
+ String contents = UUID.randomUUID().toString();
+ FileUtils.write(localKeytab, contents);
+ YarnConfiguration conf = SliderUtils.createConfiguration();
+ ServiceLauncher launcher = launch(TestSliderClient.class,
+ conf,
+ Arrays.asList(
+ ClientArgs.ACTION_KEYTAB,
+ ClientArgs.ARG_KEYTABINSTALL,
+ ClientArgs.ARG_KEYTAB,
+ localKeytab.getAbsolutePath(),
+ Arguments.ARG_FOLDER,
+ "testFolder"));
+ Path installedPath = new Path(testFileSystem
+ .buildKeytabInstallationDirPath("testFolder"), localKeytab.getName());
+ File installedKeytab = new File(installedPath.toUri().getPath());
+ assertTrue(installedKeytab.exists());
+ assertEquals(FileUtils.readFileToString(installedKeytab),
+ FileUtils.readFileToString(localKeytab));
+ }
+
+ @Test
+ public void testInstallThenDeleteKeytab() throws Throwable {
+ // create a mock keytab file
+ File localKeytab =
+ FileUtil.createLocalTempFile(getTempLocation(), "test", true);
+ String contents = UUID.randomUUID().toString();
+ FileUtils.write(localKeytab, contents);
+ YarnConfiguration conf = SliderUtils.createConfiguration();
+ ServiceLauncher launcher = launch(TestSliderClient.class,
+ conf,
+ Arrays.asList(
+ ClientArgs.ACTION_KEYTAB,
+ ClientArgs.ARG_KEYTABINSTALL,
+ ClientArgs.ARG_KEYTAB,
+ localKeytab.getAbsolutePath(),
+ Arguments.ARG_FOLDER,
+ "testFolder"));
+ Path installedPath = new Path(testFileSystem
+ .buildKeytabInstallationDirPath("testFolder"), localKeytab.getName());
+ File installedKeytab = new File(installedPath.toUri().getPath());
+ assertTrue(installedKeytab.exists());
+ assertEquals(FileUtils.readFileToString(installedKeytab),
+ FileUtils.readFileToString(localKeytab));
+
+ launcher = launch(TestSliderClient.class,
+ conf,
+ Arrays.asList(
+ ClientArgs.ACTION_KEYTAB,
+ ClientArgs.ARG_KEYTABDELETE,
+ ClientArgs.ARG_KEYTAB,
+ localKeytab.getName(),
+ Arguments.ARG_FOLDER,
+ "testFolder"));
+
+ assertFalse(installedKeytab.exists());
+
+ }
+
+ @Test
+ public void testInstallThenListKeytab() throws Throwable {
+ // create a mock keytab file
+ File localKeytab =
+ FileUtil.createLocalTempFile(getTempLocation(), "test", true);
+ String contents = UUID.randomUUID().toString();
+ FileUtils.write(localKeytab, contents);
+ YarnConfiguration conf = SliderUtils.createConfiguration();
+ ServiceLauncher launcher = launch(TestSliderClient.class,
+ conf,
+ Arrays.asList(
+ ClientArgs.ACTION_KEYTAB,
+ ClientArgs.ARG_KEYTABINSTALL,
+ ClientArgs.ARG_KEYTAB,
+ localKeytab.getAbsolutePath(),
+ Arguments.ARG_FOLDER,
+ "testFolder"));
+ Path installedPath = new Path(testFileSystem
+ .buildKeytabInstallationDirPath("testFolder"), localKeytab.getName());
+ File installedKeytab = new File(installedPath.toUri().getPath());
+ assertTrue(installedKeytab.exists());
+ assertEquals(FileUtils.readFileToString(installedKeytab),
+ FileUtils.readFileToString(localKeytab));
+
+ // install an additional copy into another folder to test listing
+ launcher = launch(TestSliderClient.class,
+ conf,
+ Arrays.asList(
+ ClientArgs.ACTION_KEYTAB,
+ ClientArgs.ARG_KEYTABINSTALL,
+ ClientArgs.ARG_KEYTAB,
+ localKeytab.getAbsolutePath(),
+ Arguments.ARG_FOLDER,
+ "testFolder2"));
+
+ TestAppender testAppender = new TestAppender();
+
+ Logger.getLogger(SliderClient.class).addAppender(testAppender);
+
+ try {
+ launcher = launch(TestSliderClient.class,
+ conf,
+ Arrays.asList(
+ ClientArgs.ACTION_KEYTAB,
+ ClientArgs.ARG_KEYTABLIST)
+ );
+ assertEquals(3, testAppender.events.size());
+ String msg = (String) testAppender.events.get(1).getMessage();
+ assertTrue(msg.contains("/.slider/keytabs/testFolder"));
+ assertTrue(msg.endsWith(installedKeytab.getName()));
+ msg = (String) testAppender.events.get(2).getMessage();
+ assertTrue(msg.contains("/.slider/keytabs/testFolder"));
+ assertTrue(msg.endsWith(installedKeytab.getName()));
+ } finally {
+ Logger.getLogger(SliderClient.class).removeAppender(testAppender);
+ }
+
+ // now listing while specifying the folder name
+ testAppender = new TestAppender();
+
+ Logger.getLogger(SliderClient.class).addAppender(testAppender);
+
+ try {
+ launcher = launch(TestSliderClient.class,
+ conf,
+ Arrays.asList(
+ ClientArgs.ACTION_KEYTAB,
+ ClientArgs.ARG_KEYTABLIST,
+ Arguments.ARG_FOLDER,
+ "testFolder"));
+ assertEquals(2, testAppender.events.size());
+ String msg = (String) testAppender.events.get(1).getMessage();
+ assertTrue(msg.contains("/.slider/keytabs/testFolder/" +
+ installedKeytab.getName()));
+ } finally {
+ Logger.getLogger(SliderClient.class).removeAppender(testAppender);
+ }
+ }
+
+ @Test
+ public void testDeleteNonExistentKeytab() throws Throwable {
+ // create a mock keytab file
+ YarnConfiguration conf = SliderUtils.createConfiguration();
+ try {
+ ServiceLauncher launcher = launch(TestSliderClient.class,
+ conf,
+ Arrays.asList(
+ ClientArgs.ACTION_KEYTAB,
+ ClientArgs.ARG_KEYTABDELETE,
+ ClientArgs.ARG_KEYTAB,
+ "HeyIDontExist.keytab",
+ Arguments.ARG_FOLDER,
+ "testFolder"));
+ fail("expected BadCommandArgumentsException from launch");
+ } catch (BadCommandArgumentsException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testInstallKeytabWithNoFolder() throws Throwable {
+ // create a mock keytab file
+ File localKeytab =
+ FileUtil.createLocalTempFile(getTempLocation(), "test", true);
+ String contents = UUID.randomUUID().toString();
+ FileUtils.write(localKeytab, contents);
+ YarnConfiguration conf = SliderUtils.createConfiguration();
+ try {
+ ServiceLauncher launcher = launch(TestSliderClient.class,
+ conf,
+ Arrays.asList(
+ ClientArgs.ACTION_KEYTAB,
+ ClientArgs.ARG_KEYTABINSTALL,
+ ClientArgs.ARG_KEYTAB,
+ localKeytab.getAbsolutePath()));
+ fail("expected BadCommandArgumentsException from launch");
+ } catch (BadCommandArgumentsException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testInstallKeytabWithNoKeytab() throws Throwable {
+ // create a mock keytab file
+ File localKeytab =
+ FileUtil.createLocalTempFile(getTempLocation(), "test", true);
+ String contents = UUID.randomUUID().toString();
+ FileUtils.write(localKeytab, contents);
+ YarnConfiguration conf = SliderUtils.createConfiguration();
+ try {
+ ServiceLauncher launcher = launch(TestSliderClient.class,
+ conf,
+ Arrays.asList(
+ ClientArgs.ACTION_KEYTAB,
+ ClientArgs.ARG_KEYTABINSTALL,
+ ClientArgs.ARG_FOLDER,
+ "testFolder"));
+ fail("expected BadCommandArgumentsException from launch");
+ } catch (BadCommandArgumentsException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testInstallKeytabAllowingOverwrite() throws Throwable {
+ // create a mock keytab file
+ File localKeytab =
+ FileUtil.createLocalTempFile(getTempLocation(), "test", true);
+ String contents = UUID.randomUUID().toString();
+ FileUtils.write(localKeytab, contents);
+ YarnConfiguration conf = SliderUtils.createConfiguration();
+ ServiceLauncher launcher = launch(TestSliderClient.class,
+ conf,
+ Arrays.asList(
+ ClientArgs.ACTION_KEYTAB,
+ ClientArgs.ARG_KEYTABINSTALL,
+ ClientArgs.ARG_KEYTAB,
+ localKeytab.getAbsolutePath(),
+ Arguments.ARG_FOLDER,
+ "testFolder"));
+ Path installedPath = new Path(testFileSystem
+ .buildKeytabInstallationDirPath("testFolder"), localKeytab.getName());
+ File installedKeytab = new File(installedPath.toUri().getPath());
+ assertTrue(installedKeytab.exists());
+ assertEquals(FileUtils.readFileToString(installedKeytab), FileUtils
+ .readFileToString(localKeytab));
+ launcher = launch(TestSliderClient.class,
+ conf,
+ Arrays.asList(
+ ClientArgs.ACTION_KEYTAB,
+ ClientArgs.ARG_KEYTABINSTALL,
+ ClientArgs.ARG_KEYTAB,
+ localKeytab.getAbsolutePath(),
+ Arguments.ARG_FOLDER,
+ "testFolder",
+ Arguments.ARG_OVERWRITE)
+ );
+ assertTrue(installedKeytab.exists());
+ assertEquals(FileUtils.readFileToString(installedKeytab),
+ FileUtils.readFileToString(localKeytab));
+ }
+
+ @Test
+ public void testInstallKeytabNotAllowingOverwrite() throws Throwable {
+ // create a mock keytab file
+ File localKeytab =
+ FileUtil.createLocalTempFile(getTempLocation(), "test", true);
+ String contents = UUID.randomUUID().toString();
+ FileUtils.write(localKeytab, contents);
+ YarnConfiguration conf = SliderUtils.createConfiguration();
+ ServiceLauncher launcher = launch(TestSliderClient.class,
+ conf,
+ Arrays.asList(
+ ClientArgs.ACTION_KEYTAB,
+ ClientArgs.ARG_KEYTABINSTALL,
+ ClientArgs.ARG_KEYTAB,
+ localKeytab.getAbsolutePath(),
+ Arguments.ARG_FOLDER,
+ "testFolder"));
+ Path installedPath = new Path(testFileSystem
+ .buildKeytabInstallationDirPath("testFolder"), localKeytab.getName());
+ File installedKeytab = new File(installedPath.toUri().getPath());
+ assertTrue(installedKeytab.exists());
+ assertEquals(FileUtils.readFileToString(installedKeytab),
+ FileUtils.readFileToString(localKeytab));
+ try {
+ launcher = launch(TestSliderClient.class,
+ conf,
+ Arrays.asList(
+ ClientArgs.ACTION_KEYTAB,
+ ClientArgs.ARG_KEYTABINSTALL,
+ ClientArgs.ARG_KEYTAB,
+ localKeytab.getAbsolutePath(),
+ Arguments.ARG_FOLDER,
+ "testFolder"));
+ fail("expected BadCommandArgumentsException from launch");
+ } catch (BadCommandArgumentsException e) {
+ // expected
+ }
+ }
+
+ @Test
+ public void testInstallKeytabWithMissingKeytab() throws Throwable {
+ // create a mock keytab file
+ YarnConfiguration conf = SliderUtils.createConfiguration();
+ try {
+ ServiceLauncher launcher = launch(TestSliderClient.class,
+ conf,
+ Arrays.asList(
+ ClientArgs.ACTION_KEYTAB,
+ ClientArgs.ARG_KEYTABINSTALL,
+ ClientArgs.ARG_KEYTAB,
+ "HeyIDontExist.keytab",
+ Arguments.ARG_FOLDER,
+ "testFolder"));
+ fail("expected BadCommandArgumentsException from launch");
+ } catch (BadCommandArgumentsException e) {
+ // expected
+ }
+ }
+
+ private File getTempLocation() {
+ return new File(System.getProperty("user.dir") + "/target");
+ }
+
+ /**
+ * Test SliderClient with overridden filesystem.
+ */
+ public static class TestSliderClient extends SliderClient {
+ public TestSliderClient() {
+ super();
+ }
+
+ @Override
+ protected void initHadoopBinding() throws IOException, SliderException {
+ sliderFileSystem = testFileSystem;
+ }
+
+ }
+
+ /**
+ * Appender that captures logging events.
+ */
+ public static class TestAppender extends AppenderSkeleton {
+ private List<LoggingEvent> events = new ArrayList<>();
+
+ public void close() {}
+
+ public boolean requiresLayout() {
+ return false;
+ }
+
+ @Override
+ protected void append(LoggingEvent event) {
+ events.add(event);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderClientMethods.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderClientMethods.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderClientMethods.java
new file mode 100644
index 0000000..32208ab
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderClientMethods.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client;
+
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.server.appmaster.model.mock.MockApplicationId;
+import org.apache.slider.utils.SliderTestBase;
+import org.easymock.EasyMock;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.api.easymock.PowerMock;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Test slider client methods.
+ */
+@RunWith(PowerMockRunner.class)
+@PrepareForTest(SliderUtils.class)
+public class TestSliderClientMethods extends SliderTestBase {
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(TestSliderClientMethods.class);
+
+ static final String AM_ENV = "LD_LIBRARY_PATH";
+ static final String PLACEHOLDER_KEY = "${distro.version}";
+ static final String PLACEHOLDER_SYSTEM_KEY = "DISTRO_VERSION";
+ static final String PLACEHOLDER_VALUE = "1.0.0";
+ static final String AM_ENV_2 = "PATH";
+ static final String PLACEHOLDER_KEY_2 = "${native.version}";
+ static final String PLACEHOLDER_SYSTEM_KEY_2 = "NATIVE_VERSION";
+ static final String PLACEHOLDER_VALUE_2 = "2.0.0";
+
+ @Test
+ public void testGeneratePlaceholderKeyValueMap() throws Throwable {
+ TestSliderClient testSliderClient = new TestSliderClient();
+
+ PowerMock.mockStatic(System.class);
+ EasyMock.expect(SliderUtils.getSystemEnv(PLACEHOLDER_SYSTEM_KEY))
+ .andReturn(PLACEHOLDER_VALUE).anyTimes();
+ PowerMock.replayAll();
+
+ Map<String, String> placeholders = testSliderClient
+ .generatePlaceholderKeyValueMap(AM_ENV + "=/usr/lib/" +
+ PLACEHOLDER_KEY);
+ Assert.assertTrue(placeholders.containsKey(PLACEHOLDER_KEY));
+ Assert.assertEquals("Should be equal", PLACEHOLDER_VALUE,
+ placeholders.get(PLACEHOLDER_KEY));
+
+ PowerMock.verifyAll();
+ LOG.info("Placeholders = {}", placeholders);
+ }
+
+ @Test
+ public void testSetAmLaunchEnv() throws Throwable {
+ TestSliderClient testSliderClient = new TestSliderClient();
+ YarnConfiguration conf = SliderUtils.createConfiguration();
+ conf.set(SliderXmlConfKeys.KEY_AM_LAUNCH_ENV, AM_ENV + "=/usr/lib/"
+ + PLACEHOLDER_KEY);
+
+ PowerMock.mockStatic(System.class);
+ EasyMock.expect(SliderUtils.getSystemEnv(PLACEHOLDER_SYSTEM_KEY))
+ .andReturn(PLACEHOLDER_VALUE);
+ PowerMock.replayAll();
+
+ Map<String, String> amLaunchEnv = testSliderClient.getAmLaunchEnv(conf);
+ Assert.assertNotNull(amLaunchEnv);
+ Assert.assertNotNull(amLaunchEnv.get(AM_ENV));
+ Assert.assertEquals("Should be equal", amLaunchEnv.get(AM_ENV),
+ (Shell.WINDOWS ? "%" + AM_ENV + "%;" : "$" + AM_ENV + ":") +
+ "/usr/lib/" + PLACEHOLDER_VALUE);
+
+ PowerMock.verifyAll();
+ LOG.info("amLaunchEnv = {}", amLaunchEnv);
+ }
+
+ @Test
+ public void testSetAmLaunchEnvMulti() throws Throwable {
+ TestSliderClient testSliderClient = new TestSliderClient();
+ YarnConfiguration conf = SliderUtils.createConfiguration();
+ conf.set(SliderXmlConfKeys.KEY_AM_LAUNCH_ENV, AM_ENV + "=/usr/lib/"
+ + PLACEHOLDER_KEY + "," + AM_ENV_2 + "=/usr/bin/" + PLACEHOLDER_KEY_2);
+
+ PowerMock.mockStatic(System.class);
+ EasyMock.expect(SliderUtils.getSystemEnv(PLACEHOLDER_SYSTEM_KEY))
+ .andReturn(PLACEHOLDER_VALUE);
+ EasyMock.expect(SliderUtils.getSystemEnv(PLACEHOLDER_SYSTEM_KEY_2))
+ .andReturn(PLACEHOLDER_VALUE_2);
+ PowerMock.replayAll();
+
+ Map<String, String> amLaunchEnv = testSliderClient.getAmLaunchEnv(conf);
+ Assert.assertNotNull(amLaunchEnv);
+ Assert.assertEquals("Should have 2 envs", amLaunchEnv.size(), 2);
+ Assert.assertNotNull(amLaunchEnv.get(AM_ENV));
+ Assert.assertEquals("Should be equal", amLaunchEnv.get(AM_ENV),
+ (Shell.WINDOWS ? "%" + AM_ENV + "%;" : "$" + AM_ENV + ":") +
+ "/usr/lib/" + PLACEHOLDER_VALUE);
+ Assert.assertNotNull(amLaunchEnv.get(AM_ENV_2));
+ Assert.assertEquals("Should be equal", amLaunchEnv.get(AM_ENV_2),
+ (Shell.WINDOWS ? "%" + AM_ENV_2 + "%;" : "$" + AM_ENV_2 + ":") +
+ "/usr/bin/" + PLACEHOLDER_VALUE_2);
+
+ PowerMock.verifyAll();
+ LOG.info("amLaunchEnv = " + amLaunchEnv);
+ }
+
+ static class TestSliderClient extends SliderClient {
+ @Override
+ public ApplicationId submitApplication(ApplicationSubmissionContext
+ context)
+ throws YarnException, IOException {
+ return new MockApplicationId(1);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderTokensCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderTokensCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderTokensCommand.java
new file mode 100644
index 0000000..f649ab7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderTokensCommand.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client;
+
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.slider.common.params.ActionTokensArgs;
+import org.apache.slider.common.params.Arguments;
+import org.apache.slider.common.params.SliderActions;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.NotFoundException;
+import org.apache.slider.utils.SliderTestBase;
+import org.junit.Test;
+
+import java.util.Arrays;
+
+/**
+ * Test the argument parsing/validation logic.
+ */
+public class TestSliderTokensCommand extends SliderTestBase {
+
+ private static YarnConfiguration config = createTestConfig();
+
+ public static YarnConfiguration createTestConfig() {
+ YarnConfiguration configuration = new YarnConfiguration();
+ configuration.set(YarnConfiguration.RM_ADDRESS, "127.0.0.1:8032");
+ return configuration;
+ }
+
+ @Test
+ public void testBadSourceArgs() throws Throwable {
+ launchExpectingException(SliderClient.class,
+ config,
+ ActionTokensArgs.DUPLICATE_ARGS,
+ Arrays.asList(SliderActions.ACTION_TOKENS,
+ Arguments.ARG_SOURCE, "target/tokens.bin",
+ Arguments.ARG_OUTPUT, "target/tokens.bin"
+ ));
+ }
+
+ @Test
+ public void testKTNoPrincipal() throws Throwable {
+ launchExpectingException(SliderClient.class,
+ config,
+ ActionTokensArgs.MISSING_KT_PROVIDER,
+ Arrays.asList(SliderActions.ACTION_TOKENS,
+ Arguments.ARG_KEYTAB, "target/keytab"
+ ));
+ }
+
+ @Test
+ public void testPrincipalNoKT() throws Throwable {
+ launchExpectingException(SliderClient.class,
+ config,
+ ActionTokensArgs.MISSING_KT_PROVIDER,
+ Arrays.asList(SliderActions.ACTION_TOKENS,
+ Arguments.ARG_PRINCIPAL, "bob@REALM"
+ ));
+ }
+
+ /**
+ * A missing keytab is an error.
+ * @throws Throwable
+ */
+ @Test
+ public void testMissingKT() throws Throwable {
+ Throwable ex = launchExpectingException(SliderClient.class,
+ config,
+ TokensOperation.E_NO_KEYTAB,
+ Arrays.asList(SliderActions.ACTION_TOKENS,
+ Arguments.ARG_PRINCIPAL, "bob@REALM",
+ Arguments.ARG_KEYTAB, "target/keytab"
+ ));
+ if (!(ex instanceof NotFoundException)) {
+ throw ex;
+ }
+ }
+
+ @Test
+ public void testMissingSourceFile() throws Throwable {
+ Throwable ex = launchExpectingException(SliderClient.class,
+ config,
+ TokensOperation.E_MISSING_SOURCE_FILE,
+ Arrays.asList(SliderActions.ACTION_TOKENS,
+ Arguments.ARG_SOURCE, "target/tokens.bin"
+ ));
+ if (!(ex instanceof NotFoundException)) {
+ throw ex;
+ }
+ }
+
+ @Test
+ public void testListHarmlessWhenInsecure() throws Throwable {
+ execSliderCommand(0, config, Arrays.asList(SliderActions.ACTION_TOKENS));
+ }
+
+ @Test
+ public void testCreateFailsWhenInsecure() throws Throwable {
+ Throwable ex = launchExpectingException(SliderClient.class,
+ config,
+ TokensOperation.E_INSECURE,
+ Arrays.asList(SliderActions.ACTION_TOKENS,
+ Arguments.ARG_OUTPUT, "target/tokens.bin"
+ ));
+ if (!(ex instanceof BadClusterStateException)) {
+ throw ex;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestClusterNames.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestClusterNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestClusterNames.java
new file mode 100644
index 0000000..efd0c2f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestClusterNames.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Test cluster name validation.
+ */
+public class TestClusterNames {
+
+ void assertValidName(String name) {
+ boolean valid = SliderUtils.isClusternameValid(name);
+ Assert.assertTrue("Clustername '" + name + "' mistakenly declared invalid",
+ valid);
+ }
+
+ void assertInvalidName(String name) {
+ boolean valid = SliderUtils.isClusternameValid(name);
+ Assert.assertFalse("Clustername '\" + name + \"' mistakenly declared valid",
+ valid);
+ }
+
+ void assertInvalid(List<String> names) {
+ for (String name : names) {
+ assertInvalidName(name);
+ }
+ }
+
+ void assertValid(List<String> names) {
+ for (String name : names) {
+ assertValidName(name);
+ }
+ }
+
+ @Test
+ public void testEmptyName() throws Throwable {
+ assertInvalidName("");
+ }
+
+ @Test
+ public void testSpaceName() throws Throwable {
+ assertInvalidName(" ");
+ }
+
+
+ @Test
+ public void testLeadingHyphen() throws Throwable {
+ assertInvalidName("-hyphen");
+ }
+
+ @Test
+ public void testTitleLetters() throws Throwable {
+ assertInvalidName("Title");
+ }
+
+ @Test
+ public void testCapitalLetters() throws Throwable {
+ assertInvalidName("UPPER-CASE-CLUSTER");
+ }
+
+ @Test
+ public void testInnerBraced() throws Throwable {
+ assertInvalidName("a[a");
+ }
+
+ @Test
+ public void testLeadingBrace() throws Throwable {
+ assertInvalidName("[");
+ }
+
+ @Test
+ public void testNonalphaLeadingChars() throws Throwable {
+ assertInvalid(Arrays.asList(
+ "[a", "#", "@", "=", "*", "."
+ ));
+ }
+
+ @Test
+ public void testNonalphaInnerChars() throws Throwable {
+ assertInvalid(Arrays.asList(
+ "a[a", "b#", "c@", "d=", "e*", "f.", "g ", "h i"
+ ));
+ }
+
+ @Test
+ public void testClusterValid() throws Throwable {
+ assertValidName("cluster");
+ }
+
+ @Test
+ public void testValidNames() throws Throwable {
+ assertValid(Arrays.asList(
+ "cluster",
+ "cluster1",
+ "very-very-very-long-cluster-name",
+ "c1234567890"
+ ));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelper.java
new file mode 100644
index 0000000..45c6118
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelper.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.slider.utils.YarnMiniClusterTestBase;
+import org.junit.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.util.Map;
+
+/**
+ * Test config helper.
+ */
+public class TestConfigHelper extends YarnMiniClusterTestBase {
+
+ @Test
+ public void testConfigLoaderIteration() throws Throwable {
+
+ String xml = "<?xml version=\"1.0\" encoding=\"UTF-8\" " +
+ "standalone=\"no\"?><configuration><property><name>key</name>" +
+ "<value>value</value><source>programatically</source></property>" +
+ "</configuration>";
+ InputStream ins = new ByteArrayInputStream(xml.getBytes("UTF8"));
+ Configuration conf = new Configuration(false);
+ conf.addResource(ins);
+ Configuration conf2 = new Configuration(false);
+ for (Map.Entry<String, String> entry : conf) {
+ conf2.set(entry.getKey(), entry.getValue(), "src");
+ }
+
+ }
+
+ @Test
+ public void testConfigDeprecation() throws Throwable {
+ ConfigHelper.registerDeprecatedConfigItems();
+ Configuration conf = new Configuration(false);
+ // test deprecated items here
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelperHDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelperHDFS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelperHDFS.java
new file mode 100644
index 0000000..f9a58d4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelperHDFS.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.slider.utils.YarnMiniClusterTestBase;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URI;
+
+/**
+ * Test config helper loading configuration from HDFS.
+ */
+public class TestConfigHelperHDFS extends YarnMiniClusterTestBase {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestConfigHelperHDFS.class);
+
+ @Test
+ public void testConfigHelperHDFS() throws Throwable {
+ YarnConfiguration config = getConfiguration();
+ createMiniHDFSCluster("testConfigHelperHDFS", config);
+
+ Configuration conf = new Configuration(false);
+ conf.set("key", "value");
+ URI fsURI = new URI(getFsDefaultName());
+ Path root = new Path(fsURI);
+ Path confPath = new Path(root, "conf.xml");
+ FileSystem dfs = FileSystem.get(fsURI, config);
+ ConfigHelper.saveConfig(dfs, confPath, conf);
+ //load time
+ Configuration loaded = ConfigHelper.loadConfiguration(dfs, confPath);
+ LOG.info(ConfigHelper.dumpConfigToString(loaded));
+ assertEquals("value", loaded.get("key"));
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org