You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/07/02 18:00:39 UTC
[10/26] git commit: SLIDER-171: connectivity tests to verify state of
funtest client config
SLIDER-171: connectivity tests to verify state of funtest client config
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fb73dbca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fb73dbca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fb73dbca
Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it
Commit: fb73dbca80cd87c2880d7e46325a67c507f58ee0
Parents: a6a4680
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jul 1 20:08:54 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jul 1 20:08:54 2014 +0100
----------------------------------------------------------------------
.../slider/client/SliderYarnClientImpl.java | 3 +-
.../funtest/framework/CommandTestBase.groovy | 4 +-
.../basic/TestClusterConnectivity.groovy | 96 ++++++++++++++++++++
.../clusters/c6401/slider/slider-client.xml | 14 ++-
.../clusters/morzine/slider/slider-client.xml | 29 ++++--
5 files changed, 132 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fb73dbca/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index 0c83e0c..e7b492b 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -19,6 +19,7 @@
package org.apache.slider.client;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
@@ -74,7 +75,7 @@ public class SliderYarnClientImpl extends YarnClientImpl {
List<ApplicationReport> allApps = getApplications(types);
List<ApplicationReport> results = new ArrayList<>();
for (ApplicationReport report : allApps) {
- if (user == null || user.equals(report.getUser())) {
+ if (StringUtils.isEmpty(user) || user.equals(report.getUser())) {
results.add(report);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fb73dbca/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 61ae804..08d352a 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -103,15 +103,13 @@ abstract class CommandTestBase extends SliderTestUtils {
log.debug("Security enabled")
SliderUtils.forceLogin()
} else {
- log.info "Security off, making cluster dirs broadly accessible"
+ log.info "Security is off"
}
SliderShell.confDir = SLIDER_CONF_DIRECTORY
SliderShell.script = SLIDER_SCRIPT
log.info("Test using ${HadoopFS.getDefaultUri(SLIDER_CONFIG)} " +
"and YARN RM @ ${SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS)}")
- // now patch the settings with the path of the conf direcotry
-
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fb73dbca/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/TestClusterConnectivity.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/TestClusterConnectivity.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/TestClusterConnectivity.groovy
new file mode 100644
index 0000000..cf9cc9d
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/TestClusterConnectivity.groovy
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.basic
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.net.NetUtils
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.client.SliderYarnClientImpl
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.core.zk.ZookeeperUtils
+import org.apache.slider.funtest.framework.CommandTestBase
+import org.junit.Test
+
+@Slf4j
+class TestClusterConnectivity extends CommandTestBase {
+
+ @Test
+ public void testFileSystemUp() throws Throwable {
+
+ def fs = clusterFS
+ def status = fs.listStatus(new Path("/"))
+ status.each {it -> log.info(it.path.toString())}
+
+ }
+
+ @Test
+ public void testZKBinding() throws Throwable {
+ def quorum = SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.REGISTRY_ZK_QUORUM)
+ assert quorum
+ def tuples = ZookeeperUtils.splitToHostsAndPortsStrictly(quorum);
+ tuples.each {it -> telnet(it.getHostText(), it.getPort())}
+
+ }
+
+ @Test
+ public void testRMTelnet() throws Throwable {
+ def rmAddr = SLIDER_CONFIG.getSocketAddr(YarnConfiguration.RM_ADDRESS, "", 0)
+ telnet(rmAddr.hostName, rmAddr.port)
+ }
+
+ @Test
+ public void testRMBinding() throws Throwable {
+ testRMTelnet()
+ SliderYarnClientImpl yarnClient = new SliderYarnClientImpl()
+ try {
+ SLIDER_CONFIG.setInt("ipc.client.connect.retry.interval",100)
+ SLIDER_CONFIG.setInt(
+ YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,5000)
+ SLIDER_CONFIG.setInt(
+ YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,50)
+
+ yarnClient.init(SLIDER_CONFIG)
+ yarnClient.start();
+ def instances = yarnClient.listInstances("")
+ instances.each {it -> log.info("Instance $it.applicationId")}
+ } finally {
+ yarnClient.stop()
+ }
+ }
+
+ def telnet(String host, int port) {
+ assert host != ""
+ assert port != 0
+ try {
+ def socket = new Socket(host, port);
+ } catch (IOException e) {
+ throw NetUtils.wrapException(host, port, "localhost", 0, e)
+ }
+/*
+
+ socket.withStreams { input, output ->
+ output << "echo testing ...\n"
+ def buffer = input.newReader().readLine()
+ println "response = ${buffer}"
+*/
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fb73dbca/src/test/clusters/c6401/slider/slider-client.xml
----------------------------------------------------------------------
diff --git a/src/test/clusters/c6401/slider/slider-client.xml b/src/test/clusters/c6401/slider/slider-client.xml
index fe2caad..526881b 100644
--- a/src/test/clusters/c6401/slider/slider-client.xml
+++ b/src/test/clusters/c6401/slider/slider-client.xml
@@ -21,25 +21,31 @@
They are not passed to the AM
-->
<configuration>
+
+ <property>
+ <name>hostname</name>
+ <value>c6401</value>
+ </property>
+
<property>
<name>slider.client.resource.origin</name>
- <value>configs/c6401/slider</value>
+ <value>configs/${hostname}/slider</value>
<description>This is just for diagnostics</description>
</property>
<property>
<name>yarn.resourcemanager.address</name>
- <value>c6401:8050</value>
+ <value>${hostname}:8050</value>
</property>
<property>
<name>fs.defaultFS</name>
- <value>hdfs://c6401.ambari.apache.org:8020</value>
+ <value>hdfs://${hostname}.ambari.apache.org:8020</value>
</property>
<property>
<name>slider.zookeeper.quorum</name>
- <value>c6401:2181</value>
+ <value>${hostname}:2181</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fb73dbca/src/test/clusters/morzine/slider/slider-client.xml
----------------------------------------------------------------------
diff --git a/src/test/clusters/morzine/slider/slider-client.xml b/src/test/clusters/morzine/slider/slider-client.xml
index d15d443..4d7ab41 100644
--- a/src/test/clusters/morzine/slider/slider-client.xml
+++ b/src/test/clusters/morzine/slider/slider-client.xml
@@ -17,28 +17,45 @@
limitations under the License.
-->
<!--
- Morzine is runing Hadoop on Windows
+ Properties set here are picked up in the client.
+ They are not passed to the AM
-->
+<configuration>
+
+ <property>
+ <name>hostname</name>
+ <value>morzine</value>
+ </property>
+
+ <property>
+ <name>slider.client.resource.origin</name>
+ <value>configs/${hostname}/slider</value>
+ <description>This is just for diagnostics</description>
+ </property>
<property>
<name>yarn.resourcemanager.address</name>
- <value>morzine:8050</value>
+ <value>${hostname}:8032</value>
</property>
-
+
<property>
<name>fs.defaultFS</name>
- <value>hdfs://morzine:8020</value>
+ <value>hdfs://${hostname}:8020</value>
</property>
<property>
<name>slider.zookeeper.quorum</name>
- <value>morzine:2181</value>
+ <value>${hostname}:2181</value>
</property>
+ <!--
+ This is a windows path as picked up from
+ http://morzine:8088/conf
+ -->
<property>
<name>yarn.application.classpath</name>
<value>
- /etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*
+ %HADOOP_CONF_DIR%,%HADOOP_COMMON_HOME%/share/hadoop/common/*,%HADOOP_COMMON_HOME%/share/hadoop/common/lib/*,%HADOOP_HDFS_HOME%/share/hadoop/hdfs/*,%HADOOP_HDFS_HOME%/share/hadoop/hdfs/lib/*,%HADOOP_MAPRED_HOME%/share/hadoop/mapreduce/*,%HADOOP_MAPRED_HOME%/share/hadoop/mapreduce/lib/*,%HADOOP_YARN_HOME%/share/hadoop/yarn/*,%HADOOP_YARN_HOME%/share/hadoop/yarn/lib/*
</value>
</property>