You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/10/15 03:07:39 UTC

[01/23] git commit: SLIDER-504 migrate to new Registry constant, retaining the old one as a deprecated value (for tests, mainly)

Repository: incubator-slider
Updated Branches:
  refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports 1ef6a88f2 -> 34e44871e


SLIDER-504 migrate to new Registry constant, retaining the old one as a deprecated value (for tests, mainly)


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9856b24e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9856b24e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9856b24e

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: 9856b24e78e6408348514157d18eb0d24f438247
Parents: da67c44
Author: Steve Loughran <st...@apache.org>
Authored: Mon Oct 13 16:19:25 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Oct 13 16:19:25 2014 -0700

----------------------------------------------------------------------
 .../funtest/accumulo/AccumuloReadWriteIT.groovy |  4 +-
 .../accumulo/AccumuloReadWriteSSLIT.groovy      |  4 +-
 .../org/apache/slider/client/SliderClient.java  |  4 +-
 .../apache/slider/common/SliderXmlConfKeys.java |  2 +-
 .../slider/common/tools/ConfigHelper.java       | 20 +++++-
 .../utility/AbstractSliderLaunchedService.java  | 13 ++--
 .../slider/common/tools/TestConfigHelper.groovy | 65 ++++++++++++++++++++
 .../common/tools/TestConfigHelperHDFS.groovy    | 20 ------
 .../test/YarnZKMiniClusterTestBase.groovy       |  2 +-
 .../agent/TestAgentProviderService.java         | 13 ++--
 .../funtest/framework/CommandTestBase.groovy    |  5 +-
 .../funtest/basic/ClusterConnectivityIT.groovy  |  4 +-
 .../lifecycle/ClusterBuildDestroyIT.groovy      |  4 +-
 .../accumulo/funtest/AccumuloCIIT.groovy        |  4 +-
 .../funtest/FunctionalHBaseClusterIT.groovy     |  4 +-
 .../funtest/HBaseClusterBuildDestroyIT.groovy   |  4 +-
 16 files changed, 123 insertions(+), 49 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9856b24e/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteIT.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteIT.groovy
index ca00a6c..12a3bff 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteIT.groovy
@@ -25,6 +25,7 @@ import org.apache.accumulo.core.client.ZooKeeperInstance
 import org.apache.accumulo.core.client.security.tokens.PasswordToken
 import org.apache.accumulo.test.TestIngest
 import org.apache.accumulo.test.VerifyIngest
+import org.apache.hadoop.registry.client.api.RegistryConstants
 import org.apache.slider.api.ClusterDescription
 import org.apache.slider.client.SliderClient
 import org.apache.slider.common.SliderXmlConfKeys
@@ -48,7 +49,8 @@ class AccumuloReadWriteIT extends AccumuloBasicIT {
   @Override
   public void clusterLoadOperations(ClusterDescription cd, SliderClient sliderClient) {
     try {
-      String zookeepers = SLIDER_CONFIG.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM,
+      String zookeepers = SLIDER_CONFIG.get(
+          RegistryConstants.KEY_REGISTRY_ZK_QUORUM,
         FuntestProperties.DEFAULT_SLIDER_ZK_HOSTS)
 
       ZooKeeperInstance instance = new ZooKeeperInstance(

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9856b24e/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteSSLIT.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteSSLIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteSSLIT.groovy
index 6f27301..cc227ed 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteSSLIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteSSLIT.groovy
@@ -21,6 +21,7 @@ import org.apache.accumulo.core.client.ClientConfiguration
 import org.apache.accumulo.core.client.Connector
 import org.apache.accumulo.core.client.ZooKeeperInstance
 import org.apache.accumulo.core.client.security.tokens.PasswordToken
+import org.apache.hadoop.registry.client.api.RegistryConstants
 import org.apache.slider.api.ClusterDescription
 import org.apache.slider.client.SliderClient
 import org.apache.slider.common.SliderXmlConfKeys
@@ -42,7 +43,8 @@ class AccumuloReadWriteSSLIT extends AccumuloSSLTestBase {
   }
 
   public ZooKeeperInstance getInstance() {
-    String zookeepers = SLIDER_CONFIG.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM,
+    String zookeepers = SLIDER_CONFIG.get(
+        RegistryConstants.KEY_REGISTRY_ZK_QUORUM,
       FuntestProperties.DEFAULT_SLIDER_ZK_HOSTS)
     ClientConfiguration conf = new ClientConfiguration()
       .withInstance(tree.global.get("site.global.accumulo_instance_name"))

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9856b24e/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 78f214e..750bfaa 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1234,8 +1234,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
     
     addConfOptionToCLI(commandLine, config, REGISTRY_PATH,
         DEFAULT_REGISTRY_PATH);
-    addMandatoryConfOptionToCLI(commandLine, config, RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
-    addMandatoryConfOptionToCLI(commandLine, config, REGISTRY_ZK_QUORUM);
+    addMandatoryConfOptionToCLI(commandLine, config,
+        RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
 
     if (clusterSecure) {
       // if the cluster is secure, make sure that

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9856b24e/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
index cc2a03f..000511f 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
@@ -123,7 +123,7 @@ public interface SliderXmlConfKeys {
   String DEFAULT_DATA_DIRECTORY_PERMISSIONS = "750";
 
 
-  String REGISTRY_PATH = RegistryConstants.KEY_REGISTRY_ZK_ROOT;
+  String REGISTRY_PATH = "slider.registry.path";
 
   /**
    * Default value for the registry: {@value}

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9856b24e/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
index b7e1323..31acb94 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
 import org.apache.slider.common.SliderKeys;
 import org.apache.slider.common.SliderXmlConfKeys;
 import org.apache.slider.core.exceptions.BadConfigException;
@@ -48,6 +49,7 @@ import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Set;
 import java.util.TreeSet;
 
 /**
@@ -65,15 +67,14 @@ public class ConfigHelper {
    * @param conf config
    * @return the sorted keyset
    */
-  public static TreeSet<String> dumpConf(Configuration conf) {
-    TreeSet<String> keys = sortedConfigKeys(conf);
+  public static Set<String> dumpConf(Configuration conf) {
+    Set<String> keys = sortedConfigKeys(conf);
     for (String key : keys) {
       log.info("{}={}", key, conf.get(key));
     }
     return keys;
   }
 
-
   /**
    * Take a configuration and return a sorted set
    * @param conf config
@@ -561,4 +562,17 @@ public class ConfigHelper {
     }
     return result;
   }
+
+  /**
+   * Register anything we consider deprecated
+   */
+  public static void registerDeprecatedConfigItems() {
+    Configuration.addDeprecation(
+        SliderXmlConfKeys.REGISTRY_ZK_QUORUM,
+        RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
+    Configuration.addDeprecation(
+        SliderXmlConfKeys.REGISTRY_PATH,
+        RegistryConstants.KEY_REGISTRY_ZK_ROOT);
+    
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9856b24e/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
index acdb882..10af3d3 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
@@ -18,11 +18,13 @@
 
 package org.apache.slider.server.services.utility;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.registry.client.api.RegistryConstants;
 import org.apache.hadoop.registry.client.api.RegistryOperations;
 import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
 import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.common.tools.ConfigHelper;
 import org.apache.slider.common.tools.SliderUtils;
 import org.apache.slider.core.exceptions.BadCommandArgumentsException;
 import org.apache.slider.core.exceptions.BadConfigException;
@@ -41,7 +43,8 @@ public abstract class AbstractSliderLaunchedService extends
   public AbstractSliderLaunchedService(String name) {
     super(name);
     // make sure all the yarn configs get loaded
-    new YarnConfiguration();
+    YarnConfiguration conf = new YarnConfiguration();
+    ConfigHelper.registerDeprecatedConfigItems();
   }
 
   /**
@@ -50,14 +53,9 @@ public abstract class AbstractSliderLaunchedService extends
    * @throws BadConfigException if it is not there or invalid
    */
   public String lookupZKQuorum() throws BadConfigException {
-    // YARN registry first
+ 
     String registryQuorum = getConfig().get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
     
-    // slider value can overrride it
-    registryQuorum = getConfig().get(
-        SliderXmlConfKeys.REGISTRY_ZK_QUORUM,
-        registryQuorum);
-    
     // though if neither is set: trouble
     if (SliderUtils.isUnset(registryQuorum)) {
       throw new BadConfigException(
@@ -79,7 +77,6 @@ public abstract class AbstractSliderLaunchedService extends
 
     // push back the slider registry entry if needed
     String quorum = lookupZKQuorum();
-    getConfig().set(RegistryConstants.KEY_REGISTRY_ZK_QUORUM, quorum);
     RegistryOperations registryWriterService =
         createRegistryOperationsInstance();
     deployChildService(registryWriterService);

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9856b24e/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelper.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelper.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelper.groovy
new file mode 100644
index 0000000..bdab926
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelper.groovy
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.registry.client.api.RegistryConstants
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.test.YarnMiniClusterTestBase
+import org.junit.Test
+
+@Slf4j
+@CompileStatic
+
+class TestConfigHelper extends YarnMiniClusterTestBase {
+
+
+  @Test
+  public void testConfigLoaderIteration() throws Throwable {
+
+    String xml =
+    """<?xml version="1.0" encoding="UTF-8" standalone="no"?><configuration>
+<property><name>key</name><value>value</value><source>programatically</source></property>
+</configuration>
+    """
+    InputStream ins = new ByteArrayInputStream(xml.bytes);
+    Configuration conf = new Configuration(false);
+    conf.addResource(ins);
+    Configuration conf2 = new Configuration(false);
+    for (Map.Entry<String, String> entry : conf) {
+      conf2.set(entry.key, entry.value, "src")
+    }
+    
+  }
+
+  @Test
+  public void testConfigDeprecation() throws Throwable {
+    ConfigHelper.registerDeprecatedConfigItems()
+    Configuration conf = new Configuration(false);
+    conf.set(SliderXmlConfKeys.REGISTRY_PATH, "path")
+    assert "path" == conf.get(SliderXmlConfKeys.REGISTRY_PATH)
+    assert "path" == conf.get(RegistryConstants.KEY_REGISTRY_ZK_ROOT)
+
+    conf.set(SliderXmlConfKeys.REGISTRY_ZK_QUORUM, "localhost")
+    assert "localhost" == conf.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM)
+    assert "localhost" == conf.get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM)
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9856b24e/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy
index d78aa1b..6558437 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestConfigHelperHDFS.groovy
@@ -32,7 +32,6 @@ import org.junit.Test
 
 class TestConfigHelperHDFS extends YarnMiniClusterTestBase {
 
-  //diabled for now; 
   @Test 
   public void testConfigHelperHDFS() throws Throwable {
     YarnConfiguration config = getConfiguration()
@@ -51,23 +50,4 @@ class TestConfigHelperHDFS extends YarnMiniClusterTestBase {
     assert loaded.get("key") == "value"
   }
 
-  @Test
-  public void testConfigLoaderIteration() throws Throwable {
-
-    String xml =
-    """<?xml version="1.0" encoding="UTF-8" standalone="no"?><configuration>
-<property><name>key</name><value>value</value><source>programatically</source></property>
-</configuration>
-    """
-    InputStream ins = new ByteArrayInputStream(xml.bytes);
-    Configuration conf = new Configuration(false);
-    conf.addResource(ins);
-    Configuration conf2 = new Configuration(false);
-    for (Map.Entry<String, String> entry : conf) {
-      String key = entry.getKey();
-      String val = entry.getValue();
-      conf2.set(key, val, "src")
-    }
-    
-  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9856b24e/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
index e182ed2..3fae9fd 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
@@ -160,7 +160,7 @@ public abstract class YarnZKMiniClusterTestBase extends YarnMiniClusterTestBase
    */
   protected List<String> getExtraCLIArgs() {
     [
-      "-D", define(SliderXmlConfKeys.REGISTRY_ZK_QUORUM, ZKBinding)
+      "-D", define(RegistryConstants.KEY_REGISTRY_ZK_QUORUM, ZKBinding)
     ]
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9856b24e/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index addfc8a..7eff45a 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FilterFileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
 import org.apache.hadoop.registry.client.api.RegistryOperations;
 import org.apache.hadoop.registry.client.types.ServiceRecord;
 import org.apache.hadoop.yarn.api.records.Container;
@@ -299,8 +300,8 @@ public class TestAgentProviderService {
     doReturn(metainfo).when(mockAps).getApplicationMetainfo(any(SliderFileSystem.class), anyString());
 
     Configuration conf = new Configuration();
-    conf.set(SliderXmlConfKeys.REGISTRY_PATH,
-        SliderXmlConfKeys.DEFAULT_REGISTRY_PATH);
+    conf.set(RegistryConstants.KEY_REGISTRY_ZK_ROOT,
+        RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
 
     try {
       doReturn(true).when(mockAps).isMaster(anyString());
@@ -427,7 +428,9 @@ public class TestAgentProviderService {
     expect(container.getPriority()).andReturn(Priority.newInstance(1));
 
     StateAccessForProviders access = createNiceMock(StateAccessForProviders.class);
-    AgentProviderService aps = createAgentProviderService(new Configuration());
+    Configuration conf = new Configuration();
+
+    AgentProviderService aps = createAgentProviderService(conf);
     AgentProviderService mockAps = Mockito.spy(aps);
 
     doReturn(access).when(mockAps).getAmState();
@@ -442,9 +445,7 @@ public class TestAgentProviderService {
         any(SliderFileSystem.class), anyString());
     doReturn(metainfo).when(mockAps).getMetainfo();
 
-    Configuration conf = new Configuration();
-    conf.set(SliderXmlConfKeys.REGISTRY_PATH,
-        SliderXmlConfKeys.DEFAULT_REGISTRY_PATH);
+
 
     try {
       doReturn(true).when(mockAps).isMaster(anyString());

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9856b24e/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index db1c562..7f3c43d 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -22,9 +22,11 @@ import groovy.transform.CompileStatic
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.FileSystem as HadoopFS
 import org.apache.hadoop.fs.Path
+import org.apache.hadoop.registry.client.api.RegistryConstants
 import org.apache.hadoop.util.ExitUtil
 import org.apache.hadoop.util.Shell
 import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.common.tools.ConfigHelper
 import org.apache.slider.core.main.LauncherExitCodes
 import org.apache.slider.core.main.ServiceLauncher
 import org.apache.slider.common.SliderKeys
@@ -81,6 +83,7 @@ abstract class CommandTestBase extends SliderTestUtils {
 
 
   static {
+    ConfigHelper.registerDeprecatedConfigItems();
     SLIDER_CONFIG = ConfLoader.loadSliderConf(SLIDER_CONF_XML);
     THAW_WAIT_TIME = getTimeOptionMillis(SLIDER_CONFIG,
         KEY_TEST_THAW_WAIT_TIME,
@@ -436,7 +439,7 @@ abstract class CommandTestBase extends SliderTestUtils {
     List<String> argsList = [action, clustername]
 
     argsList << ARG_ZKHOSTS <<
-    SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.REGISTRY_ZK_QUORUM)
+    SLIDER_CONFIG.getTrimmed(RegistryConstants.KEY_REGISTRY_ZK_QUORUM)
 
 
     if (blockUntilRunning) {

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9856b24e/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ClusterConnectivityIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ClusterConnectivityIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ClusterConnectivityIT.groovy
index 9b8fe6f..b09e8b7 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ClusterConnectivityIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ClusterConnectivityIT.groovy
@@ -21,6 +21,7 @@ package org.apache.slider.funtest.basic
 import groovy.util.logging.Slf4j
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.net.NetUtils
+import org.apache.hadoop.registry.client.api.RegistryConstants
 import org.apache.hadoop.yarn.conf.YarnConfiguration
 import org.apache.slider.client.SliderYarnClientImpl
 import org.apache.slider.common.SliderXmlConfKeys
@@ -52,7 +53,8 @@ class ClusterConnectivityIT extends CommandTestBase {
 
   @Test
   public void testZKBinding() throws Throwable {
-    def quorum = SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.REGISTRY_ZK_QUORUM)
+    def quorum = SLIDER_CONFIG.getTrimmed(
+        RegistryConstants.KEY_REGISTRY_ZK_QUORUM)
     assert quorum
     def tuples = ZookeeperUtils.splitToHostsAndPortsStrictly(quorum);
     tuples.each {

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9856b24e/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
index c5dbecb..e53473a 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
@@ -21,6 +21,7 @@ package org.apache.slider.funtest.lifecycle
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
 import org.apache.hadoop.fs.Path
+import org.apache.hadoop.registry.client.api.RegistryConstants
 import org.apache.slider.common.SliderExitCodes
 import org.apache.slider.common.SliderKeys
 import org.apache.slider.common.SliderXmlConfKeys
@@ -62,7 +63,8 @@ public class ClusterBuildDestroyIT extends AgentCommandTestBase
             ACTION_BUILD,
             CLUSTER,
             ARG_ZKHOSTS,
-            SLIDER_CONFIG.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM, DEFAULT_SLIDER_ZK_HOSTS),
+            SLIDER_CONFIG.get(
+                RegistryConstants.KEY_REGISTRY_ZK_QUORUM, DEFAULT_SLIDER_ZK_HOSTS),
             ARG_TEMPLATE, APP_TEMPLATE,
             ARG_RESOURCES, APP_RESOURCE
         ])

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9856b24e/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/AccumuloCIIT.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/AccumuloCIIT.groovy b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/AccumuloCIIT.groovy
index 4ec5ff1..bef5a03 100644
--- a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/AccumuloCIIT.groovy
+++ b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/AccumuloCIIT.groovy
@@ -25,6 +25,7 @@ import org.apache.accumulo.test.continuous.ContinuousIngest
 import org.apache.accumulo.test.continuous.ContinuousVerify
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.io.Text
+import org.apache.hadoop.registry.client.api.RegistryConstants
 import org.apache.hadoop.util.ToolRunner
 import org.apache.hadoop.yarn.conf.YarnConfiguration
 import org.apache.slider.common.SliderXmlConfKeys
@@ -59,7 +60,8 @@ class AccumuloCIIT extends FunctionalAccumuloClusterIT {
     assert clustername
 
     String currentUser = System.getProperty("user.name");
-    String zookeepers = SLIDER_CONFIG.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM,
+    String zookeepers = SLIDER_CONFIG.get(
+        RegistryConstants.KEY_REGISTRY_ZK_QUORUM,
         FuntestProperties.DEFAULT_SLIDER_ZK_HOSTS)
     ZooKeeperInstance inst = new ZooKeeperInstance(currentUser + "-" + clustername, zookeepers)
     Connector conn = inst.getConnector("root", new PasswordToken(getPassword()))

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9856b24e/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy
index 3b8d1f9..936c616 100644
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy
@@ -21,6 +21,7 @@ package org.apache.slider.providers.hbase.funtest
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
 import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.registry.client.api.RegistryConstants
 import org.apache.hadoop.yarn.conf.YarnConfiguration
 import org.apache.slider.api.ClusterDescription
 import org.apache.slider.api.RoleKeys
@@ -66,7 +67,8 @@ public class FunctionalHBaseClusterIT extends HBaseCommandTestBase
   @Before
   public void prepareCluster() {
 
-    String quorumServers = SLIDER_CONFIG.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM, DEFAULT_SLIDER_ZK_HOSTS)
+    String quorumServers = SLIDER_CONFIG.get(
+        RegistryConstants.KEY_REGISTRY_ZK_QUORUM, DEFAULT_SLIDER_ZK_HOSTS)
   
     ZooKeeper monitor = new ZooKeeper(quorumServers,
       1000, new Watcher(){

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9856b24e/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterBuildDestroyIT.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterBuildDestroyIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterBuildDestroyIT.groovy
index 3a44e30..63672d6 100644
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterBuildDestroyIT.groovy
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterBuildDestroyIT.groovy
@@ -21,6 +21,7 @@ package org.apache.slider.providers.hbase.funtest
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
 import org.apache.hadoop.fs.Path
+import org.apache.hadoop.registry.client.api.RegistryConstants
 import org.apache.slider.common.SliderKeys
 import org.apache.slider.common.SliderXmlConfKeys
 import org.apache.slider.common.params.Arguments
@@ -62,7 +63,8 @@ public class HBaseClusterBuildDestroyIT extends HBaseCommandTestBase
             CLUSTER,
             ARG_PROVIDER, HBaseKeys.PROVIDER_HBASE,
             ARG_ZKHOSTS,
-            SLIDER_CONFIG.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM, DEFAULT_SLIDER_ZK_HOSTS),
+            SLIDER_CONFIG.get(
+                RegistryConstants.KEY_REGISTRY_ZK_QUORUM, DEFAULT_SLIDER_ZK_HOSTS),
             ARG_IMAGE,
             SLIDER_CONFIG.get(KEY_TEST_HBASE_TAR),
             ARG_CONFDIR,


[23/23] git commit: Merge branch 'develop' into feature/SLIDER-481_allow_dedicated_handling_of_exports

Posted by sm...@apache.org.
Merge branch 'develop' into feature/SLIDER-481_allow_dedicated_handling_of_exports


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/34e44871
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/34e44871
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/34e44871

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: 34e44871efd17f0622fa55f94185f133b1a76a19
Parents: 1ef6a88 c31daa1
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Oct 14 18:07:25 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Oct 14 18:07:25 2014 -0700

----------------------------------------------------------------------
 app-packages/accumulo/appConfig-default.json    |  43 +++++--
 .../accumulo/configuration/accumulo-env.xml     |  89 +++++++++++++
 app-packages/accumulo/configuration/client.xml  |  44 +++++++
 app-packages/accumulo/configuration/global.xml  |  89 -------------
 app-packages/accumulo/metainfo.xml              |  10 ++
 .../package/scripts/accumulo_configuration.py   |  14 +-
 app-packages/accumulo/package/scripts/params.py |  33 +++--
 .../accumulo/package/templates/client.conf.j2   |  18 ---
 app-packages/accumulo/resources-default.json    |   4 +-
 .../accumulo/AccumuloMonitorSSLIT.groovy        |   1 -
 .../AccumuloMonitorSSLLocalCertsIT.groovy       |  72 +++++++++++
 .../funtest/accumulo/AccumuloReadWriteIT.groovy |   6 +-
 .../accumulo/AccumuloReadWriteSSLIT.groovy      |   6 +-
 .../funtest/accumulo/AccumuloSSLTestBase.groovy |   5 +-
 slider-assembly/src/conf-hdp/slider-client.xml  |  11 --
 slider-assembly/src/conf/log4j.properties       |   2 +
 slider-assembly/src/conf/slider-client.xml      |  10 +-
 .../org/apache/slider/client/SliderClient.java  | 127 ++++++++++++++-----
 .../apache/slider/client/SliderClientAPI.java   |   4 -
 .../apache/slider/common/SliderXmlConfKeys.java |   2 +-
 .../common/params/ActionDiagnosticArgs.java     |   7 +-
 .../common/params/ActionRegistryArgs.java       |   6 +-
 .../slider/common/params/ActionResolveArgs.java |   1 -
 .../apache/slider/common/params/Arguments.java  |  33 ++---
 .../apache/slider/common/params/ClientArgs.java |   2 +-
 .../slider/common/tools/ConfigHelper.java       |  22 +++-
 .../slider/core/launch/AbstractLauncher.java    |   3 +-
 .../core/registry/SliderRegistryUtils.java      |  62 +++++++++
 .../server/appmaster/SliderAppMaster.java       |   9 +-
 .../utility/AbstractSliderLaunchedService.java  |  13 +-
 .../TestStandaloneYarnRegistryAM.groovy         |  29 ++++-
 .../apache/slider/client/TestDiagnostics.groovy |  58 +++++++++
 .../slider/common/tools/TestConfigHelper.groovy |  65 ++++++++++
 .../common/tools/TestConfigHelperHDFS.groovy    |  20 ---
 .../slider/registry/TestRegistryPaths.groovy    |  17 ++-
 .../test/YarnZKMiniClusterTestBase.groovy       |   2 +-
 .../core/launch/TestAppMasterLauncher.java      |   8 +-
 .../agent/TestAgentProviderService.java         |  13 +-
 .../framework/AgentCommandTestBase.groovy       |   7 +-
 .../funtest/framework/CommandTestBase.groovy    |  35 ++++-
 .../funtest/framework/FuntestProperties.groovy  |   6 +
 .../slider/funtest/framework/SliderShell.groovy |  23 +++-
 .../funtest/basic/ClusterConnectivityIT.groovy  |   4 +-
 .../commands/DiagnosticsCommandIT.groovy        |  47 +++++++
 .../lifecycle/AgentClusterLifecycleIT.groovy    |   2 +-
 .../funtest/lifecycle/AgentFailures2IT.groovy   |  12 +-
 .../lifecycle/ClusterBuildDestroyIT.groovy      |   7 +-
 .../accumulo/funtest/AccumuloCIIT.groovy        |   4 +-
 .../funtest/FunctionalHBaseClusterIT.groovy     |   4 +-
 .../funtest/HBaseClusterBuildDestroyIT.groovy   |   4 +-
 50 files changed, 812 insertions(+), 303 deletions(-)
----------------------------------------------------------------------



[22/23] git commit: SLIDER-519. Cleanup slider-client.xml for the slider package

Posted by sm...@apache.org.
SLIDER-519. Cleanup slider-client.xml for the slider package


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c31daa16
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c31daa16
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c31daa16

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: c31daa16225f9e2f3f83c882f5f686662544fd92
Parents: 4febd77
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Oct 14 18:06:43 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Oct 14 18:06:43 2014 -0700

----------------------------------------------------------------------
 slider-assembly/src/conf-hdp/slider-client.xml | 11 -----------
 slider-assembly/src/conf/slider-client.xml     | 10 +---------
 2 files changed, 1 insertion(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c31daa16/slider-assembly/src/conf-hdp/slider-client.xml
----------------------------------------------------------------------
diff --git a/slider-assembly/src/conf-hdp/slider-client.xml b/slider-assembly/src/conf-hdp/slider-client.xml
index 8e87e53..a2cc385 100644
--- a/slider-assembly/src/conf-hdp/slider-client.xml
+++ b/slider-assembly/src/conf-hdp/slider-client.xml
@@ -27,17 +27,6 @@
     <value>localhost:2181</value>
   </property>
 
-  <property>
-    <name>slider.yarn.queue</name>
-    <value>default</value>
-    <description>YARN queue for the Application Master</description>
-  </property>
-  
-  <property>
-    <name>yarn.log-aggregation-enable</name>
-    <value>true</value>
-  </property>
-    
 <!--
   <property>
     <name>yarn.application.classpath</name>

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c31daa16/slider-assembly/src/conf/slider-client.xml
----------------------------------------------------------------------
diff --git a/slider-assembly/src/conf/slider-client.xml b/slider-assembly/src/conf/slider-client.xml
index abf07fe..b6a5b32 100644
--- a/slider-assembly/src/conf/slider-client.xml
+++ b/slider-assembly/src/conf/slider-client.xml
@@ -27,12 +27,7 @@
     <value>@ZK-QUORUM</value>
   </property>
 
-  <property>
-    <name>slider.yarn.queue</name>
-    <value>default</value>
-    <description>YARN queue for the Application Master</description>
-  </property>
-  
+<!--
   <property>
     <name>slider.client.resource.origin</name>
     <value>conf/slider-client.xml</value>
@@ -44,7 +39,6 @@
     <value>true</value>
   </property>
 
-<!--
   <property>
     <name>yarn.application.classpath</name>
     <value>/etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*</value>
@@ -64,9 +58,7 @@
     <name>fs.defaultFS</name>
     <value>@FILESYSTEM</value>
   </property>
--->
 
-<!--
   <property>
     <name>yarn.resourcemanager.principal</name>
     <value>yarn/master@MINICLUSTER</value>


[07/23] git commit: SLIDER-146 added test for providing ssl certs on local fs

Posted by sm...@apache.org.
SLIDER-146 added test for providing ssl certs on local fs


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f507a4c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f507a4c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f507a4c0

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: f507a4c04093ca28616f95417ba4c8fd3bc01298
Parents: 7704bf0
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Mon Oct 13 20:01:02 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Mon Oct 13 20:01:02 2014 -0700

----------------------------------------------------------------------
 .../package/scripts/accumulo_configuration.py   |  5 +-
 .../accumulo/AccumuloMonitorSSLIT.groovy        |  1 -
 .../AccumuloMonitorSSLLocalCertsIT.groovy       | 72 ++++++++++++++++++++
 .../funtest/accumulo/AccumuloSSLTestBase.groovy |  5 +-
 4 files changed, 77 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f507a4c0/app-packages/accumulo/package/scripts/accumulo_configuration.py
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/package/scripts/accumulo_configuration.py b/app-packages/accumulo/package/scripts/accumulo_configuration.py
index b883c6f..fb4410e 100644
--- a/app-packages/accumulo/package/scripts/accumulo_configuration.py
+++ b/app-packages/accumulo/package/scripts/accumulo_configuration.py
@@ -39,8 +39,9 @@ def setup_conf_dir(name=None): # 'master' or 'tserver' or 'monitor' or 'gc' or '
     if os.path.exists(params.keystore_path) or os.path.exists(params.truststore_path):
       if os.path.exists(params.keystore_path) and os.path.exists(params.truststore_path):
         # assume keystores were already set up properly
-        return
-      self.fail_with_error("something went wrong when certs were created")
+        pass
+      else:
+        self.fail_with_error("something went wrong when certs were created")
 
     Directory( format("{params.conf_dir}/ssl"),
                owner = params.accumulo_user,

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f507a4c0/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloMonitorSSLIT.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloMonitorSSLIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloMonitorSSLIT.groovy
index 7a901ca..b47689a 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloMonitorSSLIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloMonitorSSLIT.groovy
@@ -20,7 +20,6 @@ package org.apache.slider.funtest.accumulo
 import groovy.util.logging.Slf4j
 import org.apache.slider.api.ClusterDescription
 import org.apache.slider.client.SliderClient
-import org.apache.slider.common.tools.SliderUtils
 import org.apache.slider.core.conf.ConfTree
 
 import javax.net.ssl.KeyManager

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f507a4c0/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloMonitorSSLLocalCertsIT.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloMonitorSSLLocalCertsIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloMonitorSSLLocalCertsIT.groovy
new file mode 100644
index 0000000..b4b8c03
--- /dev/null
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloMonitorSSLLocalCertsIT.groovy
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.funtest.accumulo
+
+import groovy.json.JsonSlurper
+import org.apache.accumulo.core.conf.Property
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.client.SliderClient
+import org.apache.slider.core.conf.ConfTree
+import org.apache.slider.funtest.framework.SliderShell
+
+class AccumuloMonitorSSLLocalCertsIT extends AccumuloMonitorSSLIT {
+  protected String templateName() {
+    return sysprop("test.app.resources.dir") + "/appConfig_monitor_ssl_local_certs.json"
+  }
+
+  protected ConfTree modifyTemplate(ConfTree confTree) {
+    confTree = super.modifyTemplate(confTree)
+    confTree.global.put("site.accumulo-site." + Property.RPC_SSL_KEYSTORE_PATH.toString(),
+      clientKeyStoreFile.getAbsolutePath())
+    confTree.global.put("site.accumulo-site." + Property.RPC_SSL_TRUSTSTORE_PATH.toString(),
+      trustStoreFile.getAbsolutePath())
+    return confTree
+  }
+
+  @Override
+  public String getClusterName() {
+    return "test_monitor_ssl_local_certs";
+  }
+
+  @Override
+  public String getDescription() {
+    return "Test enable monitor SSL with local certs $clusterName"
+  }
+
+  @Override
+  public void clusterLoadOperations(ClusterDescription cd, SliderClient sliderClient) {
+    super.clusterLoadOperations(cd, sliderClient)
+    File accumuloSiteFile = new File(TEST_APP_PKG_DIR, "accumulo-site.json")
+    SliderShell shell = registry(EXIT_SUCCESS,
+      [
+        ARG_GETCONF, "accumulo-site",
+        ARG_NAME, getClusterName(),
+        ARG_FORMAT, "json",
+        ARG_DEST, accumuloSiteFile.getAbsolutePath()
+      ])
+
+    logShell(shell)
+
+    def slurper = new JsonSlurper()
+    def siteconf = slurper.parse(accumuloSiteFile)
+    assert siteconf[Property.RPC_SSL_KEYSTORE_PATH.toString()] ==
+      clientKeyStoreFile.getAbsolutePath(), "unexpected keystore path"
+    assert siteconf[Property.RPC_SSL_TRUSTSTORE_PATH.toString()] ==
+      trustStoreFile.getAbsolutePath(), "unexpected truststore path"
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f507a4c0/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSSLTestBase.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSSLTestBase.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSSLTestBase.groovy
index 5c6ba72..be19583 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSSLTestBase.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSSLTestBase.groovy
@@ -22,14 +22,13 @@ import org.apache.accumulo.core.conf.Property
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.slider.common.tools.SliderUtils
 import org.apache.slider.core.conf.ConfTree
 import org.apache.slider.funtest.framework.AgentUploads
 import org.junit.Before
 
 class AccumuloSSLTestBase extends AccumuloBasicIT {
-  File trustStoreFile = new File(TEST_APP_PKG_DIR, "truststore.jks")
-  File clientKeyStoreFile = new File(TEST_APP_PKG_DIR, "keystore.jks")
+  protected static final File trustStoreFile = new File(TEST_APP_PKG_DIR, "truststore.jks")
+  protected static final File clientKeyStoreFile = new File(TEST_APP_PKG_DIR, "keystore.jks")
 
   protected String templateName() {
     return sysprop("test.app.resources.dir") + "/appConfig_ssl.json"


[14/23] git commit: rm some @VisibleForTesting attributes from SliderClientAPI

Posted by sm...@apache.org.
rm some @VisibleForTesting attributes from SliderClientAPI


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2c55c43c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2c55c43c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2c55c43c

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: 2c55c43cb510a585ebfc5d5f74395be762d5731f
Parents: 87987a5
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 14 13:30:15 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 14 13:30:15 2014 -0700

----------------------------------------------------------------------
 .../src/main/java/org/apache/slider/client/SliderClientAPI.java  | 4 ----
 1 file changed, 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2c55c43c/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
index cacf962..3d2c8cd 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
@@ -126,14 +126,12 @@ public interface SliderClientAPI extends Service {
    * Implement the list action: list all nodes
    * @return exit code of 0 if a list was created
    */
-  @VisibleForTesting
   int actionList(String clustername) throws IOException, YarnException;
 
   /**
    * Implement the islive action: probe for a cluster of the given name existing
    * @return exit code
    */
-  @VisibleForTesting
   int actionFlex(String name, ActionFlexArgs args) throws YarnException, IOException;
 
   /**
@@ -141,7 +139,6 @@ public interface SliderClientAPI extends Service {
    * in the filesystem. If the live param is set, it must be a live cluster
    * @return exit code
    */
-  @VisibleForTesting
   int actionExists(String name, boolean checkLive) throws YarnException, IOException;
 
   /**
@@ -175,7 +172,6 @@ public interface SliderClientAPI extends Service {
    * @throws YarnException
    * @throws IOException
    */
-  @VisibleForTesting
   int actionStatus(String clustername, ActionStatusArgs statusArgs)
       throws YarnException, IOException;
 


[21/23] git commit: SLIDER-521 funtest AppsThroughAgentIT failing: propagate option name

Posted by sm...@apache.org.
SLIDER-521 funtest AppsThroughAgentIT failing: propagate option name


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4febd773
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4febd773
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4febd773

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: 4febd77379b74bde27702d228d0690e218dbfc8d
Parents: 9783581
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 14 17:49:13 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 14 17:49:13 2014 -0700

----------------------------------------------------------------------
 .../org/apache/slider/funtest/framework/CommandTestBase.groovy | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4febd773/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 33e109a..86d15d4 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -523,13 +523,13 @@ abstract class CommandTestBase extends SliderTestUtils {
     ]
     
     maybeAddCommandOption(commands,
-        [ARG_COMP_OPT, SliderKeys.COMPONENT_AM],
+        [ARG_COMP_OPT, SliderKeys.COMPONENT_AM, SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME],
         SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME));
     maybeAddCommandOption(commands,
-        [ARG_COMP_OPT, SliderKeys.COMPONENT_AM],
+        [ARG_COMP_OPT, SliderKeys.COMPONENT_AM, SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH],
         SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH));
     maybeAddCommandOption(commands,
-        [ARG_COMP_OPT, SliderKeys.COMPONENT_AM],
+        [ARG_COMP_OPT, SliderKeys.COMPONENT_AM, SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL],
         SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL));
     commands.addAll(extraArgs)
     SliderShell shell = slider(LauncherExitCodes.EXIT_SUCCESS, commands)


[04/23] git commit: Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop

Posted by sm...@apache.org.
Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3ed76a5c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3ed76a5c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3ed76a5c

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: 3ed76a5cc6db20fd04b21a9fd4fb26e6e2a342e7
Parents: 2850539 6c43743
Author: tedyu <yu...@gmail.com>
Authored: Mon Oct 13 16:49:18 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Mon Oct 13 16:49:18 2014 -0700

----------------------------------------------------------------------
 .../funtest/accumulo/AccumuloReadWriteIT.groovy |  4 +-
 .../accumulo/AccumuloReadWriteSSLIT.groovy      |  4 +-
 .../memcached-win/package/scripts/memcached.py  |  1 -
 .../memcached/package/scripts/memcached.py      |  1 -
 .../storm-win/package/scripts/service.py        |  1 -
 app-packages/storm/package/scripts/service.py   |  9 +--
 .../core/providers/system.py                    | 28 +++++----
 .../TestDirectoryResource.py                    |  8 +--
 .../resource_management/TestFileResource.py     | 10 +--
 .../org/apache/slider/client/SliderClient.java  |  4 +-
 .../apache/slider/common/SliderXmlConfKeys.java |  2 +-
 .../slider/common/tools/ConfigHelper.java       | 20 +++++-
 .../utility/AbstractSliderLaunchedService.java  | 13 ++--
 .../slider/common/tools/TestConfigHelper.groovy | 65 ++++++++++++++++++++
 .../common/tools/TestConfigHelperHDFS.groovy    | 20 ------
 .../test/YarnZKMiniClusterTestBase.groovy       |  2 +-
 .../agent/TestAgentProviderService.java         | 13 ++--
 .../funtest/framework/CommandTestBase.groovy    |  5 +-
 .../funtest/basic/ClusterConnectivityIT.groovy  |  4 +-
 .../lifecycle/ClusterBuildDestroyIT.groovy      |  4 +-
 .../accumulo/funtest/AccumuloCIIT.groovy        |  4 +-
 .../funtest/FunctionalHBaseClusterIT.groovy     |  4 +-
 .../funtest/HBaseClusterBuildDestroyIT.groovy   |  4 +-
 23 files changed, 149 insertions(+), 81 deletions(-)
----------------------------------------------------------------------



[06/23] git commit: SLIDER-511 slider diagnostic command to list env variables and client configuration

Posted by sm...@apache.org.
SLIDER-511 slider diagnostic command to list env variables and client configuration


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5456a58d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5456a58d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5456a58d

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: 5456a58dff351141a96da633465361a0028f1395
Parents: 6c43743
Author: Steve Loughran <st...@apache.org>
Authored: Mon Oct 13 19:05:03 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Oct 13 19:05:03 2014 -0700

----------------------------------------------------------------------
 .../org/apache/slider/client/SliderClient.java  | 47 +++++++++++----
 .../common/params/ActionDiagnosticArgs.java     |  7 ++-
 .../slider/common/params/ActionResolveArgs.java |  1 -
 .../apache/slider/common/params/Arguments.java  | 22 +++----
 .../slider/common/tools/ConfigHelper.java       |  2 +-
 .../core/registry/SliderRegistryUtils.java      | 62 ++++++++++++++++++++
 .../TestStandaloneYarnRegistryAM.groovy         |  9 +++
 .../apache/slider/client/TestDiagnostics.groovy | 58 ++++++++++++++++++
 .../slider/registry/TestRegistryPaths.groovy    | 17 +++++-
 .../commands/DiagnosticsCommandIT.groovy        | 36 ++++++++++++
 10 files changed, 235 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5456a58d/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 750bfaa..e44db3b 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -109,6 +109,7 @@ import org.apache.slider.core.launch.RunningApplication;
 import org.apache.slider.core.main.RunService;
 import org.apache.slider.core.persist.ConfPersister;
 import org.apache.slider.core.persist.LockAcquireFailedException;
+import org.apache.slider.core.registry.SliderRegistryUtils;
 import org.apache.slider.core.registry.YarnAppListClient;
 import org.apache.slider.core.registry.docstore.ConfigFormat;
 import org.apache.slider.core.registry.docstore.PublishedConfigSet;
@@ -124,7 +125,6 @@ import org.apache.slider.providers.agent.AgentKeys;
 import org.apache.slider.providers.slideram.SliderAMClientProvider;
 import org.apache.slider.server.appmaster.SliderAppMaster;
 import org.apache.slider.server.appmaster.rpc.RpcBinder;
-import org.apache.slider.server.appmaster.security.SecurityConfiguration;
 import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
@@ -439,6 +439,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
    * Create the zookeeper node associated with the calling user and the cluster
    */
   @VisibleForTesting
+  @Deprecated
   public String createZookeeperNode(String clusterName, Boolean nameOnly) throws YarnException, IOException {
     String user = getUsername();
     String zkPath = ZKIntegration.mkClusterPath(user, clusterName);
@@ -511,8 +512,15 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
       log.warn("Filesystem returned false from delete() operation");
     }
 
-    if(!deleteZookeeperNode(clustername)) {
-      log.warn("Unable to perform node cleanup in Zookeeper.");
+    // rm the registry entry —do not let this block the destroy operations
+    String registryPath = SliderRegistryUtils.registryPathForInstance(
+        clustername);
+    try {
+      getRegistryOperations().delete(registryPath, true);
+    } catch (IOException e) {
+      log.warn("Error deleting {}: {} ", registryPath, e, e);
+    } catch (SliderException e) {
+      log.warn("Error binding to registry {} ", e, e);
     }
 
     List<ApplicationReport> instances = findAllLiveInstances(clustername);
@@ -531,9 +539,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
   
   @Override
   public int actionAmSuicide(String clustername,
-      ActionAMSuicideArgs args) throws
-                                                              YarnException,
-                                                              IOException {
+      ActionAMSuicideArgs args) throws YarnException, IOException {
     SliderClusterOperations clusterOperations =
       createClusterOperations(clustername);
     clusterOperations.amSuicide(args.message, args.exitcode, args.waittime);
@@ -2210,7 +2216,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
     // the arguments
     args.validate();
     RegistryOperations operations = getRegistryOperations();
-    String path = args.path;
+    String path = SliderRegistryUtils.resolvePath(args.path);
     ServiceRecordMarshal serviceRecordMarshal = new ServiceRecordMarshal();
     try {
       if (args.list) {
@@ -2351,7 +2357,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
   public int actionDiagnostic(ActionDiagnosticArgs diagnosticArgs) {
 		try {
 			if (diagnosticArgs.client) {
-				actionDiagnosticClient();
+				actionDiagnosticClient(diagnosticArgs);
 			} else if (SliderUtils.isSet(diagnosticArgs.application)) {
 				actionDiagnosticApplication(diagnosticArgs);
 			} else if (SliderUtils.isSet(diagnosticArgs.slider)) {
@@ -2442,7 +2448,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
 		//assign application name from param to each sub diagnostic function
 		diagnosticArgs.application = diagnosticArgs.all;
 		diagnosticArgs.slider = diagnosticArgs.all;
-		actionDiagnosticClient();
+		actionDiagnosticClient(diagnosticArgs);
 		actionDiagnosticApplication(diagnosticArgs);
 		actionDiagnosticSlider(diagnosticArgs);
 		actionDiagnosticYarn(diagnosticArgs);
@@ -2579,7 +2585,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
 		}
 	}
 
-	private void actionDiagnosticClient() throws SliderException, IOException {
+	private void actionDiagnosticClient(ActionDiagnosticArgs diagnosticArgs)
+      throws SliderException, IOException {
 		String currentCommandPath = SliderUtils.getCurrentCommandPath();
 		SliderVersionInfo.loadAndPrintVersionInfo(log);
 		String clientConfigPath = SliderUtils.getClientConfigPath();
@@ -2589,6 +2596,25 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
 				+ clientConfigPath);
 		log.info(jdkInfo);
 
+    // verbose?
+    if (diagnosticArgs.verbose) {
+      // do the environment
+      Map<String, String> env = System.getenv();
+      Set<String> envList = ConfigHelper.sortedConfigKeys(env.entrySet());
+      StringBuilder builder = new StringBuilder("Environment variables:\n");
+      for (String key : envList) {
+        builder.append(key)
+               .append("=")
+               .append(env.get(key))
+               .append("\n");
+      }
+      log.info(builder.toString());
+      
+      // then the config
+      log.info("Slider client configuration:\n" +
+               ConfigHelper.dumpConfigToString(getConfig()));
+    }
+    
 		try {
 			SliderUtils.validateSliderClientEnvironment(log);
 		} catch (SliderException e) {
@@ -2598,6 +2624,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
 			log.error(e.toString());
 			throw e;
 		}
+    
 	}
 
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5456a58d/slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java
index b2fde07..33abb30 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java
@@ -9,11 +9,12 @@ public class ActionDiagnosticArgs extends AbstractActionArgs
 {
 	  public static final String USAGE =
 	      "Usage: " + SliderActions.ACTION_DIAGNOSTIC
-	      + Arguments.ARG_CLIENT + "| "
+        + " "
+	      + Arguments.ARG_CLIENT + " | "
 	      + Arguments.ARG_SLIDER + " <appname> " + "| "
 	      + Arguments.ARG_APPLICATION + " <appname> " + "| "
-	      + Arguments.ARG_YARN + "| "
-	      + Arguments.ARG_CREDENTIALS + "| "
+	      + Arguments.ARG_YARN + " | "
+	      + Arguments.ARG_CREDENTIALS + " | "
 	      + Arguments.ARG_ALL + " <appname> " + "| "
 	      + Arguments.ARG_LEVEL + " <appname> "
 	      + " [" + Arguments.ARG_VERBOSE + "] ";

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5456a58d/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
index 4759bcb..2ee075a 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionResolveArgs.java
@@ -26,7 +26,6 @@ import org.apache.slider.core.exceptions.UsageException;
 
 import java.io.File;
 
-import static org.apache.slider.common.params.SliderActions.ACTION_REGISTRY;
 import static org.apache.slider.common.params.SliderActions.ACTION_RESOLVE;
 import static org.apache.slider.common.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5456a58d/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index 4deebd7..59c7095 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -21,16 +21,22 @@ package org.apache.slider.common.params;
 /**
  * Here are all the arguments that may be parsed by the client or server
  * command lines. 
+ * 
+ * Important: Please keep the main list in alphabetical order
+ * so it is easier to see what arguments are there
  */
 public interface Arguments {
-
+  String ARG_ALL = "--all";
+  String ARG_APPLICATION = "--application";
   String ARG_APP_HOME = "--apphome";
+  String ARG_CLIENT = "--client";
   String ARG_CONFDIR = "--appconf";
   String ARG_COMPONENT = "--component";
   String ARG_COMPONENT_SHORT = "--comp";
   String ARG_COMP_OPT= "--compopt";
   String ARG_COMP_OPT_SHORT = "--co";
-  
+  String ARG_CREDENTIALS = "--credentials";
+
   String ARG_DEBUG = "--debug";
   String ARG_DEFINE = "-D";
   String ARG_DEST = "--dest";
@@ -51,6 +57,7 @@ public interface Arguments {
   String ARG_ID = "--id";
   String ARG_IMAGE = "--image";
   String ARG_INTERNAL = "--internal";
+  String ARG_LEVEL = "--level";
   String ARG_LIST = "--list";
   String ARG_LISTFILES = "--listfiles";
   String ARG_LISTCONF = "--listconf";
@@ -58,14 +65,15 @@ public interface Arguments {
   String ARG_MANAGER = "--manager";
   String ARG_MANAGER_SHORT = "--m";
   String ARG_MESSAGE = "--message";
+  String ARG_NAME = "--name";
   String ARG_OPTION = "--option";
   String ARG_OPTION_SHORT = "-O";
-  String ARG_NAME = "--name";
   String ARG_OUTPUT = "--out";
   String ARG_OUTPUT_SHORT = "-o";
   String ARG_PACKAGE = "--package";
   String ARG_PATH = "--path";
   String ARG_PROVIDER = "--provider";
+  String ARG_QUEUE = "--queue";
   String ARG_RESOURCES = "--resources";
   String ARG_RES_COMP_OPT = "--rescompopt";
   String ARG_RES_COMP_OPT_SHORT = "--rco";
@@ -73,6 +81,7 @@ public interface Arguments {
   String ARG_RESOURCE_OPT = "--resopt";
   String ARG_RESOURCE_OPT_SHORT = "-ro";
   String ARG_SERVICETYPE = "--servicetype";
+  String ARG_SLIDER = "--slider";
   String ARG_SYSPROP = "-S";
   String ARG_TEMPLATE = "--template";
   String ARG_VERBOSE = "--verbose";
@@ -81,14 +90,7 @@ public interface Arguments {
   String ARG_ZKPORT = "--zkport";
   String ARG_ZKHOSTS = "--zkhosts";
   String ARG_REPLACE_PKG = "--replacepkg";
-  String ARG_CLIENT = "--client";
-  String ARG_SLIDER = "--slider";
-  String ARG_APPLICATION = "--application";
   String ARG_YARN = "--yarn";
-  String ARG_CREDENTIALS = "--credentials";
-  String ARG_ALL = "--all";
-  String ARG_LEVEL = "--level";
-  String ARG_QUEUE = "--queue";
 
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5456a58d/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
index 31acb94..b710224 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
@@ -416,7 +416,7 @@ public class ConfigHelper {
    * @return listing in key=value style
    */
   public static String dumpConfigToString(Configuration conf) {
-    TreeSet<String> sorted = sortedConfigKeys(conf);
+    Set<String> sorted = sortedConfigKeys(conf);
 
     StringBuilder builder = new StringBuilder();
     for (String key : sorted) {

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5456a58d/slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java b/slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
new file mode 100644
index 0000000..37b36ea
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.slider.common.SliderKeys;
+
+/**
+ * Miscellaneous methods to assist slider registry work
+ * 
+ */
+public class SliderRegistryUtils {
+
+
+  /**
+   * Get the registry path for an instance under the user's home node
+   * @param instanceName application instance
+   * @return a path to the registry location for this application instance.
+   */
+  public static String registryPathForInstance(String instanceName) {
+    return RegistryUtils.servicePath(
+        RegistryUtils.currentUser(), SliderKeys.APP_TYPE, instanceName
+    );
+  }
+
+  /**
+   * Process a path expanding it if needed.
+   * Validation is delegated to later as the core registry will need
+   * to do that anyway
+   * @param path path
+   * @return a path maybe with some expansion
+   */
+  public static String resolvePath(String path) {
+    Preconditions.checkArgument(path!=null, "null path");
+    Preconditions.checkArgument(!path.isEmpty(), "empty path");
+    String newpath = path;
+    if (path.startsWith("~/")) {
+      // add user expansion
+      newpath = RegistryUtils.homePathForCurrentUser() + path.substring(1);
+    } else if (path.equals("~")) {
+      newpath = RegistryUtils.homePathForCurrentUser();
+    }
+    return newpath;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5456a58d/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index e75cc57..a849be3 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -257,6 +257,15 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
     assert recordFromFile[YarnRegistryAttributes.YARN_PERSISTENCE] ==
            serviceRecord[YarnRegistryAttributes.YARN_PERSISTENCE]
 
+    //resolve to the home directory
+    assert 0 == client.actionResolve(new ActionResolveArgs(
+        path: "~", list:true))
+
+    //resolve to the instance
+    assert 0 == client.actionResolve(new ActionResolveArgs(
+        path: "~/services/org-apache-slider/$clustername",
+        list: true))
+
     // hit the registry web page
     def registryEndpoint = serviceRecord.getExternalEndpoint(
         CustomRegistryConstants.REGISTRY_REST_API)

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5456a58d/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy
new file mode 100644
index 0000000..57126f8
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.common.params.ActionDiagnosticArgs
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.test.SliderTestBase
+import org.apache.slider.test.YarnMiniClusterTestBase
+import org.apache.slider.test.YarnZKMiniClusterTestBase
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+class TestDiagnostics extends YarnZKMiniClusterTestBase {
+
+  @Test
+  public void testClientDiags() throws Throwable {
+    //launch fake master
+    String clustername = createMiniCluster("", configuration, 1, true)
+    ServiceLauncher<SliderClient> launcher = launchClientAgainstMiniMR(
+        //config includes RM binding info
+        new YarnConfiguration(miniCluster.config),
+        //varargs list of command line params
+        [SliderActions.ACTION_DIAGNOSTIC,
+         Arguments.ARG_CLIENT]
+    )
+    def client = launcher.service
+    def diagnostics = new ActionDiagnosticArgs()
+    diagnostics.client = true
+    diagnostics.verbose = true
+    describe("Verbose diagnostics")
+    
+    
+    client.actionDiagnostic(diagnostics)
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5456a58d/slider-core/src/test/groovy/org/apache/slider/registry/TestRegistryPaths.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/registry/TestRegistryPaths.groovy b/slider-core/src/test/groovy/org/apache/slider/registry/TestRegistryPaths.groovy
index be5ce78..5f95e0b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/registry/TestRegistryPaths.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/registry/TestRegistryPaths.groovy
@@ -19,6 +19,7 @@
 package org.apache.slider.registry
 
 import org.apache.hadoop.registry.client.binding.RegistryUtils
+import org.apache.slider.core.registry.SliderRegistryUtils
 import org.apache.slider.test.SliderTestUtils
 import org.junit.Assert
 import org.junit.Test
@@ -44,6 +45,20 @@ class TestRegistryPaths {
       SliderTestUtils.skip("homedir filtering not yet in hadoop registry module")
     }
   }
-  
+
+  @Test
+  public void testRegistryPathForInstance() throws Throwable {
+    def path = SliderRegistryUtils.registryPathForInstance("instance")
+    assert path.endsWith("/instance")
+  }
+
+  @Test
+  public void testPathResolution() throws Throwable {
+    def home = RegistryUtils.homePathForCurrentUser()
+    assert home == SliderRegistryUtils.resolvePath("~")
+    assert (home +"/") == SliderRegistryUtils.resolvePath("~/")
+    assert (home +"/something") == SliderRegistryUtils.resolvePath("~/something")
+    assert ("~unresolved") == SliderRegistryUtils.resolvePath("~unresolved")
+  }
   
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5456a58d/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy
new file mode 100644
index 0000000..b210f4b
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.commands
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.funtest.framework.CommandTestBase
+import org.junit.BeforeClass
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class DiagnosticsCommandIT extends CommandTestBase {
+
+  @Test
+  public void testListAll() throws Throwable {
+    assertSuccess(list(null))
+  }
+
+}


[13/23] git commit: Merge branch 'develop' into feature/SLIDER-509-registry-funtests

Posted by sm...@apache.org.
Merge branch 'develop' into feature/SLIDER-509-registry-funtests


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/87987a53
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/87987a53
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/87987a53

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: 87987a53737d30d91d933a98fbe14aac11b93a4f
Parents: 5df53ea d08a61d
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 14 10:55:13 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 14 10:55:13 2014 -0700

----------------------------------------------------------------------
 app-packages/accumulo/appConfig-default.json    | 43 +++++++---
 .../accumulo/configuration/accumulo-env.xml     | 89 ++++++++++++++++++++
 app-packages/accumulo/configuration/client.xml  | 44 ++++++++++
 app-packages/accumulo/configuration/global.xml  | 89 --------------------
 app-packages/accumulo/metainfo.xml              | 10 +++
 .../package/scripts/accumulo_configuration.py   | 14 ++-
 app-packages/accumulo/package/scripts/params.py | 33 +++++---
 .../accumulo/package/templates/client.conf.j2   | 18 ----
 app-packages/accumulo/resources-default.json    |  4 +-
 .../accumulo/AccumuloMonitorSSLIT.groovy        |  1 -
 .../AccumuloMonitorSSLLocalCertsIT.groovy       | 72 ++++++++++++++++
 .../funtest/accumulo/AccumuloReadWriteIT.groovy |  2 +-
 .../accumulo/AccumuloReadWriteSSLIT.groovy      |  2 +-
 .../funtest/accumulo/AccumuloSSLTestBase.groovy |  5 +-
 .../slider/core/launch/AbstractLauncher.java    |  3 +-
 .../server/appmaster/SliderAppMaster.java       |  3 +-
 .../core/launch/TestAppMasterLauncher.java      |  8 +-
 .../funtest/lifecycle/AgentFailures2IT.groovy   | 12 ++-
 18 files changed, 292 insertions(+), 160 deletions(-)
----------------------------------------------------------------------



[18/23] git commit: (reorder Arguments for alphabetical ordering)

Posted by sm...@apache.org.
(reorder Arguments for alphabetical ordering)


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/99fd3919
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/99fd3919
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/99fd3919

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: 99fd391937c168c8307a734c12ad6edbb826eee2
Parents: 518accd
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 14 16:12:05 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 14 16:12:05 2014 -0700

----------------------------------------------------------------------
 .../java/org/apache/slider/common/params/Arguments.java   | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/99fd3919/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index c4203a8..ff064c8 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -29,6 +29,7 @@ public interface Arguments {
   String ARG_ALL = "--all";
   String ARG_APPLICATION = "--application";
   String ARG_APP_HOME = "--apphome";
+  String ARG_BASE_PATH = "--basepath";
   String ARG_CLIENT = "--client";
   String ARG_CONFDIR = "--appconf";
   String ARG_COMPONENT = "--component";
@@ -48,7 +49,6 @@ public interface Arguments {
    */
   String ARG_FILESYSTEM = "--fs";
   String ARG_FILESYSTEM_LONG = "--filesystem";
-  String ARG_BASE_PATH = "--basepath";
   String ARG_FORMAT = "--format";
   String ARG_FORCE = "--force";
   String ARG_GETCONF = "--getconf";
@@ -59,8 +59,8 @@ public interface Arguments {
   String ARG_INTERNAL = "--internal";
   String ARG_LEVEL = "--level";
   String ARG_LIST = "--list";
-  String ARG_LISTFILES = "--listfiles";
   String ARG_LISTCONF = "--listconf";
+  String ARG_LISTFILES = "--listfiles";
   String ARG_LIVE = "--live";
   String ARG_MANAGER = "--manager";
   String ARG_MANAGER_SHORT = "--m";
@@ -74,6 +74,7 @@ public interface Arguments {
   String ARG_PATH = "--path";
   String ARG_PROVIDER = "--provider";
   String ARG_QUEUE = "--queue";
+  String ARG_REPLACE_PKG = "--replacepkg";
   String ARG_RESOURCES = "--resources";
   String ARG_RES_COMP_OPT = "--rescompopt";
   String ARG_RES_COMP_OPT_SHORT = "--rco";
@@ -87,11 +88,10 @@ public interface Arguments {
   String ARG_USER = "--user";
   String ARG_VERBOSE = "--verbose";
   String ARG_WAIT = "--wait";
+  String ARG_YARN = "--yarn";
+  String ARG_ZKHOSTS = "--zkhosts";
   String ARG_ZKPATH = "--zkpath";
   String ARG_ZKPORT = "--zkport";
-  String ARG_ZKHOSTS = "--zkhosts";
-  String ARG_REPLACE_PKG = "--replacepkg";
-  String ARG_YARN = "--yarn";
 
 
   /**


[15/23] git commit: SLIDER-509 more diagnostics logging

Posted by sm...@apache.org.
SLIDER-509 more diagnostics logging


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7dcdd1cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7dcdd1cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7dcdd1cb

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: 7dcdd1cb2a633e8d4562ff20a0b5f12c793e4ba4
Parents: 2c55c43
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 14 15:45:16 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 14 15:45:16 2014 -0700

----------------------------------------------------------------------
 .../org/apache/slider/client/SliderClient.java  | 32 +++++++++++++-------
 1 file changed, 21 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7dcdd1cb/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index e44db3b..2cbbeb1 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -2587,14 +2587,14 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
 
 	private void actionDiagnosticClient(ActionDiagnosticArgs diagnosticArgs)
       throws SliderException, IOException {
-		String currentCommandPath = SliderUtils.getCurrentCommandPath();
-		SliderVersionInfo.loadAndPrintVersionInfo(log);
-		String clientConfigPath = SliderUtils.getClientConfigPath();
-		String jdkInfo = SliderUtils.getJDKInfo();
-		log.info("The slider command path: " + currentCommandPath);
-		log.info("The slider-client.xml used by current running command path: "
-				+ clientConfigPath);
-		log.info(jdkInfo);
+    String currentCommandPath = SliderUtils.getCurrentCommandPath();
+    SliderVersionInfo.loadAndPrintVersionInfo(log);
+    String clientConfigPath = SliderUtils.getClientConfigPath();
+    String jdkInfo = SliderUtils.getJDKInfo();
+    println("The slider command path: " + currentCommandPath);
+    println("The slider-client.xml used by current running command path: "
+            + clientConfigPath);
+    println(jdkInfo);
 
     // verbose?
     if (diagnosticArgs.verbose) {
@@ -2608,11 +2608,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
                .append(env.get(key))
                .append("\n");
       }
-      log.info(builder.toString());
+      println(builder.toString());
       
       // then the config
-      log.info("Slider client configuration:\n" +
-               ConfigHelper.dumpConfigToString(getConfig()));
+      println("Slider client configuration:\n" +
+              ConfigHelper.dumpConfigToString(getConfig()));
     }
     
 		try {
@@ -2868,6 +2868,16 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
   private static void print(CharSequence src) {
     System.out.append(src);
   }
+
+  /**
+   * Output to standard out/stderr with a newline after
+   * @param src source
+   */
+  private static void println(CharSequence src) {
+    print(src);
+    print("\n");
+  }
+  
 }
 
 


[10/23] git commit: SLIDER-509 registry-funtests: rename cluster names for DNS compliance

Posted by sm...@apache.org.
SLIDER-509 registry-funtests: rename cluster names for DNS compliance


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a0601ff3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a0601ff3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a0601ff3

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: a0601ff3fea3d1632734e430f0fd13c1852c2529
Parents: 5456a58
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 14 10:46:51 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 14 10:46:51 2014 -0700

----------------------------------------------------------------------
 .../slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy       | 2 +-
 .../apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy  | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a0601ff3/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
index d048aca..1d01038 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
@@ -40,7 +40,7 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
   implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
 
 
-  static String CLUSTER = "test_agent_cluster_lifecycle"
+  static String CLUSTER = "test-agent-cluster-lifecycle"
 
   static String APP_RESOURCE2 = "../slider-core/src/test/app_packages/test_command_log/resources_no_role.json"
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a0601ff3/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
index e53473a..0581ed0 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
@@ -24,7 +24,6 @@ import org.apache.hadoop.fs.Path
 import org.apache.hadoop.registry.client.api.RegistryConstants
 import org.apache.slider.common.SliderExitCodes
 import org.apache.slider.common.SliderKeys
-import org.apache.slider.common.SliderXmlConfKeys
 import org.apache.slider.common.params.Arguments
 import org.apache.slider.common.params.SliderActions
 import org.apache.slider.funtest.framework.AgentCommandTestBase
@@ -39,7 +38,7 @@ public class ClusterBuildDestroyIT extends AgentCommandTestBase
     implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
 
 
-  static String CLUSTER = "test_cluster_build_destroy"
+  static String CLUSTER = "test-cluster-build-destroy"
   
 
   @BeforeClass


[16/23] git commit: SLIDER-516 registry command to take a user like "--user steve"

Posted by sm...@apache.org.
SLIDER-516 registry command to take a user like "--user steve"


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/baa79a74
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/baa79a74
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/baa79a74

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: baa79a7464e4513e6428d66a830341c09396b1e2
Parents: d08a61d
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 14 16:07:22 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 14 16:07:22 2014 -0700

----------------------------------------------------------------------
 .../org/apache/slider/client/SliderClient.java  |  9 ++++++++-
 .../common/params/ActionRegistryArgs.java       |  6 +++++-
 .../apache/slider/common/params/Arguments.java  |  1 +
 .../TestStandaloneYarnRegistryAM.groovy         | 20 +++++++++++++++++++-
 4 files changed, 33 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/baa79a74/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 750bfaa..2589814 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -2728,7 +2728,14 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
   private ServiceRecord lookupServiceRecord(ActionRegistryArgs registryArgs) throws
       SliderException,
       IOException {
-    String path = servicePath(currentUser(), registryArgs.serviceType,
+    String user;
+    if (StringUtils.isNotEmpty(registryArgs.user)) {
+      user = RegistryPathUtils.encodeForRegistry(registryArgs.user);
+    } else {
+      user = currentUser();
+    }
+
+    String path = servicePath(user, registryArgs.serviceType,
         registryArgs.name);
     return resolve(path);
   }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/baa79a74/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
index df40ef1..deae4eb 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionRegistryArgs.java
@@ -52,6 +52,7 @@ public class ActionRegistryArgs extends AbstractActionArgs {
       + Arguments.ARG_NAME + " <name> "
       + " )"
       + "[" + Arguments.ARG_VERBOSE + "] "
+      + "[" + Arguments.ARG_USER + "] "
       + "[" + Arguments.ARG_OUTPUT + " <filename> ] "
       + "[" + Arguments.ARG_SERVICETYPE + " <servicetype> ] "
       + "[" + Arguments.ARG_FORMAT + " <xml|json|properties>] "
@@ -121,7 +122,10 @@ public class ActionRegistryArgs extends AbstractActionArgs {
   @Parameter(names = {ARG_INTERNAL},
       description = "fetch internal registry entries")
   public boolean internal;
-  
+
+  @Parameter(names = {ARG_USER},
+      description = "the name of the user whose application is being resolved")
+  public String user;
   /**
    * validate health of all the different operations
    * @throws BadCommandArgumentsException

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/baa79a74/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index 4deebd7..1ea575e 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -75,6 +75,7 @@ public interface Arguments {
   String ARG_SERVICETYPE = "--servicetype";
   String ARG_SYSPROP = "-S";
   String ARG_TEMPLATE = "--template";
+  String ARG_USER = "--user";
   String ARG_VERBOSE = "--verbose";
   String ARG_WAIT = "--wait";
   String ARG_ZKPATH = "--zkpath";

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/baa79a74/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
index e75cc57..78b4514 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
@@ -23,6 +23,7 @@ import groovy.util.logging.Slf4j
 import org.apache.hadoop.fs.FileUtil
 import org.apache.hadoop.fs.PathNotFoundException
 import org.apache.hadoop.registry.client.binding.RegistryPathUtils
+import org.apache.hadoop.registry.client.binding.RegistryUtils
 import org.apache.hadoop.yarn.api.records.ApplicationReport
 import org.apache.hadoop.yarn.api.records.YarnApplicationState
 import org.apache.hadoop.yarn.conf.YarnConfiguration
@@ -61,7 +62,7 @@ import static org.apache.slider.core.registry.info.CustomRegistryConstants.*
 /**
  *  work with a YARN registry
  */
-@CompileStatic
+//@CompileStatic
 @Slf4j
 
 class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
@@ -448,6 +449,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
 
     def yarn_site_config = PublishedArtifacts.YARN_SITE_CONFIG
     registryArgs.getConf = yarn_site_config
+    registryArgs.user = currentUser()
 
     //properties format
     registryArgs.format = "properties"
@@ -463,6 +465,7 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
     registryArgs = new ActionRegistryArgs(clustername)
     registryArgs.getConf = yarn_site_config
     registryArgs.out = outputDir
+    registryArgs.user = ""
     describe registryArgs.toString()
     client.actionRegistry(registryArgs)
     assert new File(outputDir, yarn_site_config + ".xml").exists()
@@ -477,6 +480,21 @@ class TestStandaloneYarnRegistryAM extends AgentMiniClusterTestBase {
     registryArgs.getConf = unknownFilename
     assert SliderExitCodes.EXIT_NOT_FOUND == client.actionRegistry(registryArgs)
 
+    //look for a different user
+    try {
+      def args = new ActionRegistryArgs(
+          name: clustername,
+          user: "unknown",
+          serviceType: SliderKeys.APP_TYPE,
+          listConf: true,
+          verbose: true
+      )
+      client.actionRegistry(args)
+      fail("registry operation succeeded unexpectedly on $args")
+    } catch (UnknownApplicationInstanceException expected) {
+      // expected
+    }
+    
     describe "stop cluster"
     //now kill that cluster
     assert 0 == clusterActionFreeze(client, clustername)


[19/23] git commit: SLIDER-520 add funtests for diagnostics

Posted by sm...@apache.org.
SLIDER-520 add funtests for diagnostics


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c6c5955b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c6c5955b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c6c5955b

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: c6c5955b252fe640f20c70a60fbe201d939fba68
Parents: 99fd391
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 14 17:36:49 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 14 17:36:49 2014 -0700

----------------------------------------------------------------------
 slider-assembly/src/conf/log4j.properties       |   2 +
 .../org/apache/slider/client/SliderClient.java  | 103 ++++++++++++-------
 .../apache/slider/common/params/ClientArgs.java |   2 +-
 .../server/appmaster/SliderAppMaster.java       |   6 +-
 .../funtest/framework/CommandTestBase.groovy    |  15 +++
 .../slider/funtest/framework/SliderShell.groovy |  23 ++++-
 .../commands/DiagnosticsCommandIT.groovy        |   6 +-
 7 files changed, 107 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c6c5955b/slider-assembly/src/conf/log4j.properties
----------------------------------------------------------------------
diff --git a/slider-assembly/src/conf/log4j.properties b/slider-assembly/src/conf/log4j.properties
index 3c0d08c..b4a9aa4 100644
--- a/slider-assembly/src/conf/log4j.properties
+++ b/slider-assembly/src/conf/log4j.properties
@@ -56,3 +56,5 @@ log4j.logger.org.apache.hadoop.hdfs=WARN
 log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
 log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
 log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.curator.framework.state=ERROR
+log4j.logger.org.apache.curator.framework.imps=WARN

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c6c5955b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index d5725e0..50a7097 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -2585,46 +2585,60 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
 		}
 	}
 
-	private void actionDiagnosticClient(ActionDiagnosticArgs diagnosticArgs)
+  private void actionDiagnosticClient(ActionDiagnosticArgs diagnosticArgs)
       throws SliderException, IOException {
-    String currentCommandPath = SliderUtils.getCurrentCommandPath();
-    SliderVersionInfo.loadAndPrintVersionInfo(log);
-    String clientConfigPath = SliderUtils.getClientConfigPath();
-    String jdkInfo = SliderUtils.getJDKInfo();
-    println("The slider command path: " + currentCommandPath);
-    println("The slider-client.xml used by current running command path: "
-            + clientConfigPath);
-    println(jdkInfo);
-
-    // verbose?
-    if (diagnosticArgs.verbose) {
-      // do the environment
-      Map<String, String> env = System.getenv();
-      Set<String> envList = ConfigHelper.sortedConfigKeys(env.entrySet());
-      StringBuilder builder = new StringBuilder("Environment variables:\n");
-      for (String key : envList) {
-        builder.append(key)
-               .append("=")
-               .append(env.get(key))
-               .append("\n");
+    try {
+      String currentCommandPath = SliderUtils.getCurrentCommandPath();
+      SliderVersionInfo.loadAndPrintVersionInfo(log);
+      String clientConfigPath = SliderUtils.getClientConfigPath();
+      String jdkInfo = SliderUtils.getJDKInfo();
+      println("The slider command path: %s", currentCommandPath);
+      println("The slider-client.xml used by current running command path: %s",
+          clientConfigPath);
+      println(jdkInfo);
+
+      // security info
+      Configuration config = getConfig();
+      if (SliderUtils.isHadoopClusterSecure(config)) { 
+        println("Hadoop Cluster is secure");
+        println("Login user is %s", UserGroupInformation.getLoginUser());
+        println("Current user is %s", UserGroupInformation.getCurrentUser());
+
+
+      } else {
+        println("Hadoop Cluster is insecure");
       }
-      println(builder.toString());
-      
-      // then the config
-      println("Slider client configuration:\n" +
-              ConfigHelper.dumpConfigToString(getConfig()));
+
+
+      // verbose?
+      if (diagnosticArgs.verbose) {
+        // do the environment
+        Map<String, String> env = System.getenv();
+        Set<String> envList = ConfigHelper.sortedConfigKeys(env.entrySet());
+        StringBuilder builder = new StringBuilder("Environment variables:\n");
+        for (String key : envList) {
+          builder.append(key)
+                 .append("=")
+                 .append(env.get(key))
+                 .append("\n");
+        }
+        println(builder.toString());
+
+        // then the config
+        println("Slider client configuration:\n" +
+                ConfigHelper.dumpConfigToString(config));
+      }
+
+
+      SliderUtils.validateSliderClientEnvironment(log);
+    } catch (SliderException e) {
+      log.error(e.toString());
+      throw e;
+    } catch (IOException e) {
+      log.error(e.toString());
+      throw e;
     }
     
-		try {
-			SliderUtils.validateSliderClientEnvironment(log);
-		} catch (SliderException e) {
-			log.error(e.toString());
-			throw e;
-		} catch (IOException e) {
-			log.error(e.toString());
-			throw e;
-		}
-    
 	}
 
 
@@ -2878,13 +2892,22 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
 
   /**
    * Output to standard out/stderr with a newline after
-   * @param src source
+   * @param message message
    */
-  private static void println(CharSequence src) {
-    print(src);
+  private static void println(String message) {
+    print(message);
     print("\n");
   }
-  
+  /**
+   * Output to standard out/stderr with a newline after, formatted
+   * @param message message
+   * @param args arguments for string formatting
+   */
+  private static void println(String message, Object ... args) {
+    print(String.format(message, args));
+    print("\n");
+  }
+
 }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c6c5955b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
index a4265aa..6e03d4b 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
@@ -238,7 +238,7 @@ public class ClientArgs extends CommonArgs {
       bindCoreAction(actionRegistryArgs);
 
     } else if (SliderActions.ACTION_RESOLVE.equals(action)) {
-      bindCoreAction(actionRegistryArgs);
+      bindCoreAction(actionResolveArgs);
 
     } else if (SliderActions.ACTION_STATUS.equals(action)) {
       bindCoreAction(actionStatusArgs);

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c6c5955b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index d2a8936..ef4d69c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -412,10 +412,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
       log.debug("Authenticating as {}", ugi);
       SliderUtils.verifyPrincipalSet(conf,
           DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
-      // always enforce protocol to be token-based.
-      conf.set(
-        CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
-        SaslRpcServer.AuthMethod.TOKEN.toString());
+    } else {
+      log.info("Cluster is insecure");
     }
     log.info("Login user is {}", UserGroupInformation.getLoginUser());
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c6c5955b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index cfe3006..33e109a 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -167,6 +167,21 @@ abstract class CommandTestBase extends SliderTestUtils {
   }
 
   /**
+   * Print to system out
+   * @param string
+   */
+  static void println(String s) {
+    System.out.println(s)
+  }
+  /**
+   * Print to system out
+   * @param string
+   */
+  static void println() {
+    System.out.println()
+  }
+  
+  /**
    * Exec any slider command
    * @param conf
    * @param commands

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c6c5955b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
index 6effb46..a7fd58c 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/SliderShell.groovy
@@ -144,13 +144,29 @@ class SliderShell extends Shell {
     log.error(toString())
     log.error("return code = ${signCorrectReturnCode()}")
     if (out.size() != 0) {
-      log.info("\n<stdout>\n${out.join('\n')}\n</stdout>");
+      log.info("\n<stdout>\n${stdoutHistory}\n</stdout>");
     }
     if (err.size() != 0) {
-      log.error("\n<stderr>\n${err.join('\n')}\n</stderr>");
+      log.error("\n<stderr>\n${stdErrHistory}\n</stderr>");
     }
   }
-  
+
+  /**
+   * Get the stderr history
+   * @return the history
+   */
+  public String getStdErrHistory() {
+    return err.join('\n')
+  }
+
+  /**
+   * Get the stdout history
+   * @return the history
+   */
+  public String getStdoutHistory() {
+    return out.join('\n')
+  }
+
   /**
    * Assert the shell exited with a given error code
    * if not the output is printed and an assertion is raised
@@ -161,7 +177,6 @@ class SliderShell extends Shell {
       dumpOutput()
       throw new SliderException(ret,
           "Expected exit code of command ${command} : ${errorCode} - actual=${ret}")
-      
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c6c5955b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy
index 430a75f..1a4ae7a 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy
@@ -31,13 +31,17 @@ public class DiagnosticsCommandIT extends CommandTestBase {
 
   @Test
   public void testClientDiagnostics() throws Throwable {
-    slider(0,
+    def shell = slider(0,
         [
             SliderActions.ACTION_DIAGNOSTIC,
             Arguments.ARG_CLIENT,
             Arguments.ARG_VERBOSE
         ]
     )
+    println(shell.stdoutHistory)
+    println()
+    println(shell.stdErrHistory)
+    
   }
 
 }


[12/23] git commit: SLIDER-513 funtest AgentFailures2IT test is failing

Posted by sm...@apache.org.
SLIDER-513 funtest AgentFailures2IT test is failing


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d08a61df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d08a61df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d08a61df

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: d08a61df78fa1ff66fd3ca7550e1f8abb7e9fb1c
Parents: 73117e5
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 14 10:54:38 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 14 10:54:38 2014 -0700

----------------------------------------------------------------------
 .../slider/funtest/lifecycle/AgentFailures2IT.groovy    | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d08a61df/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
index a02fc0e..8a4140c 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
@@ -47,14 +47,12 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
 
   @Test
   public void testAgentFailHeartbeatingTwiceOnce() throws Throwable {
-    if (!AGENTTESTS_ENABLED) {
-      log.info "TESTS are not run."
-      return
-    }
-
+    assumeAgentTestsEnabled()
+    
     cleanup(APPLICATION_NAME)
-    SliderShell shell = createTemplatedSliderApplication(APP_TEMPLATE3,
-        ARG_RESOURCES,
+    SliderShell shell = createTemplatedSliderApplication(
+        APPLICATION_NAME,
+        APP_TEMPLATE3,
         APP_RESOURCE)
 
     logShell(shell)


[03/23] git commit: SLIDER-510 Correctly handle null check for yarnRegistryOperations in SliderAppMaster#unregisterComponent()

Posted by sm...@apache.org.
SLIDER-510 Correctly handle null check for yarnRegistryOperations in SliderAppMaster#unregisterComponent()


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/28505391
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/28505391
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/28505391

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: 285053911736b8948b3f4477af4f4558f85caa76
Parents: eba79a4
Author: tedyu <yu...@gmail.com>
Authored: Mon Oct 13 16:48:59 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Mon Oct 13 16:48:59 2014 -0700

----------------------------------------------------------------------
 .../java/org/apache/slider/server/appmaster/SliderAppMaster.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28505391/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index a887cf8..d2a8936 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -1087,9 +1087,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
    */
   public void unregisterComponent(ContainerId id) {
     log.info("Unregistering component {}", id);
-    if (yarnRegistryOperations== null) {
+    if (yarnRegistryOperations == null) {
       log.warn("Processing unregister component event before initialization " +
                "completed; init flag =" + initCompleted);
+      return;
     }
     String cid = RegistryPathUtils.encodeYarnID(id.toString());
     try {


[17/23] git commit: Merge branch 'develop' into feature/SLIDER-509-registry-funtests

Posted by sm...@apache.org.
Merge branch 'develop' into feature/SLIDER-509-registry-funtests


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/518accd0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/518accd0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/518accd0

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: 518accd05c0d15775335666245cfcda1f1d56b76
Parents: 7dcdd1c baa79a7
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 14 16:10:17 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 14 16:10:17 2014 -0700

----------------------------------------------------------------------
 .../org/apache/slider/client/SliderClient.java  |  9 ++++++++-
 .../common/params/ActionRegistryArgs.java       |  6 +++++-
 .../apache/slider/common/params/Arguments.java  |  1 +
 .../TestStandaloneYarnRegistryAM.groovy         | 20 +++++++++++++++++++-
 4 files changed, 33 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/518accd0/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/518accd0/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --cc slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index 59c7095,1ea575e..c4203a8
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@@ -81,9 -73,9 +81,10 @@@ public interface Arguments 
    String ARG_RESOURCE_OPT = "--resopt";
    String ARG_RESOURCE_OPT_SHORT = "-ro";
    String ARG_SERVICETYPE = "--servicetype";
 +  String ARG_SLIDER = "--slider";
    String ARG_SYSPROP = "-S";
    String ARG_TEMPLATE = "--template";
+   String ARG_USER = "--user";
    String ARG_VERBOSE = "--verbose";
    String ARG_WAIT = "--wait";
    String ARG_ZKPATH = "--zkpath";

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/518accd0/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneYarnRegistryAM.groovy
----------------------------------------------------------------------


[05/23] git commit: SLIDER-508 add client conf and accumulo-env as configurations for accumulo

Posted by sm...@apache.org.
SLIDER-508 add client conf and accumulo-env as configurations for accumulo


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7704bf00
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7704bf00
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7704bf00

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: 7704bf003f4be20e5971794320f6f3994164fa4c
Parents: 3ed76a5
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Mon Oct 13 19:02:05 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Mon Oct 13 19:02:40 2014 -0700

----------------------------------------------------------------------
 app-packages/accumulo/appConfig-default.json    | 43 +++++++---
 .../accumulo/configuration/accumulo-env.xml     | 89 ++++++++++++++++++++
 app-packages/accumulo/configuration/client.xml  | 44 ++++++++++
 app-packages/accumulo/configuration/global.xml  | 89 --------------------
 app-packages/accumulo/metainfo.xml              | 10 +++
 .../package/scripts/accumulo_configuration.py   |  9 +-
 app-packages/accumulo/package/scripts/params.py | 33 +++++---
 .../accumulo/package/templates/client.conf.j2   | 18 ----
 app-packages/accumulo/resources-default.json    |  4 +-
 .../funtest/accumulo/AccumuloReadWriteIT.groovy |  2 +-
 .../accumulo/AccumuloReadWriteSSLIT.groovy      |  2 +-
 11 files changed, 202 insertions(+), 141 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7704bf00/app-packages/accumulo/appConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/appConfig-default.json b/app-packages/accumulo/appConfig-default.json
index d5723da..3f9f26a 100644
--- a/app-packages/accumulo/appConfig-default.json
+++ b/app-packages/accumulo/appConfig-default.json
@@ -5,36 +5,50 @@
   "global": {
     "application.def": ".slider/package/ACCUMULO/${app.package.name}.zip",
     "java_home": "${app.java.home}",
-    "site.global.app_user": "${app.user}",
-    "site.global.app_log_dir": "${AGENT_LOG_ROOT}",
-    "site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
+
     "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}",
-    "site.global.tserver_heapsize": "256m",
-    "site.global.master_heapsize": "128m",
-    "site.global.monitor_heapsize": "64m",
-    "site.global.gc_heapsize": "64m",
-    "site.global.other_heapsize": "128m",
-    "site.global.hadoop_prefix": "${hadoop.dir}",
-    "site.global.hadoop_conf_dir": "/etc/hadoop/conf",
-    "site.global.zookeeper_home": "${zk.dir}",
-    "site.global.accumulo_instance_name": "${USER}-${CLUSTER_NAME}",
-    "site.global.accumulo_root_password": "NOT_USED",
+    "site.global.app_user": "${app.user}",
     "site.global.user_group": "${app.user.group}",
+
+    "site.accumulo-env.java_home": "${app.java.home}",
+    "site.accumulo-env.tserver_heapsize": "256m",
+    "site.accumulo-env.master_heapsize": "128m",
+    "site.accumulo-env.monitor_heapsize": "64m",
+    "site.accumulo-env.gc_heapsize": "64m",
+    "site.accumulo-env.other_heapsize": "128m",
+    "site.accumulo-env.hadoop_prefix": "${hadoop.dir}",
+    "site.accumulo-env.hadoop_conf_dir": "/etc/hadoop/conf",
+    "site.accumulo-env.zookeeper_home": "${zk.dir}",
+
+    "site.client.instance.name": "${USER}-${CLUSTER_NAME}",
+    "site.client.instance.zookeeper.host": "${@//site/accumulo-site/instance.zookeeper.host}",
+    "site.client.instance.rpc.ssl.enabled": "${@//site/accumulo-site/instance.rpc.ssl.enabled}",
+    "site.client.instance.rpc.ssl.clientAuth": "${@//site/accumulo-site/instance.rpc.ssl.clientAuth}",
+    "site.client.general.security.credential.provider.paths": "${@//site/accumulo-site/general.security.credential.provider.paths}",
+
+    "site.global.accumulo_root_password": "NOT_USED",
     "site.global.ssl_cert_dir": "ssl",
     "site.global.monitor_protocol": "http",
+
     "site.accumulo-site.instance.volumes": "${DEFAULT_DATA_DIR}/data",
     "site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
     "site.accumulo-site.instance.security.authenticator": "org.apache.slider.accumulo.CustomAuthenticator",
+
     "site.accumulo-site.general.security.credential.provider.paths": "jceks://hdfs/user/${USER}/accumulo-${CLUSTER_NAME}.jceks",
+    "site.accumulo-site.instance.rpc.ssl.enabled": "false",
+    "site.accumulo-site.instance.rpc.ssl.clientAuth": "false",
     "site.accumulo-site.general.kerberos.keytab": "${accumulo.keytab}",
     "site.accumulo-site.general.kerberos.principal": "${accumulo.principal}",
+
     "site.accumulo-site.tserver.memory.maps.native.enabled": "false",
     "site.accumulo-site.tserver.memory.maps.max": "80M",
     "site.accumulo-site.tserver.cache.data.size": "7M",
     "site.accumulo-site.tserver.cache.index.size": "20M",
-    "site.accumulo-site.trace.user": "root",
     "site.accumulo-site.tserver.sort.buffer.size": "50M",
     "site.accumulo-site.tserver.walog.max.size": "40M",
+
+    "site.accumulo-site.trace.user": "root",
+
     "site.accumulo-site.master.port.client": "0",
     "site.accumulo-site.trace.port.client": "0",
     "site.accumulo-site.tserver.port.client": "0",
@@ -43,6 +57,7 @@
     "site.accumulo-site.monitor.port.log4j": "0",
     "site.accumulo-site.master.replication.coordinator.port": "0",
     "site.accumulo-site.replication.receipt.service.port": "0",
+
     "site.accumulo-site.general.classpaths": "$ACCUMULO_HOME/lib/accumulo-server.jar,\n$ACCUMULO_HOME/lib/accumulo-core.jar,\n$ACCUMULO_HOME/lib/accumulo-start.jar,\n$ACCUMULO_HOME/lib/accumulo-fate.jar,\n$ACCUMULO_HOME/lib/accumulo-proxy.jar,\n$ACCUMULO_HOME/lib/[^.].*.jar,\n$ZOOKEEPER_HOME/zookeeper[^.].*.jar,\n$HADOOP_CONF_DIR,\n$HADOOP_PREFIX/[^.].*.jar,\n$HADOOP_PREFIX/lib/[^.].*.jar,\n$HADOOP_PREFIX/share/hadoop/common/.*.jar,\n$HADOOP_PREFIX/share/hadoop/common/lib/.*.jar,\n$HADOOP_PREFIX/share/hadoop/hdfs/.*.jar,\n$HADOOP_PREFIX/share/hadoop/mapreduce/.*.jar,\n$HADOOP_PREFIX/share/hadoop/yarn/.*.jar,\n${hadoop.dir}/.*.jar,\n${hadoop.dir}/lib/.*.jar,\n${hdfs.dir}/.*.jar,\n${mapred.dir}/.*.jar,\n${yarn.dir}/.*.jar,"
   },
   "credentials": {

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7704bf00/app-packages/accumulo/configuration/accumulo-env.xml
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/configuration/accumulo-env.xml b/app-packages/accumulo/configuration/accumulo-env.xml
new file mode 100644
index 0000000..65b6804
--- /dev/null
+++ b/app-packages/accumulo/configuration/accumulo-env.xml
@@ -0,0 +1,89 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+  <property>
+    <name>tserver_heapsize</name>
+    <value>256m</value>
+    <description>TServer heap size.</description>
+  </property>
+  <property>
+    <name>master_heapsize</name>
+    <value>128m</value>
+    <description>Master Heap Size</description>
+  </property>
+  <property>
+    <name>monitor_heapsize</name>
+    <value>64m</value>
+    <description>Monitor Heap Size</description>
+  </property>
+  <property>
+    <name>gc_heapsize</name>
+    <value>64m</value>
+    <description>GC Heap Size</description>
+  </property>
+  <property>
+    <name>other_heapsize</name>
+    <value>128m</value>
+    <description>Other Heap Size</description>
+  </property>
+  <property>
+    <name>hadoop_prefix</name>
+    <value>/usr/lib/hadoop</value>
+    <description>Hadoop directory.</description>
+  </property>
+  <property>
+    <name>hadoop_conf_dir</name>
+    <value>/etc/hadoop/conf</value>
+    <description>Hadoop configuration directory.</description>
+  </property>
+  <property>
+    <name>zookeeper_home</name>
+    <value>/usr/lib/zookeeper</value>
+    <description>Zookeeper directory.</description>
+  </property>
+  <property>
+    <name>accumulo_instance_name</name>
+    <value>accumulo-instance</value>
+    <description>Accumulo Instance Name.</description>
+  </property>
+  <property>
+    <name>content</name>
+    <description>This is the template for a client accumulo-env.sh file</description>
+    <value>
+#! /usr/bin/env bash
+export HADOOP_PREFIX=${@//site/accumulo-env/hadoop_prefix}
+export HADOOP_CONF_DIR=${@//site/accumulo-env/hadoop_conf_dir}
+export JAVA_HOME=${@//site/accumulo-env/java_home}
+export ZOOKEEPER_HOME=${@//site/accumulo-env/zookeeper_home}
+export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
+export ACCUMULO_TSERVER_OPTS="-Xmx${@//site/accumulo-env/tserver_heapsize} -Xms${@//site/accumulo-env/tserver_heapsize}"
+export ACCUMULO_MASTER_OPTS="-Xmx${@//site/accumulo-env/master_heapsize} -Xms${@//site/accumulo-env/master_heapsize}"
+export ACCUMULO_MONITOR_OPTS="-Xmx${@//site/accumulo-env/monitor_heapsize} -Xms${@//site/accumulo-env/monitor_heapsize}"
+export ACCUMULO_GC_OPTS="-Xmx${@//site/accumulo-env/gc_heapsize} -Xms${@//site/accumulo-env/gc_heapsize}"
+export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -Djava.net.preferIPv4Stack=true"
+export ACCUMULO_OTHER_OPTS="-Xmx${@//site/accumulo-env/other_heapsize} -Xms${@//site/accumulo-env/other_heapsize}"
+# what do when the JVM runs out of heap memory
+export ACCUMULO_KILL_CMD='kill -9 %p'
+    </value>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7704bf00/app-packages/accumulo/configuration/client.xml
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/configuration/client.xml b/app-packages/accumulo/configuration/client.xml
new file mode 100644
index 0000000..86e90d5
--- /dev/null
+++ b/app-packages/accumulo/configuration/client.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+  <property>
+    <name>instance.name</name>
+    <value>accumulo-instance</value>
+    <description>Accumulo Instance Name.</description>
+  </property>
+  <property>
+    <name>instance.zookeeper.host</name>
+    <value>localhost:2181</value>
+    <description>Zookeeper hosts.</description>
+  </property>
+  <property>
+    <name>instance.rpc.ssl.enabled</name>
+    <value>false</value>
+    <description>SSL enabled.</description>
+  </property>
+  <property>
+    <name>instance.rpc.ssl.clientAuth</name>
+    <value>false</value>
+    <description>SSL client auth enabled.</description>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7704bf00/app-packages/accumulo/configuration/global.xml
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/configuration/global.xml b/app-packages/accumulo/configuration/global.xml
deleted file mode 100644
index 904a645..0000000
--- a/app-packages/accumulo/configuration/global.xml
+++ /dev/null
@@ -1,89 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-
-<configuration>
-  <property>
-    <name>app_log_dir</name>
-    <value>/var/log/accumulo</value>
-    <description>Log Directories for Accumulo.</description>
-  </property>
-  <property>
-    <name>app_pid_dir</name>
-    <value>/var/run/accumulo</value>
-    <description>Pid Directories for Accumulo.</description>
-  </property>
-  <property>
-    <name>tserver_heapsize</name>
-    <value>128m</value>
-    <description>TServer heap size.</description>
-  </property>
-  <property>
-    <name>master_heapsize</name>
-    <value>128m</value>
-    <description>Master Heap Size</description>
-  </property>
-  <property>
-    <name>monitor_heapsize</name>
-    <value>64m</value>
-    <description>Monitor Heap Size</description>
-  </property>
-  <property>
-    <name>gc_heapsize</name>
-    <value>64m</value>
-    <description>GC Heap Size</description>
-  </property>
-  <property>
-    <name>other_heapsize</name>
-    <value>128m</value>
-    <description>Other Heap Size</description>
-  </property>
-  <property>
-    <name>accumulo_hdfs_root_dir</name>
-    <value>${DEFAULT_DATA_DIR}</value>
-    <description>Accumulo Relative Path to HDFS.</description>
-  </property>
-  <property>
-    <name>app_user</name>
-    <value>yarn</value>
-    <description>Accumulo User Name.</description>
-  </property>
-  <property>
-    <name>hadoop_prefix</name>
-    <value>/usr/lib/hadoop</value>
-    <description>Hadoop directory.</description>
-  </property>
-  <property>
-    <name>hadoop_conf_dir</name>
-    <value>/etc/hadoop/conf</value>
-    <description>Hadoop configuration directory.</description>
-  </property>
-  <property>
-    <name>zookeeper_home</name>
-    <value>/usr/lib/zookeeper</value>
-    <description>Zookeeper directory.</description>
-  </property>
-  <property>
-    <name>accumulo_instance_name</name>
-    <value>accumulo-instance</value>
-    <description>Accumulo Instance Name.</description>
-  </property>
-</configuration>

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7704bf00/app-packages/accumulo/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/metainfo.xml b/app-packages/accumulo/metainfo.xml
index a926561..d3fb263 100644
--- a/app-packages/accumulo/metainfo.xml
+++ b/app-packages/accumulo/metainfo.xml
@@ -158,6 +158,16 @@
         <fileName>accumulo-site.xml</fileName>
         <dictionaryName>accumulo-site</dictionaryName>
       </configFile>
+      <configFile>
+        <type>env</type>
+        <fileName>accumulo-env.sh</fileName>
+        <dictionaryName>accumulo-env</dictionaryName>
+      </configFile>
+      <configFile>
+        <type>properties</type>
+        <fileName>client.conf</fileName>
+        <dictionaryName>client</dictionaryName>
+      </configFile>
     </configFiles>
 
   </application>

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7704bf00/app-packages/accumulo/package/scripts/accumulo_configuration.py
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/package/scripts/accumulo_configuration.py b/app-packages/accumulo/package/scripts/accumulo_configuration.py
index 935d591..b883c6f 100644
--- a/app-packages/accumulo/package/scripts/accumulo_configuration.py
+++ b/app-packages/accumulo/package/scripts/accumulo_configuration.py
@@ -121,12 +121,17 @@ def setup_conf_dir(name=None): # 'master' or 'tserver' or 'monitor' or 'gc' or '
             owner = params.accumulo_user,
             group = params.user_group
     )
-    if params.ssl_enabled:
-      accumulo_TemplateConfig( 'client.conf')
 
   # create env file
   accumulo_TemplateConfig( 'accumulo-env.sh')
 
+  # create client.conf file
+  PropertiesFile(format("{params.conf_dir}/client.conf"),
+       properties = params.config['configurations']['client'],
+       owner = params.accumulo_user,
+       group = params.user_group
+  )
+
   # create host files
   accumulo_StaticFile( 'masters')
   accumulo_StaticFile( 'slaves')

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7704bf00/app-packages/accumulo/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/package/scripts/params.py b/app-packages/accumulo/package/scripts/params.py
index 7d0e52d..9e6e8dd 100644
--- a/app-packages/accumulo/package/scripts/params.py
+++ b/app-packages/accumulo/package/scripts/params.py
@@ -32,22 +32,20 @@ pid_dir = status_params.pid_dir
 
 # accumulo env
 java64_home = config['hostLevelParams']['java_home']
-hadoop_prefix = config['configurations']['global']['hadoop_prefix']
-hadoop_conf_dir = config['configurations']['global']['hadoop_conf_dir']
-zookeeper_home = config['configurations']['global']['zookeeper_home']
-master_heapsize = config['configurations']['global']['master_heapsize']
-tserver_heapsize = config['configurations']['global']['tserver_heapsize']
-monitor_heapsize = config['configurations']['global']['monitor_heapsize']
-gc_heapsize = config['configurations']['global']['gc_heapsize']
-other_heapsize = config['configurations']['global']['other_heapsize']
+hadoop_prefix = config['configurations']['accumulo-env']['hadoop_prefix']
+hadoop_conf_dir = config['configurations']['accumulo-env']['hadoop_conf_dir']
+zookeeper_home = config['configurations']['accumulo-env']['zookeeper_home']
+zookeeper_host = config['configurations']['accumulo-site']['instance.zookeeper.host']
+master_heapsize = config['configurations']['accumulo-env']['master_heapsize']
+tserver_heapsize = config['configurations']['accumulo-env']['tserver_heapsize']
+monitor_heapsize = config['configurations']['accumulo-env']['monitor_heapsize']
+gc_heapsize = config['configurations']['accumulo-env']['gc_heapsize']
+other_heapsize = config['configurations']['accumulo-env']['other_heapsize']
+env_sh_template = config['configurations']['accumulo-env']['content']
 
 # accumulo local directory structure
 accumulo_root = config['configurations']['global']['app_root']
-conf_dir = None
-if ('accumulo_conf_dir' in config['configurations']['global']):
-  conf_dir = config['configurations']['global']['accumulo_conf_dir']
-else:
-  conf_dir = format("{accumulo_root}/conf")
+conf_dir = format("{accumulo_root}/conf")
 log_dir = config['configurations']['global']['app_log_dir']
 daemon_script = format("{accumulo_root}/bin/accumulo")
 
@@ -60,6 +58,9 @@ monitor_truststore_property = "monitor.ssl.trustStore"
 ssl_enabled = False
 if 'instance.rpc.ssl.enabled' in config['configurations']['accumulo-site']:
   ssl_enabled = config['configurations']['accumulo-site']['instance.rpc.ssl.enabled']
+clientauth_enabled = False
+if 'instance.rpc.ssl.clientAuth' in config['configurations']['accumulo-site']:
+  clientauth_enabled = config['configurations']['accumulo-site']['instance.rpc.ssl.clientAuth']
 ssl_cert_dir = config['configurations']['global']['ssl_cert_dir']
 keystore_path = format("{conf_dir}/ssl/keystore.jks")
 truststore_path = format("{conf_dir}/ssl/truststore.jks")
@@ -67,9 +68,13 @@ ssl_keystore_file_property = "rpc.javax.net.ssl.keyStore"
 ssl_truststore_file_property = "rpc.javax.net.ssl.trustStore"
 credential_provider = config['configurations']['accumulo-site']["general.security.credential.provider.paths"]
 #credential_provider = credential_provider.replace("${HOST}", hostname) # if enabled, must propagate to configuration
+if ssl_keystore_file_property in config['configurations']['accumulo-site']:
+  keystore_path = config['configurations']['accumulo-site'][ssl_keystore_file_property]
+if ssl_truststore_file_property in config['configurations']['accumulo-site']:
+  truststore_path = config['configurations']['accumulo-site'][ssl_truststore_file_property]
 
 # accumulo initialization parameters
-accumulo_instance_name = config['configurations']['global']['accumulo_instance_name']
+accumulo_instance_name = config['configurations']['client']['instance.name']
 accumulo_root_password = config['configurations']['global']['accumulo_root_password']
 accumulo_hdfs_root_dir = config['configurations']['accumulo-site']['instance.volumes'].split(",")[0]
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7704bf00/app-packages/accumulo/package/templates/client.conf.j2
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/package/templates/client.conf.j2 b/app-packages/accumulo/package/templates/client.conf.j2
deleted file mode 100644
index 4e6a4be..0000000
--- a/app-packages/accumulo/package/templates/client.conf.j2
+++ /dev/null
@@ -1,18 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-instance.rpc.ssl.clientAuth true
-instance.rpc.ssl.enabled true
-general.security.credential.provider.paths {{credential_provider}}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7704bf00/app-packages/accumulo/resources-default.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/resources-default.json b/app-packages/accumulo/resources-default.json
index f9cae60..48bc772 100644
--- a/app-packages/accumulo/resources-default.json
+++ b/app-packages/accumulo/resources-default.json
@@ -5,7 +5,7 @@
   "global": {
     "yarn.log.include.patterns": "",
     "yarn.log.exclude.patterns": "",
-    "yarn.log.interval": "0"
+    "yarn.log.interval": "60"
   },
   "components": {
     "ACCUMULO_MASTER": {
@@ -18,7 +18,7 @@
     "ACCUMULO_TSERVER": {
       "yarn.role.priority": "2",
       "yarn.component.instances": "1",
-      "yarn.memory": "256"
+      "yarn.memory": "512"
     },
     "ACCUMULO_MONITOR": {
       "yarn.role.priority": "3",

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7704bf00/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteIT.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteIT.groovy
index 12a3bff..b4118d2 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteIT.groovy
@@ -54,7 +54,7 @@ class AccumuloReadWriteIT extends AccumuloBasicIT {
         FuntestProperties.DEFAULT_SLIDER_ZK_HOSTS)
 
       ZooKeeperInstance instance = new ZooKeeperInstance(
-        tree.global.get("site.global.accumulo_instance_name"), zookeepers)
+        tree.global.get("site.client.instance.name"), zookeepers)
       Connector connector = instance.getConnector(USER, new PasswordToken(PASSWORD))
 
       ingest(connector, 200000, 1, 50, 0);

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7704bf00/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteSSLIT.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteSSLIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteSSLIT.groovy
index cc227ed..0464cec 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteSSLIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteSSLIT.groovy
@@ -47,7 +47,7 @@ class AccumuloReadWriteSSLIT extends AccumuloSSLTestBase {
         RegistryConstants.KEY_REGISTRY_ZK_QUORUM,
       FuntestProperties.DEFAULT_SLIDER_ZK_HOSTS)
     ClientConfiguration conf = new ClientConfiguration()
-      .withInstance(tree.global.get("site.global.accumulo_instance_name"))
+      .withInstance(tree.global.get("site.client.instance.name"))
       .withZkHosts(zookeepers)
       .withSsl(true)
       .withKeystore(clientKeyStoreFile.toString(), KEY_PASS, null)


[20/23] git commit: Merge branch 'feature/SLIDER-509-registry-funtests' into develop

Posted by sm...@apache.org.
Merge branch 'feature/SLIDER-509-registry-funtests' into develop


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/97835815
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/97835815
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/97835815

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: 97835815c13f6711c1b00aea8f1f55c309910288
Parents: baa79a7 c6c5955
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 14 17:37:49 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 14 17:37:49 2014 -0700

----------------------------------------------------------------------
 slider-assembly/src/conf/log4j.properties       |   2 +
 .../org/apache/slider/client/SliderClient.java  | 114 ++++++++++++++-----
 .../apache/slider/client/SliderClientAPI.java   |   4 -
 .../common/params/ActionDiagnosticArgs.java     |   7 +-
 .../slider/common/params/ActionResolveArgs.java |   1 -
 .../apache/slider/common/params/Arguments.java  |  32 +++---
 .../apache/slider/common/params/ClientArgs.java |   2 +-
 .../slider/common/tools/ConfigHelper.java       |   2 +-
 .../core/registry/SliderRegistryUtils.java      |  62 ++++++++++
 .../server/appmaster/SliderAppMaster.java       |   6 +-
 .../TestStandaloneYarnRegistryAM.groovy         |   9 ++
 .../apache/slider/client/TestDiagnostics.groovy |  58 ++++++++++
 .../slider/registry/TestRegistryPaths.groovy    |  17 ++-
 .../framework/AgentCommandTestBase.groovy       |   7 +-
 .../funtest/framework/CommandTestBase.groovy    |  24 +++-
 .../funtest/framework/FuntestProperties.groovy  |   6 +
 .../slider/funtest/framework/SliderShell.groovy |  23 +++-
 .../commands/DiagnosticsCommandIT.groovy        |  47 ++++++++
 .../lifecycle/AgentClusterLifecycleIT.groovy    |   2 +-
 .../lifecycle/ClusterBuildDestroyIT.groovy      |   3 +-
 20 files changed, 360 insertions(+), 68 deletions(-)
----------------------------------------------------------------------



[11/23] git commit: SLIDER-509 agent launcher sets up homedir; add diagnostics command IT

Posted by sm...@apache.org.
SLIDER-509 agent launcher sets up homedir; add diagnostics command IT


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5df53ea3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5df53ea3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5df53ea3

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: 5df53ea37d9e1fda6c5f7bf841558a5cdd08bccd
Parents: a0601ff
Author: Steve Loughran <st...@apache.org>
Authored: Tue Oct 14 10:47:37 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Oct 14 10:47:37 2014 -0700

----------------------------------------------------------------------
 .../funtest/framework/AgentCommandTestBase.groovy      |  7 +++++--
 .../slider/funtest/framework/CommandTestBase.groovy    |  9 +++++++--
 .../slider/funtest/framework/FuntestProperties.groovy  |  6 ++++++
 .../funtest/commands/DiagnosticsCommandIT.groovy       | 13 ++++++++++---
 4 files changed, 28 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5df53ea3/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
index ec7d3e3..d5448bf 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
@@ -20,6 +20,7 @@ package org.apache.slider.funtest.framework
 
 import groovy.util.logging.Slf4j
 import org.apache.hadoop.fs.Path
+import org.apache.hadoop.security.UserGroupInformation
 import org.apache.slider.common.SliderExitCodes
 import org.apache.slider.common.params.Arguments
 import org.apache.slider.common.params.SliderActions
@@ -99,7 +100,9 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
   @BeforeClass
   public static void setupAgent() {
     assumeAgentTestsEnabled()
-
+    def uploader = new FileUploader(SLIDER_CONFIG, 
+        UserGroupInformation.currentUser)
+    uploader.mkHomeDir();
   }
 
   @Before
@@ -210,7 +213,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
 
     if (shell.ret != 0 && shell.ret != EXIT_UNKNOWN_INSTANCE) {
       logShell(shell)
-      assert fail("Old cluster either should not exist or should get destroyed.")
+      assert fail("Old cluster either should not exist or should get destroyed; destroy exit code = ${shell.ret}")
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5df53ea3/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 7f3c43d..cfe3006 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -579,8 +579,13 @@ abstract class CommandTestBase extends SliderTestUtils {
   }
 
   protected void ensureApplicationIsUp(String clusterName) {
-    repeatUntilTrue(this.&isApplicationUp, 15, 1000 * 3, ['arg1': clusterName],
-      true, 'Application did not start, aborting test.')
+    repeatUntilTrue(this.&isApplicationUp,
+        SLIDER_CONFIG.getInt(KEY_TEST_INSTANCE_LAUNCH_TIME,
+            DEFAULT_INSTANCE_LAUNCH_TIME_SECONDS),
+        1000,
+        ['arg1': clusterName],
+        true,
+        'Application did not start, aborting test.')
   }
 
   protected boolean isApplicationUp(Map<String, String> args) {

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5df53ea3/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
index 730639b..d9d6dd6 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
@@ -63,4 +63,10 @@ public interface FuntestProperties extends SliderXMLConfKeysForTesting {
 
   String AGENT_TAR_FILENAME = "slider-agent.tar.gz"
   String AGENT_SLIDER_GZ_IN_SLIDER_TAR = "agent/" + AGENT_TAR_FILENAME
+
+
+  String KEY_TEST_INSTANCE_LAUNCH_TIME =
+          "slider.test.instance.launch.wait.seconds";
+  int DEFAULT_INSTANCE_LAUNCH_TIME_SECONDS = 60 * 3;
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5df53ea3/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy
index b210f4b..430a75f 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/DiagnosticsCommandIT.groovy
@@ -20,8 +20,9 @@ package org.apache.slider.funtest.commands
 
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
 import org.apache.slider.funtest.framework.CommandTestBase
-import org.junit.BeforeClass
 import org.junit.Test
 
 @CompileStatic
@@ -29,8 +30,14 @@ import org.junit.Test
 public class DiagnosticsCommandIT extends CommandTestBase {
 
   @Test
-  public void testListAll() throws Throwable {
-    assertSuccess(list(null))
+  public void testClientDiagnostics() throws Throwable {
+    slider(0,
+        [
+            SliderActions.ACTION_DIAGNOSTIC,
+            Arguments.ARG_CLIENT,
+            Arguments.ARG_VERBOSE
+        ]
+    )
   }
 
 }


[02/23] git commit: Merge branch 'feature/SLIDER-504-registry-config-docs' into develop

Posted by sm...@apache.org.
Merge branch 'feature/SLIDER-504-registry-config-docs' into develop


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6c437439
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6c437439
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6c437439

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: 6c4374392302bc497a5a6bde859d81a63e3e5400
Parents: 0769354 9856b24
Author: Steve Loughran <st...@apache.org>
Authored: Mon Oct 13 16:20:04 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Oct 13 16:20:04 2014 -0700

----------------------------------------------------------------------
 .../funtest/accumulo/AccumuloReadWriteIT.groovy |  4 +-
 .../accumulo/AccumuloReadWriteSSLIT.groovy      |  4 +-
 .../org/apache/slider/client/SliderClient.java  |  4 +-
 .../apache/slider/common/SliderXmlConfKeys.java |  2 +-
 .../slider/common/tools/ConfigHelper.java       | 20 +++++-
 .../utility/AbstractSliderLaunchedService.java  | 13 ++--
 .../slider/common/tools/TestConfigHelper.groovy | 65 ++++++++++++++++++++
 .../common/tools/TestConfigHelperHDFS.groovy    | 20 ------
 .../test/YarnZKMiniClusterTestBase.groovy       |  2 +-
 .../agent/TestAgentProviderService.java         | 13 ++--
 .../funtest/framework/CommandTestBase.groovy    |  5 +-
 .../funtest/basic/ClusterConnectivityIT.groovy  |  4 +-
 .../lifecycle/ClusterBuildDestroyIT.groovy      |  4 +-
 .../accumulo/funtest/AccumuloCIIT.groovy        |  4 +-
 .../funtest/FunctionalHBaseClusterIT.groovy     |  4 +-
 .../funtest/HBaseClusterBuildDestroyIT.groovy   |  4 +-
 16 files changed, 123 insertions(+), 49 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6c437439/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
----------------------------------------------------------------------


[08/23] git commit: SLIDER-512 Build broken against changes to 2.6.0 YARN LogAggregationContext

Posted by sm...@apache.org.
SLIDER-512 Build broken against changes to 2.6.0 YARN LogAggregationContext


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f5ec0955
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f5ec0955
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f5ec0955

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: f5ec09556a3bf3dccdd04933a6acd8107a39a6fd
Parents: f507a4c
Author: Gour Saha <go...@apache.org>
Authored: Mon Oct 13 22:36:10 2014 -0700
Committer: Gour Saha <go...@apache.org>
Committed: Mon Oct 13 22:36:10 2014 -0700

----------------------------------------------------------------------
 .../main/java/org/apache/slider/core/launch/AbstractLauncher.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f5ec0955/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
index 658a03a..0d0f1c6 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -311,7 +311,8 @@ public abstract class AbstractLauncher extends Configured {
           .getOptionInt(ResourceKeys.YARN_LOG_INTERVAL, 0);
       log.info("Log interval: {}", logInterval);
       logAggregationContext = LogAggregationContext.newInstance(
-          logIncludePattern, logExcludePattern, logInterval);
+          logIncludePattern, logExcludePattern);
+      // log interval has been removed from signature - fix required
     }
   }
 


[09/23] git commit: SLIDER-512 Build broken against changes to 2.6.0 YARN LogAggregationContext

Posted by sm...@apache.org.
SLIDER-512 Build broken against changes to 2.6.0 YARN LogAggregationContext


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/73117e51
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/73117e51
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/73117e51

Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: 73117e516b50cb6c3623a3deeabe2b4ebd0aca29
Parents: f5ec095
Author: Gour Saha <go...@apache.org>
Authored: Mon Oct 13 22:38:53 2014 -0700
Committer: Gour Saha <go...@apache.org>
Committed: Mon Oct 13 22:38:53 2014 -0700

----------------------------------------------------------------------
 .../org/apache/slider/core/launch/TestAppMasterLauncher.java | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/73117e51/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java b/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
index ca1e9ec..f4fe53d 100644
--- a/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
+++ b/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
@@ -74,8 +74,8 @@ public class TestAppMasterLauncher {
     Assert.assertEquals(expectedExclude,
         appMasterLauncher.logAggregationContext.getExcludePattern());
 
-    Assert.assertEquals(30,
-        appMasterLauncher.logAggregationContext.getRollingIntervalSeconds());
+//    Assert.assertEquals(30,
+//        appMasterLauncher.logAggregationContext.getRollingIntervalSeconds());
 
     EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp);
   }
@@ -103,8 +103,8 @@ public class TestAppMasterLauncher {
     Assert.assertEquals(expectedExclude,
         appMasterLauncher.logAggregationContext.getExcludePattern());
 
-    Assert.assertEquals(600,
-        appMasterLauncher.logAggregationContext.getRollingIntervalSeconds());
+//    Assert.assertEquals(600,
+//        appMasterLauncher.logAggregationContext.getRollingIntervalSeconds());
 
     EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp);
   }