You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/08/19 14:22:15 UTC

[01/50] [abbrv] git commit: rm unused import from SliderAppMaster

Repository: incubator-slider
Updated Branches:
  refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options 258fb54c0 -> e8f772e7a (forced update)


rm unused import from SliderAppMaster


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d38b31a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d38b31a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d38b31a7

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: d38b31a7a0fc65309dd5723d7baf61093cd63d53
Parents: 7091397
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 12 17:30:50 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 12 17:30:50 2014 +0100

----------------------------------------------------------------------
 .../java/org/apache/slider/server/appmaster/SliderAppMaster.java    | 1 -
 1 file changed, 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d38b31a7/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 9f4567d..9a2851a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -132,7 +132,6 @@ import org.apache.slider.server.services.utility.WebAppService;
 import org.apache.slider.server.services.workflow.ServiceThreadFactory;
 import org.apache.slider.server.services.workflow.WorkflowExecutorService;
 import org.apache.slider.server.services.workflow.WorkflowRpcService;
-import org.apache.slider.server.services.workflow.WorkflowScheduledExecutorService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 


[36/50] [abbrv] git commit: SLIDER-320 TestAgentEcho starts two containers

Posted by st...@apache.org.
SLIDER-320 TestAgentEcho starts two containers


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4610c25e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4610c25e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4610c25e

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 4610c25e9f57ad980447bdd83f1d2f1563be467b
Parents: 01c7390
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 15 15:33:51 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 15 15:33:51 2014 +0100

----------------------------------------------------------------------
 .../groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4610c25e/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
index 304815e..a29c8cb 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
@@ -66,7 +66,7 @@ class TestAgentEcho extends AgentTestBase {
 
     def role = "echo"
     Map<String, Integer> roles = [
-        (role): 1,
+        (role): 2,
     ];
     ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
         roles,


[47/50] [abbrv] git commit: SLIDER-306 test checks are not consistent with current listing semantics: succeeds if live or not live cluster is in the history

Posted by st...@apache.org.
SLIDER-306 test checks are not consistent with current listing semantics: succeeds if live or not live cluster is in the history


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7b589d98
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7b589d98
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7b589d98

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 7b589d986e468b83669860e807b74bff0136125b
Parents: f06b37c
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 19 12:38:26 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 19 12:38:26 2014 +0100

----------------------------------------------------------------------
 .../slider/agent/actions/TestActionList.groovy  | 58 +++++++++++++++++++-
 1 file changed, 55 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7b589d98/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
index b7196ac..5ccaeca 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
@@ -23,6 +23,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport
 import org.apache.hadoop.yarn.conf.YarnConfiguration
 import org.apache.slider.agent.AgentMiniClusterTestBase
 import org.apache.slider.client.SliderClient
+import org.apache.slider.common.params.ActionListArgs
+import org.apache.slider.common.params.ActionThawArgs
 import org.apache.slider.common.params.Arguments
 import org.apache.slider.common.params.SliderActions
 import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
@@ -40,7 +42,7 @@ class TestActionList extends AgentMiniClusterTestBase {
   @Before
   public void setup() {
     super.setup()
-    createMiniCluster("", configuration, 1, false)
+    createMiniCluster("", configuration, 1, true)
   }
 
   /**
@@ -55,6 +57,7 @@ class TestActionList extends AgentMiniClusterTestBase {
     testListAllUsersNoClusters()
     testListLiveCluster()
     testListMissingCluster()
+//    testActionListHistory()
   }
   
   public void testListThisUserNoClusters() throws Throwable {
@@ -128,7 +131,7 @@ class TestActionList extends AgentMiniClusterTestBase {
   }
 
   public void testListMissingCluster() throws Throwable {
-    describe("exec the status command against an unknown cluster")
+    describe("exec the list command against an unknown cluster")
 
     ServiceLauncher<SliderClient> launcher
     try {
@@ -138,7 +141,7 @@ class TestActionList extends AgentMiniClusterTestBase {
           //varargs list of command line params
           [
               SliderActions.ACTION_LIST,
-              createClusterName()
+              "no-instance"
           ]
       )
       fail("expected an exception, got a status code " + launcher.serviceExitCode)
@@ -148,4 +151,53 @@ class TestActionList extends AgentMiniClusterTestBase {
   }
 
 
+  @Test
+  public void testActionListHistory() {
+    String clustername = createClusterName()
+    ServiceLauncher<SliderClient> launcher = createStandaloneAM(
+        clustername,
+        true,
+        true)
+    addToTeardown(launcher)
+    SliderClient sliderClient = launcher.service
+    waitForClusterLive(sliderClient)
+
+    ActionListArgs args = new ActionListArgs();
+    //Listing only live instances
+    args.live = true;
+    assert sliderClient.actionList(clustername, args) == 0;
+    clusterActionFreeze(sliderClient, clustername, "stopping first cluster")
+    waitForAppToFinish(sliderClient)
+
+    //Listing only live instances but prints nothing since instance is freezed/stopped
+    
+    args.live = true;
+    args.history = false;
+    try {
+      int exitCode = sliderClient.actionList(clustername, args);
+      fail("expected an exception, got a status code $exitCode")
+    } catch (UnknownApplicationInstanceException expected) {
+    }
+
+    // historical list will work
+    args.history = true;
+    assert sliderClient.actionList(clustername, args) == 0;
+
+    // thaw
+    ActionThawArgs thawArgs = new ActionThawArgs();
+    sliderClient.actionThaw(clustername, thawArgs);
+    waitForClusterLive(sliderClient)
+
+    //Listing only live instances
+    args.live = true;
+    args.history = false;
+    assert 0 == sliderClient.actionList(clustername, args);
+    assert launcher.serviceExitCode == 0
+
+    //Listing all the instance both history (previously freezed instance) and live
+    args.live = true
+    args.history = true
+    assert 0 == sliderClient.actionList("", args);
+  }
+
 }


[03/50] [abbrv] git commit: SLIDER-202 initial chaos monkey

Posted by st...@apache.org.
SLIDER-202 initial chaos monkey


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c951e251
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c951e251
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c951e251

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: c951e2519e3d4327bce04bc2ee826d3140f8c0ed
Parents: 8d4217e
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 13 10:18:01 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 13 10:18:01 2014 +0100

----------------------------------------------------------------------
 .../server/appmaster/SliderAppMaster.java       |  6 ++
 .../server/appmaster/monkey/ChaosEntry.java     | 35 ++++++++++++
 .../appmaster/monkey/ChaosMonkeyService.java    | 60 ++++++++++++++++++++
 .../server/appmaster/monkey/ChaosTarget.java    | 24 ++++++++
 4 files changed, 125 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c951e251/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 9a2851a..1135a58 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -18,6 +18,7 @@
 
 package org.apache.slider.server.appmaster;
 
+import com.codahale.metrics.MetricRegistry;
 import com.google.protobuf.BlockingService;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -196,6 +197,11 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
   public static final int NUM_RPC_HANDLERS = 5;
   public static final int SCHEDULED_EXECUTOR_POOL_SIZE = 1;
 
+  /**
+   * Singleton of metrics registry
+   */
+  public static final MetricRegistry metrics = new MetricRegistry();
+  
   /** YARN RPC to communicate with the Resource Manager or Node Manager */
   private YarnRPC yarnRPC;
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c951e251/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
new file mode 100644
index 0000000..779cca1
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.monkey;
+
+/**
+ * Entry in the chaos list
+ */
+public class ChaosEntry {
+  
+  public final String name;
+  public final ChaosTarget target;
+  public final long probability;
+
+  public ChaosEntry(String name, ChaosTarget target, long probability) {
+    this.name = name;
+    this.target = target;
+    this.probability = probability;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c951e251/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
new file mode 100644
index 0000000..7c5638c
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.monkey;
+
+import com.codahale.metrics.MetricRegistry;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.slider.server.appmaster.actions.QueueAccess;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ChaosMonkeyService extends AbstractService {
+  protected static final Logger log =
+      LoggerFactory.getLogger(ChaosMonkeyService.class);
+  private final MetricRegistry metrics;
+  private final QueueAccess queues;
+
+  private static final List<ChaosEntry> chaosEntries =
+      new ArrayList<ChaosEntry>();
+      
+  public ChaosMonkeyService(MetricRegistry metrics, QueueAccess queues) {
+    super("ChaosMonkeyService");
+    this.metrics = metrics;
+    this.queues = queues;
+  }
+
+  
+  @Override
+  protected void serviceStart() throws Exception {
+    super.serviceStart();
+    
+  }
+  
+  
+  public synchronized void addTarget(ChaosTarget target,  long probability) {
+    Preconditions.checkArgument(target != null, "null target");
+    Preconditions.checkArgument(probability > 0, "negative probability");
+    Preconditions.checkArgument(probability > 10000, "probability over 100%");
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c951e251/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosTarget.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosTarget.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosTarget.java
new file mode 100644
index 0000000..1c3a9ac
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosTarget.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.monkey;
+
+public interface ChaosTarget {
+
+  public void chaosAction();
+}


[39/50] [abbrv] git commit: SLIDER-324. Exports should be attributed to individual master components rather than only one master component

Posted by st...@apache.org.
SLIDER-324. Exports should be attributed to individual master components rather than only one master component


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2a25ab91
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2a25ab91
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2a25ab91

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 2a25ab9192bc96a2f6f4409f9eea916e94c9484c
Parents: bc9561d
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Fri Aug 15 17:18:52 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Fri Aug 15 17:18:52 2014 -0700

----------------------------------------------------------------------
 app-packages/storm/metainfo.xml                 |   4 +-
 .../providers/agent/AgentProviderService.java   | 216 +++++++++++--------
 .../application/metadata/MetainfoParser.java    |   8 +-
 .../agent/TestAgentProviderService.java         |  34 ++-
 .../metadata/MetainfoParserTest.java            |  59 -----
 .../metadata/TestMetainfoParser.java            |  63 ++++++
 6 files changed, 221 insertions(+), 163 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2a25ab91/app-packages/storm/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index d9004ad..dbe8549 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -45,7 +45,7 @@
             <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value>
           </export>
           <export>
-            <name>nimbus.url</name>
+            <name>nimbus.host_port</name>
             <value>http://${NIMBUS_HOST}:${site.storm-site.nimbus.thrift.port}</value>
           </export>
         </exports>
@@ -82,7 +82,7 @@
         <name>NIMBUS</name>
         <category>MASTER</category>
         <autoStartOnFailure>true</autoStartOnFailure>
-        <appExports>QuickLinks-nimbus.url,QuickLinks-ganglia.ui,QuickLinks-app.metrics</appExports>
+        <appExports>QuickLinks-nimbus.host_port,QuickLinks-ganglia.ui,QuickLinks-app.metrics</appExports>
         <commandScript>
           <script>scripts/nimbus.py</script>
           <scriptType>PYTHON</scriptType>

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2a25ab91/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 91bc0a6..842f4f6 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -56,6 +56,7 @@ import org.apache.slider.providers.ProviderRole;
 import org.apache.slider.providers.ProviderUtils;
 import org.apache.slider.providers.agent.application.metadata.Application;
 import org.apache.slider.providers.agent.application.metadata.Component;
+import org.apache.slider.providers.agent.application.metadata.ComponentExport;
 import org.apache.slider.providers.agent.application.metadata.Export;
 import org.apache.slider.providers.agent.application.metadata.ExportGroup;
 import org.apache.slider.providers.agent.application.metadata.Metainfo;
@@ -137,6 +138,8 @@ public class AgentProviderService extends AbstractProviderService implements
       new ConcurrentHashMap<String, ComponentInstanceState>();
   private final Map<String, Map<String, String>> componentInstanceData =
       new ConcurrentHashMap<String, Map<String, String>>();
+  private final Map<String, Map<String, String>> exportGroups =
+      new ConcurrentHashMap<String, Map<String, String>>();
   private final Map<String, Map<String, String>> allocatedPorts =
       new ConcurrentHashMap<String, Map<String, String>>();
   private final Map<String, String> workFolders =
@@ -451,22 +454,8 @@ public class AgentProviderService extends AbstractProviderService implements
     Boolean isMaster = isMaster(roleName);
     ComponentInstanceState componentStatus = getComponentStatuses().get(label);
     componentStatus.heartbeat(System.currentTimeMillis());
-    // If no Master can explicitly publish then publish if its a master
-    // Otherwise, wait till the master that can publish is ready
-    if (isMaster &&
-        (canAnyMasterPublishConfig() == false || canPublishConfig(roleName))) {
-      publishConfigAndExportGroups(heartBeat, componentStatus);
-    } else {
-      // Ack that config has been reported
-      List<ComponentStatus> statuses = heartBeat.getComponentStatus();
-      if (statuses != null && !statuses.isEmpty()) {
-        ComponentStatus status = statuses.get(0);
-        if(status.getConfigs().size() > 0) {
-          log.info("Config got reported by {} but discarded as component {} cannot publish.", label, roleName);
-          componentStatus.setConfigReported(true);
-        }
-      }
-    }
+
+    publishConfigAndExportGroups(heartBeat, componentStatus, roleName);
 
     List<CommandReport> reports = heartBeat.getReports();
     if (reports != null && !reports.isEmpty()) {
@@ -513,6 +502,7 @@ public class AgentProviderService extends AbstractProviderService implements
           }
         }
       }
+
       // if there is no outstanding command then retrieve config
       if (isMaster && componentStatus.getState() == State.STARTED
           && command == Command.NOP) {
@@ -758,39 +748,60 @@ public class AgentProviderService extends AbstractProviderService implements
 
   /**
    * Process return status for component instances
+   *
    * @param heartBeat
    * @param componentStatus
    */
-  protected void publishConfigAndExportGroups(HeartBeat heartBeat, ComponentInstanceState componentStatus) {
+  protected void publishConfigAndExportGroups(
+      HeartBeat heartBeat, ComponentInstanceState componentStatus, String roleName) {
     List<ComponentStatus> statuses = heartBeat.getComponentStatus();
     if (statuses != null && !statuses.isEmpty()) {
       log.info("Processing {} status reports.", statuses.size());
-      Application application = getMetainfo().getApplication();
-      List<ExportGroup> exportGroups = application.getExportGroups();
-      boolean hasExportGroups = exportGroups != null && !exportGroups.isEmpty();
-      Set<String> exportedConfigs = new HashSet();
-      String exportedConfigsStr = application.getExportedConfigs();
-      boolean exportedAllConfigs = exportedConfigsStr == null || exportedConfigsStr.isEmpty();
-      if (application.getExportedConfigs() != null && application.getExportedConfigs().length() > 0) {
-        for (String exportedConfig : application.getExportedConfigs().split(",")) {
-          if (exportedConfig.trim().length() > 0) {
-            exportedConfigs.add(exportedConfig.trim());
-          }
-        }
-      }
-
       for (ComponentStatus status : statuses) {
         log.info("Status report: " + status.toString());
+
         if (status.getConfigs() != null) {
-          for (String key : status.getConfigs().keySet()) {
-            if ((!exportedAllConfigs && exportedConfigs.contains(key)) ||
-                exportedAllConfigs) {
-              Map<String, String> configs = status.getConfigs().get(key);
-              publishApplicationInstanceData(key, key, configs.entrySet());
+          Application application = getMetainfo().getApplication();
+
+          if (canAnyMasterPublishConfig() == false || canPublishConfig(roleName)) {
+            // If no Master can explicitly publish then publish if its a master
+            // Otherwise, wait till the master that can publish is ready
+
+            Set<String> exportedConfigs = new HashSet();
+            String exportedConfigsStr = application.getExportedConfigs();
+            boolean exportedAllConfigs = exportedConfigsStr == null || exportedConfigsStr.isEmpty();
+            if (!exportedAllConfigs) {
+              for (String exportedConfig : exportedConfigsStr.split(",")) {
+                if (exportedConfig.trim().length() > 0) {
+                  exportedConfigs.add(exportedConfig.trim());
+                }
+              }
+            }
+
+            for (String key : status.getConfigs().keySet()) {
+              if ((!exportedAllConfigs && exportedConfigs.contains(key)) ||
+                  exportedAllConfigs) {
+                Map<String, String> configs = status.getConfigs().get(key);
+                publishApplicationInstanceData(key, key, configs.entrySet());
+              }
             }
           }
 
-          if (hasExportGroups) {
+          List<ExportGroup> exportGroups = application.getExportGroups();
+          boolean hasExportGroups = exportGroups != null && !exportGroups.isEmpty();
+
+          Set<String> appExports = new HashSet();
+          String appExportsStr = getApplicationComponent(roleName).getAppExports();
+          boolean hasNoAppExports = appExportsStr == null || appExportsStr.isEmpty();
+          if (!hasNoAppExports) {
+            for (String appExport : appExportsStr.split(",")) {
+              if (appExport.trim().length() > 0) {
+                appExports.add(appExport.trim());
+              }
+            }
+          }
+
+          if (hasExportGroups && appExports.size() > 0) {
             String configKeyFormat = "${site.%s.%s}";
             String hostKeyFormat = "${%s_HOST}";
 
@@ -809,33 +820,63 @@ public class AgentProviderService extends AbstractProviderService implements
               }
             }
 
+            Set<String> modifiedGroups = new HashSet<String>();
             for (ExportGroup exportGroup : exportGroups) {
               List<Export> exports = exportGroup.getExports();
               if (exports != null && !exports.isEmpty()) {
                 String exportGroupName = exportGroup.getName();
-                Map<String, String> map = new HashMap<String, String>();
+                Map<String, String> map = getCurrentExports(exportGroupName);
                 for (Export export : exports) {
-                  String value = export.getValue();
-                  // replace host names
-                  for (String token : replaceTokens.keySet()) {
-                    if (value.contains(token)) {
-                      value = value.replace(token, replaceTokens.get(token));
+                  if (canBeExported(exportGroupName, export.getName(), appExports)) {
+                    String value = export.getValue();
+                    // replace host names
+                    for (String token : replaceTokens.keySet()) {
+                      if (value.contains(token)) {
+                        value = value.replace(token, replaceTokens.get(token));
+                      }
                     }
+                    map.put(export.getName(), value);
+                    log.info("Preparing to publish. Key {} and Value {}", export.getName(), value);
                   }
-                  map.put(export.getName(), value);
-                  log.info("Preparing to publish. Key {} and Value {}", export.getName(), value);
                 }
-                publishApplicationInstanceData(exportGroupName, exportGroupName, map.entrySet());
+                modifiedGroups.add(exportGroupName);
               }
             }
+            publishModifiedExportGroups(modifiedGroups);
           }
-          log.info("Received and stored config for {}", heartBeat.getHostname());
+
+          log.info("Received and processed config for {}", heartBeat.getHostname());
           componentStatus.setConfigReported(true);
+
         }
       }
     }
   }
 
+  private boolean canBeExported(String exportGroupName, String name, Set<String> appExports) {
+    return  appExports.contains(String.format("%s-%s", exportGroupName, name));
+  }
+
+  protected Map<String, String> getCurrentExports(String groupName) {
+    if(!this.exportGroups.containsKey(groupName)) {
+       synchronized (this.exportGroups) {
+         if(!this.exportGroups.containsKey(groupName)) {
+           this.exportGroups.put(groupName, new ConcurrentHashMap<String, String>());
+         }
+       }
+    }
+
+    return this.exportGroups.get(groupName);
+  }
+
+  private void publishModifiedExportGroups(Set<String> modifiedGroups) {
+    synchronized (this.exportGroups) {
+      for(String groupName : modifiedGroups) {
+        publishApplicationInstanceData(groupName, groupName, this.exportGroups.get(groupName).entrySet());
+      }
+    }
+  }
+
   /** Publish component instance specific data if the component demands it */
   protected void processAndPublishComponentSpecificData(Map<String, String> ports,
                                                         String containerId,
@@ -848,9 +889,9 @@ public class AgentProviderService extends AbstractProviderService implements
     Application application = getMetainfo().getApplication();
     for (Component component : application.getComponents()) {
       if (component.getName().equals(roleName)) {
-        if (component.getExports().size() > 0) {
+        if (component.getComponentExports().size() > 0) {
 
-          for (Export export : component.getExports()) {
+          for (ComponentExport export : component.getComponentExports()) {
             String templateToExport = export.getValue();
             for (String portName : ports.keySet()) {
               boolean publishData = false;
@@ -901,26 +942,37 @@ public class AgentProviderService extends AbstractProviderService implements
   }
 
   /**
-   * Extract script path from the application metainfo
-   *
+   * Return Component based on name
    * @param roleName
-   *
    * @return
    */
-  protected String getScriptPathFromMetainfo(String roleName) {
-    String scriptPath = null;
+  protected Component getApplicationComponent(String roleName) {
     Application application = getMetainfo().getApplication();
     if (application == null) {
       log.error("Malformed app definition: Expect application as the top level element for metainfo.xml");
-      return scriptPath;
-    }
-    for (Component component : application.getComponents()) {
-      if (component.getName().equals(roleName)) {
-        scriptPath = component.getCommandScript().getScript();
-        break;
+    } else {
+      for (Component component : application.getComponents()) {
+        if (component.getName().equals(roleName)) {
+          return component;
+        }
       }
     }
-    return scriptPath;
+    return null;
+  }
+
+  /**
+   * Extract script path from the application metainfo
+   *
+   * @param roleName
+   *
+   * @return
+   */
+  protected String getScriptPathFromMetainfo(String roleName) {
+    Component component = getApplicationComponent(roleName);
+    if (component != null) {
+      return component.getCommandScript().getScript();
+    }
+    return null;
   }
 
   /**
@@ -931,18 +983,10 @@ public class AgentProviderService extends AbstractProviderService implements
    * @return
    */
   protected boolean isMaster(String roleName) {
-    Application application = getMetainfo().getApplication();
-    if (application == null) {
-      log.error("Malformed app definition: Expect application as the top level element for metainfo.xml");
-    } else {
-      for (Component component : application.getComponents()) {
-        if (component.getName().equals(roleName)) {
-          if (component.getCategory().equals("MASTER")) {
-            return true;
-          } else {
-            return false;
-          }
-        }
+    Component component = getApplicationComponent(roleName);
+    if (component != null) {
+      if (component.getCategory().equals("MASTER")) {
+        return true;
       }
     }
     return false;
@@ -956,34 +1000,24 @@ public class AgentProviderService extends AbstractProviderService implements
    * @return
    */
   protected boolean canPublishConfig(String roleName) {
-    Application application = getMetainfo().getApplication();
-    if (application == null) {
-      log.error("Malformed app definition: Expect application as the top level element for metainfo.xml");
-    } else {
-      for (Component component : application.getComponents()) {
-        if (component.getName().equals(roleName)) {
-          return Boolean.TRUE.toString().equals(component.getPublishConfig());
-        }
-      }
+    Component component = getApplicationComponent(roleName);
+    if (component != null) {
+      return Boolean.TRUE.toString().equals(component.getPublishConfig());
     }
     return false;
   }
 
   /**
    * Checks if the role is marked auto-restart
+   *
    * @param roleName
+   *
    * @return
    */
   protected boolean isMarkedAutoRestart(String roleName) {
-    Application application = getMetainfo().getApplication();
-    if (application == null) {
-      log.error("Malformed app definition: Expect application as the top level element for metainfo.xml");
-    } else {
-      for (Component component : application.getComponents()) {
-        if (component.getName().equals(roleName)) {
-          return component.getRequiresAutoRestart();
-        }
-      }
+    Component component = getApplicationComponent(roleName);
+    if (component != null) {
+      return component.getRequiresAutoRestart();
     }
     return false;
   }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2a25ab91/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
index 4b91ff7..bc93d6f 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
@@ -61,10 +61,10 @@ public class MetainfoParser {
     digester.addBeanPropertySetter("*/component/maxInstanceCount");
     digester.addBeanPropertySetter("*/component/autoStartOnFailure");
     digester.addBeanPropertySetter("*/component/appExports");
-    digester.addObjectCreate("*/component/componentExport", ComponentExport.class);
-    digester.addBeanPropertySetter("*/component/componentExport/name");
-    digester.addBeanPropertySetter("*/component/componentExport/value");
-    digester.addSetNext("*/component/componentExport", "addComponentExport");
+    digester.addObjectCreate("*/componentExport", ComponentExport.class);
+    digester.addBeanPropertySetter("*/componentExport/name");
+    digester.addBeanPropertySetter("*/componentExport/value");
+    digester.addSetNext("*/componentExport", "addComponentExport");
     digester.addSetNext("*/component", "addComponent");
 
     digester.addObjectCreate("*/commandScript", CommandScript.class);

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2a25ab91/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 1ea11a6..6ed950f 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -136,8 +136,18 @@ public class TestAgentProviderService {
                                                + "      </commandOrders>\n"
                                                + "      <components>\n"
                                                + "        <component>\n"
+                                               + "          <name>HBASE_REST</name>\n"
+                                               + "          <category>MASTER</category>\n"
+                                               + "          <commandScript>\n"
+                                               + "            <script>scripts/hbase_rest.py</script>\n"
+                                               + "            <scriptType>PYTHON</scriptType>\n"
+                                               + "            <timeout>600</timeout>\n"
+                                               + "          </commandScript>\n"
+                                               + "        </component>\n"
+                                               + "        <component>\n"
                                                + "          <name>HBASE_MASTER</name>\n"
                                                + "          <category>MASTER</category>\n"
+                                               + "          <publishConfig>true</publishConfig>\n"
                                                + "          <autoStartOnFailure>true</autoStartOnFailure>\n"
                                                + "          <appExports>QuickLinks-JMX_Endpoint,QuickLinks-Master_Status</appExports>\n"
                                                + "          <minInstanceCount>1</minInstanceCount>\n"
@@ -166,7 +176,7 @@ public class TestAgentProviderService {
                                                + "              <name>PropertyB</name>\n"
                                                + "              <value>AConstant</value>\n"
                                                + "            </componentExport>\n"
-                                               + "          </componentExport>\n"
+                                               + "          </componentExports>\n"
                                                + "        </component>\n"
                                                + "      </components>\n"
                                                + "      <osSpecifics>\n"
@@ -458,15 +468,15 @@ public class TestAgentProviderService {
     doReturn(metainfo).when(mockAps).getMetainfo();
     doReturn(roleClusterNodeMap).when(mockAps).getRoleClusterNodeMapping();
 
-    mockAps.publishConfigAndExportGroups(hb, componentStatus);
-    assert componentStatus.getConfigReported() == true;
+    mockAps.publishConfigAndExportGroups(hb, componentStatus, "HBASE_MASTER");
+    Assert.assertTrue(componentStatus.getConfigReported());
     ArgumentCaptor<Collection> entriesCaptor = ArgumentCaptor.
         forClass(Collection.class);
     Mockito.verify(mockAps, Mockito.times(3)).publishApplicationInstanceData(
         anyString(),
         anyString(),
         entriesCaptor.capture());
-    assert entriesCaptor.getAllValues().size() == 3;
+    Assert.assertEquals(3, entriesCaptor.getAllValues().size());
     for (Collection coll : entriesCaptor.getAllValues()) {
       Set<Map.Entry<String, String>> entrySet = (Set<Map.Entry<String, String>>) coll;
       for (Map.Entry entry : entrySet) {
@@ -476,6 +486,16 @@ public class TestAgentProviderService {
         }
       }
     }
+
+    Map<String, String> exports = mockAps.getCurrentExports("QuickLinks");
+    Assert.assertEquals(2, exports.size());
+    Assert.assertEquals(exports.get("JMX_Endpoint"), "http://HOST1:60012/jmx");
+
+    mockAps.publishConfigAndExportGroups(hb, componentStatus, "HBASE_REST");
+    Mockito.verify(mockAps, Mockito.times(3)).publishApplicationInstanceData(
+        anyString(),
+        anyString(),
+        entriesCaptor.capture());
   }
 
   @Test
@@ -487,7 +507,7 @@ public class TestAgentProviderService {
     log.info("Service: " + application.toString());
     Assert.assertEquals(application.getName(), "HBASE");
     Assert.assertEquals(application.getExportedConfigs(), "hbase-site,global");
-    Assert.assertEquals(application.getComponents().size(), 2);
+    Assert.assertEquals(application.getComponents().size(), 3);
     List<Component> components = application.getComponents();
     int found = 0;
     for (Component component : components) {
@@ -596,9 +616,9 @@ public class TestAgentProviderService {
 
     Assert.assertTrue(mockAps.isMaster(role_hm));
     Assert.assertFalse(mockAps.isMaster(role_hrs));
-    Assert.assertFalse(mockAps.canPublishConfig(role_hm));
+    Assert.assertTrue(mockAps.canPublishConfig(role_hm));
     Assert.assertFalse(mockAps.canPublishConfig(role_hrs));
-    Assert.assertFalse(mockAps.canAnyMasterPublishConfig());
+    Assert.assertTrue(mockAps.canAnyMasterPublishConfig());
 
     Assert.assertTrue(mockAps2.isMaster(role_hm));
     Assert.assertFalse(mockAps2.isMaster(role_hrs));

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2a25ab91/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/MetainfoParserTest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/MetainfoParserTest.java b/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/MetainfoParserTest.java
deleted file mode 100644
index 61c53df..0000000
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/MetainfoParserTest.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- *
- */
-public class MetainfoParserTest {
-  protected static final Logger log =
-      LoggerFactory.getLogger(MetainfoParserTest.class);
-  public static final String METAINFO_XML =
-      "/org/apache/slider/providers/agent/application/metadata/metainfo.xml";
-
-  @Test
-  public void testParse() throws IOException {
-
-    InputStream resStream = this.getClass().getResourceAsStream(
-        METAINFO_XML);
-    MetainfoParser parser = new MetainfoParser();
-    Metainfo metainfo = parser.parse(resStream);
-    Assert.assertNotNull(metainfo);
-    Assert.assertNotNull(metainfo.getApplication());
-    Application application = metainfo.getApplication();
-    assert "STORM".equals(application.getName());
-    assert 5 == application.getComponents().size();
-    OSPackage pkg = application.getOSSpecifics().get(0).getPackages().get(0);
-    assert "tarball".equals(pkg.getType());
-    assert "files/apache-storm-0.9.1.2.1.1.0-237.tar.gz".equals(pkg.getName());
-    boolean found = false;
-    for (Component comp : application.getComponents()) {
-      if (comp != null && comp.getName().equals("NIMBUS")) {
-        found = true;
-      }
-    }
-    assert found;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2a25ab91/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java b/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java
new file mode 100644
index 0000000..98f0afb
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ *
+ */
+public class TestMetainfoParser {
+  protected static final Logger log =
+      LoggerFactory.getLogger(TestMetainfoParser.class);
+  public static final String METAINFO_XML =
+      "/org/apache/slider/providers/agent/application/metadata/metainfo.xml";
+
+  @Test
+  public void testParse() throws IOException {
+
+    InputStream resStream = this.getClass().getResourceAsStream(
+        METAINFO_XML);
+    MetainfoParser parser = new MetainfoParser();
+    Metainfo metainfo = parser.parse(resStream);
+    Assert.assertNotNull(metainfo);
+    Assert.assertNotNull(metainfo.getApplication());
+    Application application = metainfo.getApplication();
+    assert "STORM".equals(application.getName());
+    assert 5 == application.getComponents().size();
+    OSPackage pkg = application.getOSSpecifics().get(0).getPackages().get(0);
+    assert "tarball".equals(pkg.getType());
+    assert "files/apache-storm-0.9.1.2.1.1.0-237.tar.gz".equals(pkg.getName());
+    boolean found = false;
+    for (Component comp : application.getComponents()) {
+      if (comp != null && comp.getName().equals("NIMBUS")) {
+        found = true;
+        Assert.assertEquals(0, comp.getComponentExports().size());
+      }
+      if (comp != null && comp.getName().equals("SUPERVISOR")) {
+        Assert.assertEquals(1, comp.getComponentExports().size());
+      }
+    }
+    assert found;
+  }
+}


[49/50] [abbrv] git commit: SLIDER-306 list tests

Posted by st...@apache.org.
SLIDER-306 list tests


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9645d56e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9645d56e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9645d56e

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 9645d56e339bcc423c6c84bf4ceba8e549c45246
Parents: fa289e8
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 19 13:15:39 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 19 13:15:39 2014 +0100

----------------------------------------------------------------------
 .../apache/slider/agent/actions/TestActionList.groovy   | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9645d56e/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
index b80658d..787d96d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
@@ -57,7 +57,7 @@ class TestActionList extends AgentMiniClusterTestBase {
     testListAllUsersNoClusters()
     testListLiveCluster()
     testListMissingCluster()
-//    testActionListHistory()
+    testActionListHistory()
   }
   
   public void testListThisUserNoClusters() throws Throwable {
@@ -151,7 +151,6 @@ class TestActionList extends AgentMiniClusterTestBase {
   }
 
 
-  @Test
   public void testActionListHistory() {
     String clustername = createClusterName()
     ServiceLauncher<SliderClient> launcher = createStandaloneAM(
@@ -173,9 +172,14 @@ class TestActionList extends AgentMiniClusterTestBase {
     
     args.live = true;
     args.history = false;
-    assert 0 == sliderClient.actionList(clustername, args);
+    try {
+      sliderClient.actionList(clustername, args);
+      fail("expected a failure")
+    } catch (UnknownApplicationInstanceException e) {
+    
+    }
 
-    // historical list will work
+      // historical list will work
     args.history = true;
     assert 0 == sliderClient.actionList(clustername, args)
 


[28/50] [abbrv] git commit: fix pom.xml for UTF-8 inputs and outputs (remove 1 maven warning)

Posted by st...@apache.org.
fix pom.xml for UTF-8 inputs and outputs (remove 1 maven warning)


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/15da2799
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/15da2799
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/15da2799

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 15da2799ea6f6f5107d19ff8f693f47b5c8307ae
Parents: 599e972
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 15 13:06:31 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 15 13:06:31 2014 +0100

----------------------------------------------------------------------
 pom.xml | 2 ++
 1 file changed, 2 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/15da2799/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 43f9231..223ff2b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -93,6 +93,8 @@
 
   <properties>
 
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
 
     <!-- 
     Java versions


[50/50] [abbrv] git commit: SLIDER-306 list exit code/exception depends on output of report is o

Posted by st...@apache.org.
SLIDER-306 list exit code/exception depends on output of report is o


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e8f772e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e8f772e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e8f772e7

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: e8f772e7a1411384c46fe9e0d2930aa0a1b54660
Parents: 9645d56
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 19 13:16:20 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 19 13:16:20 2014 +0100

----------------------------------------------------------------------
 .../main/java/org/apache/slider/client/SliderClient.java    | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e8f772e7/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index dc186ce..b0aa561 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1450,8 +1450,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
         ApplicationReport report =
             findClusterInInstanceList(instances, clustername);
         if (report != null) {
-          logAppReport(report, true, true);
-          instanceFound = true;
+          instanceFound = logAppReport(report, true, false);
         }
       }
       // exit code if the instance was found
@@ -1468,8 +1467,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
    * @param report report to log
    * @param live only list live apps
    * @param history list historical containers
+   * @return whether the report was logged or not
    */
-  public void logAppReport(ApplicationReport report, boolean live, boolean history) {
+  public boolean logAppReport(ApplicationReport report,
+      boolean live,
+      boolean history) {
     // app is active if it is accepted or running
     boolean active = isApplicationActive(report);
     
@@ -1477,6 +1479,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
     if (toLog) {
       log.info(SliderUtils.appReportToString(report, "\n"));
     }
+    return toLog;
   }
 
   /**


[19/50] [abbrv] git commit: SLIDER-322 POM-> java6

Posted by st...@apache.org.
SLIDER-322 POM-> java6


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b6782e1d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b6782e1d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b6782e1d

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: b6782e1dc0f2393746b91170e5e02a0c07616d2c
Parents: 280afd8
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 14 17:47:24 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 14 17:47:24 2014 +0100

----------------------------------------------------------------------
 pom.xml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b6782e1d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0816479..43f9231 100644
--- a/pom.xml
+++ b/pom.xml
@@ -97,7 +97,7 @@
     <!-- 
     Java versions
     -->
-    <project.java.src.version>1.7</project.java.src.version>
+    <project.java.src.version>1.6</project.java.src.version>
     <enforced.java.version>${project.java.src.version}</enforced.java.version>
     <groovy.version>2.2.2</groovy.version>
     
@@ -227,8 +227,8 @@
           <version>${maven-compiler-plugin.version}</version>
           <configuration>
             <compilerId>groovy-eclipse-compiler</compilerId>
-            <source>1.7</source>
-            <target>1.7</target>
+            <source>${project.java.src.version}</source>
+            <target>${project.java.src.version}</target>
             <!-- set verbose to be true if you want lots of uninteresting messages -->
             <!-- <verbose>true</verbose> -->
           </configuration>
@@ -248,7 +248,7 @@
             <configuration>
               <rules>
                 <requireJavaVersion>
-                  <version>[1.7,)</version>
+                  <version>[${project.java.src.version},)</version>
                 </requireJavaVersion>
               </rules>
             </configuration>


[04/50] [abbrv] git commit: SLIDER-202 Monkey implemented but not conditional/integrated

Posted by st...@apache.org.
SLIDER-202 Monkey implemented but not conditional/integrated


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b111268a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b111268a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b111268a

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: b111268af07a75797c9f56b3ab96c0dd052a62a8
Parents: c951e25
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 13 12:00:27 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 13 12:00:27 2014 +0100

----------------------------------------------------------------------
 .../server/appmaster/SliderAppMaster.java       | 52 +++++++++++----
 .../server/appmaster/actions/ActionHalt.java    |  4 +-
 .../appmaster/actions/ActionKillContainer.java  | 51 +++++++++++++++
 .../server/appmaster/monkey/ChaosEntry.java     | 53 ++++++++++++++-
 .../server/appmaster/monkey/ChaosKillAM.java    | 48 ++++++++++++++
 .../appmaster/monkey/ChaosKillContainer.java    | 68 ++++++++++++++++++++
 .../appmaster/monkey/ChaosMonkeyService.java    | 41 ++++++++----
 .../appmaster/monkey/MonkeyPlayAction.java      | 48 ++++++++++++++
 .../slider/server/appmaster/state/AppState.java | 14 +++-
 .../standalone/TestStandaloneAgentAM.groovy     |  2 +-
 10 files changed, 346 insertions(+), 35 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b111268a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 1135a58..ee0e59e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -97,6 +97,7 @@ import org.apache.slider.providers.ProviderService;
 import org.apache.slider.providers.SliderProviderFactory;
 import org.apache.slider.providers.slideram.SliderAMClientProvider;
 import org.apache.slider.providers.slideram.SliderAMProviderService;
+import org.apache.slider.server.appmaster.actions.ActionKillContainer;
 import org.apache.slider.server.appmaster.actions.QueueExecutor;
 import org.apache.slider.server.appmaster.actions.ActionHalt;
 import org.apache.slider.server.appmaster.actions.QueueService;
@@ -104,6 +105,9 @@ import org.apache.slider.server.appmaster.actions.ActionStopSlider;
 import org.apache.slider.server.appmaster.actions.AsyncAction;
 import org.apache.slider.server.appmaster.actions.RenewingAction;
 import org.apache.slider.server.appmaster.actions.ResetFailureWindow;
+import org.apache.slider.server.appmaster.monkey.ChaosKillAM;
+import org.apache.slider.server.appmaster.monkey.ChaosKillContainer;
+import org.apache.slider.server.appmaster.monkey.ChaosMonkeyService;
 import org.apache.slider.server.appmaster.operations.AsyncRMOperationHandler;
 import org.apache.slider.server.appmaster.operations.ProviderNotifyingOperationHandler;
 import org.apache.slider.server.appmaster.rpc.RpcBinder;
@@ -112,7 +116,6 @@ import org.apache.slider.server.appmaster.rpc.SliderClusterProtocolPBImpl;
 import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
 import org.apache.slider.server.appmaster.state.AppState;
 import org.apache.slider.server.appmaster.state.ContainerAssignment;
-import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
 import org.apache.slider.server.appmaster.state.ProviderAppState;
 import org.apache.slider.server.appmaster.operations.RMOperationHandler;
 import org.apache.slider.server.appmaster.state.RoleInstance;
@@ -147,7 +150,6 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutorService;
@@ -301,6 +303,11 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
   private ContainerId appMasterContainerID;
 
   /**
+   * Monkey Service -may be null
+   */
+  private ChaosMonkeyService monkey;
+  
+  /**
    * ProviderService of this cluster
    */
   @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
@@ -780,6 +787,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     // starts the node review process
     launchProviderService(instanceDefinition, confDir);
 
+    // chaos monkey
+    maybeStartMonkey();
 
     try {
       //now block waiting to be told to exit the process
@@ -1103,7 +1112,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     }
     
     //for all the operations, exec them
-    rmOperationHandler.execute(operations);
+    executeRMOperations(operations);
     log.info("Diagnostics: " + getContainerDiagnosticInfo());
   }
 
@@ -1221,7 +1230,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
       // tell the provider
       providerRMOperationHandler.execute(allOperations);
       //now apply the operations
-      rmOperationHandler.execute(allOperations);
+      executeRMOperations(allOperations);
       return !allOperations.isEmpty();
     } catch (TriggerClusterTeardownException e) {
 
@@ -1237,7 +1246,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
    */
   private void releaseAllContainers() {
     //now apply the operations
-    rmOperationHandler.execute(appState.releaseAllContainers());
+    executeRMOperations(appState.releaseAllContainers());
   }
 
   /**
@@ -1441,19 +1450,17 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     //throws NoSuchNodeException if it is missing
     RoleInstance instance =
       appState.getLiveInstanceByContainerID(containerID);
-    List<AbstractRMOperation> opsList =
-      new LinkedList<>();
-    ContainerReleaseOperation release =
-      new ContainerReleaseOperation(instance.getId());
-    opsList.add(release);
-    //now apply the operations
-    rmOperationHandler.execute(opsList);
+    queue(new ActionKillContainer(instance.getId(), 0, TimeUnit.MILLISECONDS));
     Messages.KillContainerResponseProto.Builder builder =
       Messages.KillContainerResponseProto.newBuilder();
     builder.setSuccess(true);
     return builder.build();
   }
 
+  public void executeRMOperations(List<AbstractRMOperation> operations) {
+    rmOperationHandler.execute(operations);
+  }
+
   @Override
   public Messages.AMSuicideResponseProto amSuicide(
       Messages.AMSuicideRequestProto request)
@@ -1462,7 +1469,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     String text = request.getText();
     int delay = request.getDelay();
     log.info("AM Suicide with signal {}, message {} delay = {}", signal, text, delay);
-    ActionHalt action = new ActionHalt(signal, text, delay);
+    ActionHalt action = new ActionHalt(signal, text, delay,
+        TimeUnit.MILLISECONDS);
     schedule(action);
     return Messages.AMSuicideResponseProto.getDefaultInstance();
   }
@@ -1537,7 +1545,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
         containerId);
     RoleInstance activeContainer = appState.getActiveContainer(containerId);
     if (activeContainer != null) {
-      rmOperationHandler.execute(appState.releaseContainer(containerId));
+      executeRMOperations(appState.releaseContainer(containerId));
       // ask for more containers if needed
       log.info("Container released; triggering review");
       reviewRequestAndReleaseNodes();
@@ -1725,6 +1733,22 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     signalAMComplete(exitCode, exception.toString());
   }
   
+  public boolean maybeStartMonkey() {
+    monkey = new ChaosMonkeyService(metrics, actionQueues);
+    int amKillProbability = 100;
+    int containerKillProbability = 200;
+    monkey.addTarget("AM killer", 
+        new ChaosKillAM(this, actionQueues, -1),
+        amKillProbability);
+    monkey.addTarget("Container killer", 
+        new ChaosKillContainer(this, actionQueues),
+        amKillProbability);
+    initAndAddService(monkey);
+    // and schedule it
+    schedule(monkey.getChaosAction(60, TimeUnit.SECONDS));
+    return true;
+  }
+  
   /**
    * This is the main entry point for the service launcher.
    * @param args command line arguments.

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b111268a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
index b46a791..c21e249 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
@@ -22,6 +22,8 @@ import org.apache.hadoop.util.ExitUtil;
 import org.apache.slider.server.appmaster.SliderAppMaster;
 import org.apache.slider.server.appmaster.state.AppState;
 
+import java.util.concurrent.TimeUnit;
+
 /**
  * Exit a JVM halt.
  * @see ExitUtil#halt(int, String) 
@@ -34,7 +36,7 @@ public class ActionHalt extends AsyncAction {
   public ActionHalt(
       int status,
       String text,
-      long delay) {
+      long delay, TimeUnit timeUnit) {
     super("Halt", delay, ActionAttributes.HALTS_CLUSTER);
     this.status = status;
     this.text = text;

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b111268a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
new file mode 100644
index 0000000..7668bd5
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
+import org.apache.slider.server.appmaster.state.AppState;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+public class ActionKillContainer extends AsyncAction {
+
+  private final ContainerId containerId;
+  
+  public ActionKillContainer(ContainerId containerId, long delay,
+      TimeUnit timeUnit) {
+    super("kill container", delay, timeUnit);
+    this.containerId = containerId;
+  }
+
+  @Override
+  public void execute(SliderAppMaster appMaster,
+      QueueAccess queueService,
+      AppState appState) throws Exception {
+    List<AbstractRMOperation> opsList = new LinkedList<>();
+    ContainerReleaseOperation release = new ContainerReleaseOperation(containerId);
+    opsList.add(release);
+    //now apply the operations
+    appMaster.executeRMOperations(opsList);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b111268a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
index 779cca1..13bda57 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
@@ -18,18 +18,67 @@
 
 package org.apache.slider.server.appmaster.monkey;
 
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.MetricRegistry;
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * Entry in the chaos list
  */
 public class ChaosEntry {
-  
+
+  protected static final Logger log =
+      LoggerFactory.getLogger(ChaosEntry.class);
   public final String name;
   public final ChaosTarget target;
   public final long probability;
 
-  public ChaosEntry(String name, ChaosTarget target, long probability) {
+  private final MetricRegistry metrics;
+  private final Counter invocationCounter;
+
+
+  /**
+   * Constructor -includes validaton of all arguments
+   * @param name
+   * @param target
+   * @param probability
+   */
+  public ChaosEntry(String name, ChaosTarget target, long probability,
+      MetricRegistry metrics) {
+    Preconditions.checkArgument(!StringUtils.isEmpty(name), "missing name");
+    Preconditions.checkArgument(target != null, "null target");
+    Preconditions.checkArgument(probability > 0, "negative probability");
+    Preconditions.checkArgument(probability < 10000, "probability over 100%");
     this.name = name;
     this.target = target;
     this.probability = probability;
+    this.metrics = metrics;
+    invocationCounter =
+        metrics.counter(MetricRegistry.name(ChaosEntry.class, name));
+  }
+
+  /**
+   * Trigger the chaos action
+   */
+  public void invokeChaos() {
+    log.info("Invoking {}", name);
+    invocationCounter.inc();
+    target.chaosAction();
+  }
+
+  /**
+   * Invoke Chaos if the trigger value is in range of the probability
+   * @param trigger trigger value, 0-10K
+   * @return true if the chaos method was invoked
+   */
+  public boolean maybeInvokeChaos(long trigger) {
+    if (probability < 0) {
+      invokeChaos();
+      return true;
+    }
+    return false;
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b111268a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java
new file mode 100644
index 0000000..c464ac3
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.monkey;
+
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.actions.ActionHalt;
+import org.apache.slider.server.appmaster.actions.QueueAccess;
+
+import java.util.concurrent.TimeUnit;
+
+public class ChaosKillAM implements ChaosTarget {
+
+  private final SliderAppMaster appMaster;
+  private final QueueAccess queues;
+  private final int exitCode;
+
+  public ChaosKillAM(SliderAppMaster appMaster,
+      QueueAccess queues, int exitCode) {
+    this.appMaster = appMaster;
+    this.queues = queues;
+    this.exitCode = exitCode;
+  }
+
+  /**
+   * Trigger a delayed halt
+   */
+  @Override
+  public void chaosAction() {
+    queues.schedule(new ActionHalt(exitCode, "Chaos invoked halt", 1000,
+        TimeUnit.MILLISECONDS));
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b111268a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
new file mode 100644
index 0000000..0ffcc8a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.monkey;
+
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.actions.ActionKillContainer;
+import org.apache.slider.server.appmaster.actions.QueueAccess;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Kill a container
+ */
+public class ChaosKillContainer implements ChaosTarget {
+  protected static final Logger log =
+      LoggerFactory.getLogger(ChaosKillContainer.class);
+  private final SliderAppMaster appMaster;
+  private final QueueAccess queues;
+  private final Random random = new Random();
+
+  public ChaosKillContainer(SliderAppMaster appMaster,
+      QueueAccess queues) {
+    this.appMaster = appMaster;
+    this.queues = queues;
+  }
+
+  /**
+   * Trigger a container kill halt
+   */
+  
+  @Override
+  public void chaosAction() {
+    List<RoleInstance> liveContainers =
+        appMaster.getAppState().cloneLiveContainerInfoList();
+    int size = liveContainers.size();
+    if (size == 0) {
+      log.info("No containers to kill");
+      return;
+    }
+    int target = random.nextInt(size);
+    RoleInstance roleInstance = liveContainers.get(target);
+    log.info("Killing {}", roleInstance);
+
+    queues.schedule(new ActionKillContainer(roleInstance.getId(),
+        100, TimeUnit.MILLISECONDS));
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b111268a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
index 7c5638c..c525d02 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
@@ -19,15 +19,20 @@
 package org.apache.slider.server.appmaster.monkey;
 
 import com.codahale.metrics.MetricRegistry;
-import com.google.common.base.Preconditions;
 import org.apache.hadoop.service.AbstractService;
+import org.apache.slider.server.appmaster.actions.AsyncAction;
 import org.apache.slider.server.appmaster.actions.QueueAccess;
+import org.apache.slider.server.appmaster.actions.RenewingAction;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
+/**
+ * A chaos monkey service which will invoke ChaosTarget events 
+ */
 public class ChaosMonkeyService extends AbstractService {
   protected static final Logger log =
       LoggerFactory.getLogger(ChaosMonkeyService.class);
@@ -36,25 +41,33 @@ public class ChaosMonkeyService extends AbstractService {
 
   private static final List<ChaosEntry> chaosEntries =
       new ArrayList<ChaosEntry>();
-      
+
   public ChaosMonkeyService(MetricRegistry metrics, QueueAccess queues) {
     super("ChaosMonkeyService");
     this.metrics = metrics;
     this.queues = queues;
   }
 
-  
-  @Override
-  protected void serviceStart() throws Exception {
-    super.serviceStart();
-    
+
+  public synchronized void addTarget(String name,
+      ChaosTarget target,
+      long probability) {
+
+    chaosEntries.add(new ChaosEntry(name, target, probability, metrics));
+  }
+
+  public void play() {
+
   }
-  
-  
-  public synchronized void addTarget(ChaosTarget target,  long probability) {
-    Preconditions.checkArgument(target != null, "null target");
-    Preconditions.checkArgument(probability > 0, "negative probability");
-    Preconditions.checkArgument(probability > 10000, "probability over 100%");
+
+  public RenewingAction<MonkeyPlayAction> getChaosAction(long time, TimeUnit timeUnit) {
+    RenewingAction<MonkeyPlayAction> action = new RenewingAction<>(
+        new MonkeyPlayAction(this, 0, TimeUnit.MILLISECONDS),
+        time,
+        time,
+        timeUnit,
+        0
+    );
+    return action;
   }
-  
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b111268a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/MonkeyPlayAction.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/MonkeyPlayAction.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/MonkeyPlayAction.java
new file mode 100644
index 0000000..20e4466
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/MonkeyPlayAction.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.monkey;
+
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.actions.AsyncAction;
+import org.apache.slider.server.appmaster.actions.QueueAccess;
+import org.apache.slider.server.appmaster.state.AppState;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Queueable action which calls {@link ChaosMonkeyService#play()} when
+ * executed.
+ */
+public class MonkeyPlayAction extends AsyncAction {
+
+  private final ChaosMonkeyService monkey;
+
+  public MonkeyPlayAction(ChaosMonkeyService monkey, long delay,
+      TimeUnit timeUnit) {
+    super("chaos monkey", delay, timeUnit);
+    this.monkey = monkey;
+  }
+
+  @Override
+  public void execute(SliderAppMaster appMaster,
+      QueueAccess queueService,
+      AppState appState) throws Exception {
+    monkey.play();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b111268a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 48f97bb..d616678 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -101,7 +101,7 @@ public class AppState {
    * Flag set to indicate the application is live -this only happens
    * after the buildInstance operation
    */
-  boolean applicationLive = false;
+  private boolean applicationLive = false;
 
   /**
    * The definition of the instance. Flexing updates the resources section
@@ -811,6 +811,10 @@ public class AppState {
     return activeContainers.remove(id);
   }
 
+  /**
+   * Clone the live container list. This is synchronized.
+   * @return a snapshot of the live node list
+   */
   public synchronized List<RoleInstance> cloneLiveContainerInfoList() {
     List<RoleInstance> allRoleInstances;
     Collection<RoleInstance> values = getLiveNodes().values();
@@ -818,8 +822,12 @@ public class AppState {
     return allRoleInstances;
   }
 
-
-
+  /**
+   * Lookup live instance by string value of container ID
+   * @param containerId container ID
+   * @return the role instance for that container
+   * @throws NoSuchNodeException if it does not exist
+   */
   public synchronized RoleInstance getLiveInstanceByContainerID(String containerId)
     throws NoSuchNodeException {
     Collection<RoleInstance> nodes = getLiveNodes().values();

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b111268a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
index 8498ee2..73e9b07 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
@@ -37,10 +37,10 @@ import org.junit.Test
 @CompileStatic
 @Slf4j
 class TestStandaloneAgentAM  extends AgentMiniClusterTestBase {
+  
   @Test
   public void testStandaloneAgentAM() throws Throwable {
 
-
     describe "create a masterless AM then get the service and look it up via the AM"
 
     //launch fake master


[16/50] [abbrv] git commit: Add get-hbase-site.py - python version of get-hbase-site.sh

Posted by st...@apache.org.
Add get-hbase-site.py - python version of get-hbase-site.sh


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9aa02997
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9aa02997
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9aa02997

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 9aa029973fffc7e9de80c1fd85b1c81b5c8ff36f
Parents: 52f1f6f
Author: tedyu <yu...@gmail.com>
Authored: Thu Aug 14 09:01:32 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Thu Aug 14 09:01:32 2014 -0700

----------------------------------------------------------------------
 app-packages/hbase/get-hbase-site.py | 48 +++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9aa02997/app-packages/hbase/get-hbase-site.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/get-hbase-site.py b/app-packages/hbase/get-hbase-site.py
new file mode 100644
index 0000000..9760781
--- /dev/null
+++ b/app-packages/hbase/get-hbase-site.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""Gets hbase-site.xml from running HBase instance
+First argument is the name of cluster instance
+"""
+
+import sys
+import urllib2
+import subprocess
+
+f=subprocess.Popen("slider status "+sys.argv[1], shell=True, stdout=subprocess.PIPE).stdout
+for line in f:
+  pos = line.find("info.am.web.url")
+  if pos > 0 :
+    part = line[(pos+20) :]
+    endPos = part.find("\"")
+    url = part[: (endPos-1)]
+    url = url + "/ws/v1/slider/publisher/slider/hbase-site.xml"
+    print url
+    response = urllib2.urlopen(url)
+    html = response.read()
+
+    fout=open("hbase-site.xml", "w")
+    fout.write(html)
+    fout.close()
+    f.close()
+
+    sys.exit(0)
+
+print "info.am.web.url key was not found for " + sys.argv[1]
+sys.exit(1)


[31/50] [abbrv] git commit: Merge branch 'develop' into feature/SLIDER-320-endpoints-in-roleinstances

Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-320-endpoints-in-roleinstances


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/95dcbc9c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/95dcbc9c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/95dcbc9c

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 95dcbc9c670bf919016a5266d3100b4f2b14f2fa
Parents: 74cca8a 4ea66fa
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 15 14:05:41 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 15 14:05:41 2014 +0100

----------------------------------------------------------------------
 app-packages/hbase/get-hbase-site.py            |  48 ++++++
 app-packages/memcached-win/appConfig.json       |   2 +-
 .../memcached-win/package/scripts/memcached.py  |   5 +-
 .../memcached-win/package/scripts/params.py     |   1 +
 app-packages/memcached/appConfig.json           |   1 +
 .../memcached/package/scripts/memcached.py      |   5 +-
 .../memcached/package/scripts/params.py         |   2 +
 app-packages/storm/appConfig.json               |   6 +-
 app-packages/storm/metainfo.xml                 |  12 ++
 app-packages/storm/package/scripts/service.py   |  24 +--
 .../storm/package/scripts/status_params.py      |   1 +
 pom.xml                                         |  10 +-
 .../src/main/python/agent/ActionQueue.py        |  26 ++-
 .../src/main/python/agent/CommandStatusDict.py  |   5 +
 slider-agent/src/main/python/agent/Constants.py |   1 +
 .../src/main/python/agent/Controller.py         |  50 +++++-
 .../python/agent/CustomServiceOrchestrator.py   |  81 +++++++---
 slider-agent/src/main/python/agent/Heartbeat.py |  15 +-
 slider-agent/src/main/python/agent/Register.py  |   7 +-
 .../core/providers/system.py                    |   3 +-
 .../core/providers/windows/system.py            |  11 +-
 .../core/resources/system.py                    |   4 +
 .../python/resource_management/core/shell.py    |  14 +-
 .../src/test/python/agent/TestActionQueue.py    |  10 +-
 .../test/python/agent/TestCommandStatusDict.py  |  94 +++++++++++
 .../src/test/python/agent/TestController.py     | 137 +++++++++++++++-
 .../agent/TestCustomServiceOrchestrator.py      |  18 ++-
 .../src/test/python/agent/TestHeartbeat.py      | 107 ++++++++++++-
 .../src/test/python/agent/TestRegistration.py   |   8 +-
 .../src/main/java/org/apache/slider/Slider.java |   2 +-
 .../apache/slider/api/ClusterDescription.java   |  14 +-
 .../java/org/apache/slider/api/OptionKeys.java  |   1 -
 .../org/apache/slider/client/SliderClient.java  | 100 +++++++++++-
 .../slider/client/SliderYarnClientImpl.java     |   8 +-
 .../common/params/AbstractActionArgs.java       |   6 +-
 .../params/AppAndResouceOptionArgsDelegate.java |   8 +-
 .../org/apache/slider/common/params/ArgOps.java |   8 +-
 .../apache/slider/common/params/CommonArgs.java |   4 +-
 .../common/params/DontSplitArguments.java       |   2 +-
 .../slider/common/tools/ConfigHelper.java       |  60 +++++--
 .../slider/common/tools/CoreFileSystem.java     |   4 +-
 .../apache/slider/common/tools/SliderUtils.java |  10 +-
 .../org/apache/slider/core/conf/ConfTree.java   |   6 +-
 .../slider/core/conf/ConfTreeOperations.java    |   4 +-
 .../apache/slider/core/conf/MapOperations.java  |   2 +-
 .../slider/core/launch/AbstractLauncher.java    |  10 +-
 .../core/launch/ClasspathConstructor.java       |   2 +-
 .../slider/core/launch/CommandLineBuilder.java  |   2 +-
 .../slider/core/main/ServiceLauncher.java       |   2 +-
 .../slider/core/persist/JsonSerDeser.java       |  27 +++-
 .../registry/docstore/PublishedConfigSet.java   |   2 +-
 .../docstore/PublishedConfiguration.java        |   4 +-
 .../PublishedConfigurationOutputter.java        |  12 ++
 .../slider/core/registry/docstore/UriMap.java   |   2 +-
 .../slider/core/registry/info/RegistryView.java |   3 +-
 .../core/registry/info/ServiceInstanceData.java |   4 +-
 .../apache/slider/core/zk/ZKIntegration.java    |   2 +-
 .../apache/slider/core/zk/ZookeeperUtils.java   |   6 +-
 .../providers/AbstractProviderService.java      |   4 +-
 .../apache/slider/providers/ProviderUtils.java  |   2 +-
 .../providers/agent/AgentClientProvider.java    |   8 +-
 .../providers/agent/AgentLaunchParameter.java   |   2 +-
 .../providers/agent/AgentProviderService.java   | 159 +++++++++++++------
 .../providers/agent/ComponentCommandOrder.java  |   9 +-
 .../agent/application/metadata/Application.java |   8 +-
 .../agent/application/metadata/Component.java   |  15 +-
 .../application/metadata/MetainfoParser.java    |   1 +
 .../slideram/SliderAMClientProvider.java        |  32 ++--
 .../slideram/SliderAMProviderService.java       |   2 +-
 .../server/appmaster/SliderAppMaster.java       |  37 +++--
 .../appmaster/actions/ActionKillContainer.java  |   2 +-
 .../server/appmaster/actions/QueueService.java  |   6 +-
 .../appmaster/monkey/ChaosMonkeyService.java    |   2 +-
 .../slider/server/appmaster/state/AppState.java |  56 +++----
 .../MostRecentContainerReleaseSelector.java     |   2 +-
 .../server/appmaster/state/NodeInstance.java    |   2 +-
 .../slider/server/appmaster/state/NodeMap.java  |   2 +-
 .../state/OutstandingRequestTracker.java        |   6 +-
 .../appmaster/state/ProviderAppState.java       |   4 +-
 .../server/appmaster/state/RoleHistory.java     |  15 +-
 .../server/appmaster/state/RoleStatus.java      |   2 +-
 .../server/appmaster/web/SliderAMWebApp.java    |   2 +-
 .../web/SliderAmFilterInitializer.java          |   2 +-
 .../server/appmaster/web/SliderAmIpFilter.java  |   8 +-
 .../server/appmaster/web/WebAppApiImpl.java     |   2 +-
 .../appmaster/web/rest/agent/HeartBeat.java     |  10 --
 .../web/rest/agent/HeartBeatResponse.java       |  11 ++
 .../appmaster/web/rest/agent/Register.java      |  58 +++++--
 .../web/rest/publisher/PublisherResource.java   |   4 +-
 .../slider/server/avro/RoleHistoryWriter.java   |   6 +-
 .../server/servicemonitor/MonitorUtils.java     |   2 +-
 .../services/curator/RegistryBinderService.java |   6 +-
 .../registry/RegistryRestResources.java         |   3 +-
 .../registry/SliderRegistryService.java         |   2 +-
 .../agent/TestAgentProviderService.java         |  96 +++++++----
 .../agent/TestComponentInstanceState.java       |  11 +-
 .../providers/agent/TestHeartbeatMonitor.java   |   4 +-
 .../management/TestAMManagementWebServices.java |   9 +-
 .../publisher/TestAgentProviderService.java     |   2 +-
 .../workflow/ProcessCommandFactory.java         |   6 +-
 .../slider/test/MiniZooKeeperCluster.java       |   6 +-
 .../funtest/framework/CommandTestBase.groovy    |  16 +-
 .../lifecycle/AgentClusterLifecycleIT.groovy    |  11 +-
 .../accumulo/AccumuloProviderService.java       |  19 ++-
 .../funtest/HBaseClusterLifecycleIT.groovy      |  16 +-
 .../hbase/funtest/HBaseIntegrationIT.groovy     |   5 +-
 .../providers/hbase/funtest/HBaseLoadIT.groovy  |   7 +
 .../providers/hbase/HBaseClientProvider.java    |   6 +-
 .../providers/hbase/HBaseProviderService.java   |  31 +++-
 .../clusters/sandbox/slider/slider-client.xml   |   2 +-
 110 files changed, 1350 insertions(+), 416 deletions(-)
----------------------------------------------------------------------



[24/50] [abbrv] git commit: SLIDER-323. Allocated ports must be reported back upon AM restart SLIDER-262. Slider agent should provide process supervision such as auto-restart

Posted by st...@apache.org.
SLIDER-323. Allocated ports must be reported back upon AM restart
SLIDER-262. Slider agent should provide process supervision such as auto-restart


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/47822dff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/47822dff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/47822dff

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 47822dffa3e086c198fed820ec1e4b099a453659
Parents: 18b0545
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Aug 14 17:36:59 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Aug 14 17:57:43 2014 -0700

----------------------------------------------------------------------
 .../src/main/python/agent/ActionQueue.py        | 11 ++-
 .../src/main/python/agent/CommandStatusDict.py  |  5 +-
 .../src/main/python/agent/Controller.py         | 27 ++++---
 .../python/agent/CustomServiceOrchestrator.py   |  8 +-
 slider-agent/src/main/python/agent/Heartbeat.py |  9 ++-
 slider-agent/src/main/python/agent/Register.py  |  7 +-
 .../src/test/python/agent/TestActionQueue.py    |  2 -
 .../src/test/python/agent/TestController.py     | 29 +++++--
 .../agent/TestCustomServiceOrchestrator.py      |  2 +
 .../src/test/python/agent/TestHeartbeat.py      | 82 +++++++++++++++++++-
 .../src/test/python/agent/TestRegistration.py   |  8 +-
 .../providers/agent/AgentProviderService.java   | 51 +++++++++---
 .../appmaster/web/rest/agent/HeartBeat.java     | 10 ---
 .../appmaster/web/rest/agent/Register.java      | 58 ++++++++++----
 .../agent/TestAgentProviderService.java         | 17 ++++
 15 files changed, 254 insertions(+), 72 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-agent/src/main/python/agent/ActionQueue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/ActionQueue.py b/slider-agent/src/main/python/agent/ActionQueue.py
index fba9aa2..4c45a76 100644
--- a/slider-agent/src/main/python/agent/ActionQueue.py
+++ b/slider-agent/src/main/python/agent/ActionQueue.py
@@ -122,7 +122,8 @@ class ActionQueue(threading.Thread):
 
     taskId = command['taskId']
 
-    reportResult = not command[Constants.AUTO_GENERATED]
+    # if auto generated then do not report result
+    reportResult = CommandStatusDict.shouldReportResult(command)
 
     # Preparing 'IN_PROGRESS' report
     in_progress_status = self.commandStatuses.generate_report_template(command)
@@ -140,7 +141,8 @@ class ActionQueue(threading.Thread):
     if ActionQueue.STORE_APPLIED_CONFIG in command['commandParams']:
       store_config = 'true' == command['commandParams'][ActionQueue.STORE_APPLIED_CONFIG]
     store_command = False
-    if ActionQueue.AUTO_RESTART in command['roleParams']:
+    if 'roleParams' in command and ActionQueue.AUTO_RESTART in command['roleParams']:
+      logger.info("Component has indicated auto-restart. Saving details from START command.")
       store_command = 'true' == command['roleParams'][ActionQueue.AUTO_RESTART]
 
 
@@ -195,10 +197,7 @@ class ActionQueue(threading.Thread):
       cluster = command['clusterName']
       service = command['serviceName']
       component = command['componentName']
-      reportResult = True
-      if Constants.AUTO_GENERATED in command:
-        reportResult = not command[Constants.AUTO_GENERATED]
-
+      reportResult = CommandStatusDict.shouldReportResult(command)
       component_status = self.customServiceOrchestrator.requestComponentStatus(command)
 
       result = {"componentName": component,

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-agent/src/main/python/agent/CommandStatusDict.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CommandStatusDict.py b/slider-agent/src/main/python/agent/CommandStatusDict.py
index f20e0fa..bcbce9e 100644
--- a/slider-agent/src/main/python/agent/CommandStatusDict.py
+++ b/slider-agent/src/main/python/agent/CommandStatusDict.py
@@ -114,7 +114,7 @@ class CommandStatusDict():
     grep = Grep()
     output = grep.tail(tmpout, Grep.OUTPUT_LAST_LINES)
     inprogress = self.generate_report_template(command)
-    reportResult = not command[Constants.AUTO_GENERATED]
+    reportResult = CommandStatusDict.shouldReportResult(command)
     inprogress.update({
       'stdout': grep.filterMarkup(output),
       'stderr': tmperr,
@@ -142,3 +142,6 @@ class CommandStatusDict():
     return stub
 
 
+  @staticmethod
+  def shouldReportResult(command):
+    return not (Constants.AUTO_GENERATED in command and command[Constants.AUTO_GENERATED])

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-agent/src/main/python/agent/Controller.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Controller.py b/slider-agent/src/main/python/agent/Controller.py
index 6bdfc26..1e27efa 100644
--- a/slider-agent/src/main/python/agent/Controller.py
+++ b/slider-agent/src/main/python/agent/Controller.py
@@ -118,7 +118,11 @@ class Controller(threading.Thread):
 
     while not self.isRegistered:
       try:
-        data = json.dumps(self.register.build(id))
+        data = json.dumps(self.register.build(
+          self.componentActualState,
+          self.componentExpectedState,
+          self.actionQueue.customServiceOrchestrator.allocated_ports,
+          id))
         logger.info("Registering with the server at " + self.registerUrl +
                     " with data " + pprint.pformat(data))
         response = self.sendRequest(self.registerUrl, data)
@@ -211,7 +215,7 @@ class Controller(threading.Thread):
       try:
         if not retry:
           data = json.dumps(
-            self.heartbeat.build(commandResult, self.componentActualState,
+            self.heartbeat.build(commandResult,
                                  self.responseId, self.hasMappedComponents))
           self.updateStateBasedOnResult(commandResult)
           logger.debug("Sending request: " + data)
@@ -225,8 +229,11 @@ class Controller(threading.Thread):
 
         serverId = int(response['responseId'])
 
+        restartEnabled = False
         if 'restartEnabled' in response:
-          restartEnabled = 'true' == response['restartEnabled']
+          restartEnabled = response['restartEnabled']
+          if restartEnabled:
+            logger.info("Component auto-restart is enabled.")
 
         if 'hasMappedComponents' in response.keys():
           self.hasMappedComponents = response['hasMappedComponents'] != False
@@ -262,15 +269,16 @@ class Controller(threading.Thread):
           pass
 
         # Add a start command
-        if self.componentActualState == State.INSTALLED and \
+        if self.componentActualState == State.FAILED and \
                 self.componentExpectedState == State.STARTED and restartEnabled:
           stored_command = self.actionQueue.customServiceOrchestrator.stored_command
           if len(stored_command) > 0:
-             auto_start_command = self.create_start_command(stored_command)
-             if auto_start_command:
-               logger.info("Automatically adding a start command.")
-               self.updateStateBasedOnCommand([auto_start_command], False)
-               self.addToQueue([auto_start_command])
+            auto_start_command = self.create_start_command(stored_command)
+            if auto_start_command:
+              logger.info("Automatically adding a start command.")
+              logger.debug("Auto start command: " + pprint.pformat(auto_start_command))
+              self.updateStateBasedOnCommand([auto_start_command], False)
+              self.addToQueue([auto_start_command])
           pass
 
         # Add a status command
@@ -388,6 +396,7 @@ class Controller(threading.Thread):
 
       if "healthStatus" in commandResult:
         if commandResult["healthStatus"] == "INSTALLED":
+          # Mark it FAILED as its a failure remedied by auto-start or container restart
           self.componentActualState = State.FAILED
           self.failureCount += 1
           self.logStates()

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 6b2ace5..15f1664 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -57,6 +57,7 @@ class CustomServiceOrchestrator():
                                                                   'status_command_stderr.txt'))
     self.public_fqdn = hostname.public_hostname()
     self.stored_command = {}
+    self.allocated_ports = {}
     # Clean up old status command files if any
     try:
       os.unlink(self.status_commands_stdout)
@@ -69,7 +70,7 @@ class CustomServiceOrchestrator():
 
   def runCommand(self, command, tmpoutfile, tmperrfile,
                  override_output_files=True, store_command=False):
-    allocated_port = {}
+    allocated_ports = {}
     try:
       script_type = command['commandParams']['script_type']
       script = command['commandParams']['script']
@@ -86,7 +87,7 @@ class CustomServiceOrchestrator():
       # We don't support anything else yet
         message = "Unknown script type {0}".format(script_type)
         raise AgentException(message)
-      json_path = self.dump_command_to_json(command, allocated_port, store_command)
+      json_path = self.dump_command_to_json(command, allocated_ports, store_command)
       py_file_list = [script_tuple]
       # filter None values
       filtered_py_file_list = [i for i in py_file_list if i]
@@ -132,7 +133,8 @@ class CustomServiceOrchestrator():
       }
 
     if Constants.EXIT_CODE in ret and ret[Constants.EXIT_CODE] == 0:
-      ret[Constants.ALLOCATED_PORTS] = allocated_port
+      ret[Constants.ALLOCATED_PORTS] = allocated_ports
+      self.allocated_ports = allocated_ports
 
     # Irrespective of the outcome report the folder paths
     if command_name == 'INSTALL':

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-agent/src/main/python/agent/Heartbeat.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Heartbeat.py b/slider-agent/src/main/python/agent/Heartbeat.py
index aa403d4..b107d92 100644
--- a/slider-agent/src/main/python/agent/Heartbeat.py
+++ b/slider-agent/src/main/python/agent/Heartbeat.py
@@ -36,7 +36,7 @@ class Heartbeat:
     self.config = config
     self.reports = []
 
-  def build(self, commandResult, componentActualState, id='-1',
+  def build(self, commandResult, id='-1',
             componentsMapped=False):
     timestamp = int(time.time() * 1000)
     queueResult = self.actionQueue.result()
@@ -49,8 +49,7 @@ class Heartbeat:
                  'timestamp': timestamp,
                  'hostname': self.config.getLabel(),
                  'nodeStatus': nodeStatus,
-                 'fqdn': hostname.public_hostname(),
-                 'agentState': componentActualState
+                 'fqdn': hostname.public_hostname()
     }
 
     commandsInProgress = False
@@ -62,6 +61,10 @@ class Heartbeat:
         if report['reportResult']:
           del report['reportResult']
           heartbeat['reports'].append(report)
+        else:
+          # dropping the result but only recording the status
+          commandResult["commandStatus"] = report["status"]
+          pass
       if len(heartbeat['reports']) > 0:
         # There may be IN_PROGRESS tasks
         commandsInProgress = True

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-agent/src/main/python/agent/Register.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Register.py b/slider-agent/src/main/python/agent/Register.py
index 7c7ff06..b59154f 100644
--- a/slider-agent/src/main/python/agent/Register.py
+++ b/slider-agent/src/main/python/agent/Register.py
@@ -29,7 +29,7 @@ class Register:
   def __init__(self, config):
     self.config = config
 
-  def build(self, id='-1'):
+  def build(self, actualState, expectedState, allocated_ports, id='-1'):
     timestamp = int(time.time() * 1000)
 
     version = self.read_agent_version()
@@ -38,7 +38,10 @@ class Register:
                 'timestamp': timestamp,
                 'hostname': self.config.getLabel(),
                 'publicHostname': hostname.public_hostname(),
-                'agentVersion': version
+                'agentVersion': version,
+                'actualState': actualState,
+                'expectedState': expectedState,
+                'allocatedPorts': allocated_ports
     }
     return register
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-agent/src/test/python/agent/TestActionQueue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestActionQueue.py b/slider-agent/src/test/python/agent/TestActionQueue.py
index 4a53c2e..8071ee8 100644
--- a/slider-agent/src/test/python/agent/TestActionQueue.py
+++ b/slider-agent/src/test/python/agent/TestActionQueue.py
@@ -61,8 +61,6 @@ class TestActionQueue(TestCase):
     'serviceName': u'HBASE',
     'configurations': {'global': {}},
     'configurationTags': {'global': {'tag': 'v1'}},
-    'auto_generated': False,
-    'roleParams': {'auto_restart':'false'},
     'commandParams': {'script_type': 'PYTHON',
                       'script': 'scripts/abc.py',
                       'command_timeout': '600'}

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-agent/src/test/python/agent/TestController.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestController.py b/slider-agent/src/test/python/agent/TestController.py
index 91e12f1..401d69a 100644
--- a/slider-agent/src/test/python/agent/TestController.py
+++ b/slider-agent/src/test/python/agent/TestController.py
@@ -55,6 +55,7 @@ class TestController(unittest.TestCase):
     self.controller = Controller.Controller(config)
     self.controller.netutil.MINIMUM_INTERVAL_BETWEEN_HEARTBEATS = 0.1
     self.controller.netutil.HEARTBEAT_NOT_IDDLE_INTERVAL_SEC = 0.1
+    self.controller.actionQueue = ActionQueue.ActionQueue(config, self.controller)
 
 
   @patch("json.dumps")
@@ -289,7 +290,7 @@ class TestController(unittest.TestCase):
     self.controller.sendRequest = sendRequest
 
     self.controller.responseId = 1
-    response = {"responseId":"2", "restartAgent":"false"}
+    response = {"responseId":"2", "restartAgent": False}
     loadsMock.return_value = response
 
     def one_heartbeat(*args, **kwargs):
@@ -651,8 +652,9 @@ class TestController(unittest.TestCase):
     self.controller.sendRequest = sendRequest
 
     self.controller.responseId = 1
-    response = {"responseId":"2", "restartAgent":"false", "restartEnabled":'true'}
-    loadsMock.return_value = response
+    response1 = {"responseId": "2", "restartAgent": False, "restartEnabled": True}
+    response2 = {"responseId": "2", "restartAgent": False, "restartEnabled": False}
+    loadsMock.side_effect = [response1, response2, response1]
 
     def one_heartbeat(*args, **kwargs):
       self.controller.DEBUG_STOP_HEARTBEATING = True
@@ -665,9 +667,9 @@ class TestController(unittest.TestCase):
 
     # one successful request, after stop
     self.controller.actionQueue = actionQueue
-    self.controller.componentActualState = State.INSTALLED
+    self.controller.componentActualState = State.FAILED
     self.controller.componentExpectedState = State.STARTED
-    self.assertTrue(self.controller.componentActualState, State.INSTALLED)
+    self.assertTrue(self.controller.componentActualState, State.FAILED)
     self.controller.actionQueue.customServiceOrchestrator.stored_command = {
       'commandType': 'EXECUTION_COMMAND',
       'role': u'HBASE',
@@ -699,6 +701,23 @@ class TestController(unittest.TestCase):
       'commandId': '8-1',
       'auto_generated': True}])])
     self.controller.config = original_value
+
+    # restartEnabled = False
+    self.controller.componentActualState = State.FAILED
+    self.controller.heartbeatWithServer()
+
+    self.assertTrue(sendRequest.called)
+    self.assertTrue(self.controller.componentActualState, State.FAILED)
+    self.assertTrue(self.controller.componentExpectedState, State.STARTED)
+
+    # restartEnabled = True
+    self.controller.componentActualState = State.INSTALLED
+    self.controller.componentExpectedState = State.INSTALLED
+    self.controller.heartbeatWithServer()
+
+    self.assertTrue(sendRequest.called)
+    self.assertTrue(self.controller.componentActualState, State.INSTALLED)
+    self.assertTrue(self.controller.componentExpectedState, State.INSTALLED)
     pass
 
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index 6408809..e545afe 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -230,6 +230,8 @@ class TestCustomServiceOrchestrator(TestCase):
     self.assertEqual(ret['allocated_ports'], {'a.a.port': '10233'})
     self.assertTrue(run_file_mock.called)
     self.assertEqual(run_file_mock.call_count, 1)
+    self.assertEqual(orchestrator.allocated_ports, {'a.a.port': '10233'})
+    self.assertEqual(orchestrator.stored_command, {})
 
 
   @patch.object(socket, "close")

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-agent/src/test/python/agent/TestHeartbeat.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestHeartbeat.py b/slider-agent/src/test/python/agent/TestHeartbeat.py
index 70870ec..b012218 100644
--- a/slider-agent/src/test/python/agent/TestHeartbeat.py
+++ b/slider-agent/src/test/python/agent/TestHeartbeat.py
@@ -50,7 +50,7 @@ class TestHeartbeat(TestCase):
     dummy_controller = MagicMock()
     actionQueue = ActionQueue(config, dummy_controller)
     heartbeat = Heartbeat(actionQueue, config)
-    result = heartbeat.build({}, State.STARTED, 100)
+    result = heartbeat.build({}, 100)
     print "Heartbeat: " + str(result)
     self.assertEquals(result['hostname'] != '', True,
                       "hostname should not be empty")
@@ -63,7 +63,6 @@ class TestHeartbeat(TestCase):
     self.assertEquals(len(result['nodeStatus']), 2)
     self.assertEquals(result['nodeStatus']['cause'], "NONE")
     self.assertEquals(result['nodeStatus']['status'], "HEALTHY")
-    self.assertEquals(result['agentState'], State.STARTED)
     # result may or may NOT have an agentEnv structure in it
     self.assertEquals((len(result) is 6) or (len(result) is 7), True)
     self.assertEquals(not heartbeat.reports, True,
@@ -147,7 +146,7 @@ class TestHeartbeat(TestCase):
     }
     heartbeat = Heartbeat(actionQueue, config)
     # State.STARTED results in agentState to be set to 4 (enum order)
-    hb = heartbeat.build({}, State.STARTED, 10)
+    hb = heartbeat.build({}, 10)
     hb['hostname'] = 'hostname'
     hb['timestamp'] = 'timestamp'
     hb['fqdn'] = 'fqdn'
@@ -155,7 +154,7 @@ class TestHeartbeat(TestCase):
                   {'status': 'HEALTHY',
                    'cause': 'NONE'},
                 'timestamp': 'timestamp', 'hostname': 'hostname', 'fqdn': 'fqdn',
-                'agentState': 4, 'responseId': 10, 'reports': [
+                'responseId': 10, 'reports': [
       {'status': 'IN_PROGRESS', 'roleCommand': u'INSTALL',
        'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1',
        'stderr': 'Read from /tmp/errors-3.txt',
@@ -178,6 +177,81 @@ class TestHeartbeat(TestCase):
     self.assertEqual.__self__.maxDiff = None
     self.assertEquals(hb, expected)
 
+  @patch.object(ActionQueue, "result")
+  def test_build_result2(self, result_mock):
+    config = AgentConfig("", "")
+    config.set('agent', 'prefix', 'tmp')
+    dummy_controller = MagicMock()
+    actionQueue = ActionQueue(config, dummy_controller)
+    result_mock.return_value = {
+      'reports': [{'status': 'IN_PROGRESS',
+                   'stderr': 'Read from /tmp/errors-3.txt',
+                   'stdout': 'Read from /tmp/output-3.txt',
+                   'clusterName': u'cc',
+                   'roleCommand': u'INSTALL',
+                   'serviceName': u'HDFS',
+                   'role': u'DATANODE',
+                   'actionId': '1-1',
+                   'taskId': 3,
+                   'exitcode': 777,
+                   'reportResult' : False}
+      ],
+      'componentStatus': []
+      }
+    heartbeat = Heartbeat(actionQueue, config)
+
+    commandResult = {}
+    hb = heartbeat.build(commandResult, 10)
+    hb['hostname'] = 'hostname'
+    hb['timestamp'] = 'timestamp'
+    hb['fqdn'] = 'fqdn'
+    expected = {'nodeStatus':
+                  {'status': 'HEALTHY',
+                   'cause': 'NONE'},
+                'timestamp': 'timestamp', 'hostname': 'hostname', 'fqdn': 'fqdn',
+                'responseId': 10, 'reports': []}
+    self.assertEqual.__self__.maxDiff = None
+    self.assertEquals(hb, expected)
+    self.assertEquals(commandResult, {'commandStatus': 'IN_PROGRESS'})
+
+  @patch.object(ActionQueue, "result")
+  def test_build_result3(self, result_mock):
+    config = AgentConfig("", "")
+    config.set('agent', 'prefix', 'tmp')
+    dummy_controller = MagicMock()
+    actionQueue = ActionQueue(config, dummy_controller)
+    result_mock.return_value = {
+      'reports': [{'status': 'COMPLETED',
+                   'stderr': 'Read from /tmp/errors-3.txt',
+                   'stdout': 'Read from /tmp/output-3.txt',
+                   'clusterName': u'cc',
+                   'roleCommand': u'INSTALL',
+                   'serviceName': u'HDFS',
+                   'role': u'DATANODE',
+                   'actionId': '1-1',
+                   'taskId': 3,
+                   'exitcode': 777,
+                   'reportResult' : False}
+      ],
+      'componentStatus': []
+    }
+    heartbeat = Heartbeat(actionQueue, config)
+
+    commandResult = {}
+    hb = heartbeat.build(commandResult, 10)
+    hb['hostname'] = 'hostname'
+    hb['timestamp'] = 'timestamp'
+    hb['fqdn'] = 'fqdn'
+    expected = {'nodeStatus':
+                  {'status': 'HEALTHY',
+                   'cause': 'NONE'},
+                'timestamp': 'timestamp', 'hostname': 'hostname', 'fqdn': 'fqdn',
+                'responseId': 10, 'reports': []}
+    self.assertEqual.__self__.maxDiff = None
+    self.assertEquals(hb, expected)
+    self.assertEquals(commandResult, {'commandStatus': 'COMPLETED'})
+
+
 
 if __name__ == "__main__":
   logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-agent/src/test/python/agent/TestRegistration.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestRegistration.py b/slider-agent/src/test/python/agent/TestRegistration.py
index 5245af9..f91fe29 100644
--- a/slider-agent/src/test/python/agent/TestRegistration.py
+++ b/slider-agent/src/test/python/agent/TestRegistration.py
@@ -26,6 +26,7 @@ import tempfile
 from mock.mock import patch
 from mock.mock import MagicMock
 from Register import Register
+from Controller import State
 from AgentConfig import AgentConfig
 
 class TestRegistration(TestCase):
@@ -47,14 +48,17 @@ class TestRegistration(TestCase):
       text_file.write("1.3.0")
 
     register = Register(config)
-    data = register.build(1)
+    data = register.build(State.INIT, State.INIT, {}, 1)
     #print ("Register: " + pprint.pformat(data))
     self.assertEquals(data['hostname'] != "", True, "hostname should not be empty")
     self.assertEquals(data['publicHostname'] != "", True, "publicHostname should not be empty")
     self.assertEquals(data['responseId'], 1)
     self.assertEquals(data['timestamp'] > 1353678475465L, True, "timestamp should not be empty")
     self.assertEquals(data['agentVersion'], '1.3.0', "agentVersion should not be empty")
-    self.assertEquals(len(data), 5)
+    self.assertEquals(data['actualState'], State.INIT, "actualState should not be empty")
+    self.assertEquals(data['expectedState'], State.INIT, "expectedState should not be empty")
+    self.assertEquals(data['allocatedPorts'], {}, "allocatedPorts should be empty")
+    self.assertEquals(len(data), 8)
 
     self.assertEquals(os.path.join(tmpdir, "app/definition"), config.getResolvedPath("app_pkg_dir"))
     self.assertEquals(os.path.join(tmpdir, "app/install"), config.getResolvedPath("app_install_dir"))

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index b22ef39..8ce1f18 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -398,9 +398,19 @@ public class AgentProviderService extends AbstractProviderService implements
     log.info("Handling registration: " + registration);
     RegistrationResponse response = new RegistrationResponse();
     String label = registration.getHostname();
+    State agentState = registration.getActualState();
     if (getComponentStatuses().containsKey(label)) {
       response.setResponseStatus(RegistrationStatus.OK);
-      getComponentStatuses().get(label).heartbeat(System.currentTimeMillis());
+      ComponentInstanceState componentStatus = getComponentStatuses().get(label);
+      componentStatus.heartbeat(System.currentTimeMillis());
+      updateComponentStatusWithAgentState(componentStatus, agentState);
+
+      Map<String, String> ports = registration.getAllocatedPorts();
+      if (ports != null && !ports.isEmpty()) {
+        String roleName = getRoleName(label);
+        String containerId = getContainerId(label);
+        processAllocatedPorts(registration.getPublicHostname(), roleName, containerId, ports);
+      }
     } else {
       response.setResponseStatus(RegistrationStatus.FAILED);
       response.setLog("Label not recognized.");
@@ -424,9 +434,8 @@ public class AgentProviderService extends AbstractProviderService implements
 
     String label = heartBeat.getHostname();
     String roleName = getRoleName(label);
-    State agentState = heartBeat.getAgentState();
-
     String containerId = getContainerId(label);
+
     StateAccessForProviders accessor = getAmState();
     String scriptPath = getScriptPathFromMetainfo(roleName);
 
@@ -441,13 +450,22 @@ public class AgentProviderService extends AbstractProviderService implements
 
     Boolean isMaster = isMaster(roleName);
     ComponentInstanceState componentStatus = getComponentStatuses().get(label);
-    updateComponentStatusWithAgentState(componentStatus, agentState);
     componentStatus.heartbeat(System.currentTimeMillis());
     // If no Master can explicitly publish then publish if its a master
     // Otherwise, wait till the master that can publish is ready
     if (isMaster &&
         (canAnyMasterPublishConfig() == false || canPublishConfig(roleName))) {
       publishConfigAndExportGroups(heartBeat, componentStatus);
+    } else {
+      // Ack that config has been reported
+      List<ComponentStatus> statuses = heartBeat.getComponentStatus();
+      if (statuses != null && !statuses.isEmpty()) {
+        ComponentStatus status = statuses.get(0);
+        if(status.getConfigs().size() > 0) {
+          log.info("Config got reported by {} but discarded as component {} cannot publish.", label, roleName);
+          componentStatus.setConfigReported(true);
+        }
+      }
     }
 
     List<CommandReport> reports = heartBeat.getReports();
@@ -455,14 +473,7 @@ public class AgentProviderService extends AbstractProviderService implements
       CommandReport report = reports.get(0);
       Map<String, String> ports = report.getAllocatedPorts();
       if (ports != null && !ports.isEmpty()) {
-        for (Map.Entry<String, String> port : ports.entrySet()) {
-          log.info("Recording allocated port for {} as {}", port.getKey(), port.getValue());
-          this.getAllocatedPorts().put(port.getKey(), port.getValue());
-          this.getAllocatedPorts(containerId).put(port.getKey(), port.getValue());
-        }
-
-        // component specific publishes
-        processAndPublishComponentSpecificData(ports, containerId, heartBeat.getFqdn(), roleName);
+        processAllocatedPorts(heartBeat.getFqdn(), roleName, containerId, ports);
       }
       CommandResult result = CommandResult.getCommandResult(report.getStatus());
       Command command = Command.getCommand(report.getRoleCommand());
@@ -506,6 +517,7 @@ public class AgentProviderService extends AbstractProviderService implements
       if (isMaster && componentStatus.getState() == State.STARTED
           && command == Command.NOP) {
         if (!componentStatus.getConfigReported()) {
+          log.info("Requesting applied config for {} on {}.", roleName, containerId);
           addGetConfigCommand(roleName, containerId, response);
         }
       }
@@ -525,6 +537,20 @@ public class AgentProviderService extends AbstractProviderService implements
     return response;
   }
 
+  protected void processAllocatedPorts(String fqdn,
+                                     String roleName,
+                                     String containerId,
+                                     Map<String, String> ports) {
+    for (Map.Entry<String, String> port : ports.entrySet()) {
+      log.info("Recording allocated port for {} as {}", port.getKey(), port.getValue());
+      this.getAllocatedPorts().put(port.getKey(), port.getValue());
+      this.getAllocatedPorts(containerId).put(port.getKey(), port.getValue());
+    }
+
+    // component specific publishes
+    processAndPublishComponentSpecificData(ports, containerId, fqdn, roleName);
+  }
+
   private void updateComponentStatusWithAgentState(
       ComponentInstanceState componentStatus, State agentState) {
     if (agentState != null) {
@@ -803,6 +829,7 @@ public class AgentProviderService extends AbstractProviderService implements
               }
             }
           }
+          log.info("Received and stored config for {}", heartBeat.getHostname());
           componentStatus.setConfigReported(true);
         }
       }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java
index 62df18d..a08d46d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java
@@ -45,7 +45,6 @@ public class HeartBeat {
   HostStatus nodeStatus;
   private AgentEnv agentEnv = null;
   private String fqdn;
-  private State agentState;
 
   public long getResponseId() {
     return responseId;
@@ -125,14 +124,6 @@ public class HeartBeat {
     this.mounts = mounts;
   }
 
-  public State getAgentState() {
-    return agentState;
-  }
-
-  public void setAgentState(State agentState) {
-    this.agentState = agentState;
-  }
-
   @Override
   public String toString() {
     return "HeartBeat{" +
@@ -142,7 +133,6 @@ public class HeartBeat {
            ", reports=" + reports +
            ", componentStatus=" + componentStatus +
            ", nodeStatus=" + nodeStatus +
-           ", agentState=" + agentState +
            '}';
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
index 9299a16..a44c3a4 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
@@ -16,15 +16,14 @@
  */
 package org.apache.slider.server.appmaster.web.rest.agent;
 
+import org.apache.slider.providers.agent.State;
 import org.codehaus.jackson.annotate.JsonIgnoreProperties;
 import org.codehaus.jackson.annotate.JsonProperty;
 import org.codehaus.jackson.map.annotate.JsonSerialize;
 
-/**
- *
- * Data model for agent to send heartbeat to ambari and/or app master.
- *
- */
+import java.util.Map;
+
+/** Data model for agent to send heartbeat to ambari and/or app master. */
 @JsonIgnoreProperties(ignoreUnknown = true)
 @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
 public class Register {
@@ -36,6 +35,9 @@ public class Register {
   private String publicHostname;
   private AgentEnv agentEnv;
   private String agentVersion;
+  private State actualState;
+  private State expectedState;
+  private Map<String, String> allocatedPorts;
 
   @JsonProperty("responseId")
   public int getResponseId() {
@@ -44,13 +46,17 @@ public class Register {
 
   @JsonProperty("responseId")
   public void setResponseId(int responseId) {
-    this.responseId=responseId;
+    this.responseId = responseId;
   }
 
   public long getTimestamp() {
     return timestamp;
   }
 
+  public void setTimestamp(long timestamp) {
+    this.timestamp = timestamp;
+  }
+
   public String getHostname() {
     return hostname;
   }
@@ -67,10 +73,6 @@ public class Register {
     this.hardwareProfile = hardwareProfile;
   }
 
-  public void setTimestamp(long timestamp) {
-    this.timestamp = timestamp;
-  }
-
   public String getPublicHostname() {
     return publicHostname;
   }
@@ -103,15 +105,45 @@ public class Register {
     this.currentPingPort = currentPingPort;
   }
 
+  public State getActualState() {
+    return actualState;
+  }
+
+  public void setActualState(State actualState) {
+    this.actualState = actualState;
+  }
+
+  public State getExpectedState() {
+    return expectedState;
+  }
+
+  public void setExpectedState(State expectedState) {
+    this.expectedState = expectedState;
+  }
+
+  /** @return the allocated ports, or <code>null</code> if none are present */
+  @JsonProperty("allocatedPorts")
+  public Map<String, String> getAllocatedPorts() {
+    return allocatedPorts;
+  }
+
+  /** @param ports allocated ports */
+  @JsonProperty("allocatedPorts")
+  public void setAllocatedPorts(Map<String, String> ports) {
+    this.allocatedPorts = ports;
+  }
+
   @Override
   public String toString() {
     String ret = "responseId=" + responseId + "\n" +
                  "timestamp=" + timestamp + "\n" +
-                 "hostname="  + hostname + "\n" +
-                 "currentPingPort=" + currentPingPort + "\n";
+                 "hostname=" + hostname + "\n" +
+                 "expectedState=" + expectedState + "\n" +
+                 "actualState=" + actualState + "\n";
 
-    if (hardwareProfile != null)
+    if (hardwareProfile != null) {
       ret = ret + "hardwareprofile=" + this.hardwareProfile.toString();
+    }
     return ret;
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 1aeedbc..20c16ff 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -87,6 +87,7 @@ import static org.easymock.EasyMock.replay;
 import static org.junit.Assert.assertEquals;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyCollection;
+import static org.mockito.Matchers.anyMap;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doReturn;
@@ -271,6 +272,12 @@ public class TestAgentProviderService {
     } catch (SliderException e) {
     }
 
+    doNothing().when(mockAps).processAllocatedPorts(
+        anyString(),
+        anyString(),
+        anyString(),
+        anyMap()
+    );
     expect(access.isApplicationLive()).andReturn(true).anyTimes();
     ClusterDescription desc = new ClusterDescription();
     desc.setOption(OptionKeys.ZOOKEEPER_QUORUM, "host1:2181");
@@ -305,10 +312,20 @@ public class TestAgentProviderService {
     Register reg = new Register();
     reg.setResponseId(0);
     reg.setHostname("mockcontainer_1___HBASE_MASTER");
+    Map<String,String> ports = new HashMap();
+    ports.put("a","100");
+    reg.setAllocatedPorts(ports);
     RegistrationResponse resp = mockAps.handleRegistration(reg);
     Assert.assertEquals(0, resp.getResponseId());
     Assert.assertEquals(RegistrationStatus.OK, resp.getResponseStatus());
 
+    Mockito.verify(mockAps, Mockito.times(1)).processAllocatedPorts(
+        anyString(),
+        anyString(),
+        anyString(),
+        anyMap()
+    );
+
     HeartBeat hb = new HeartBeat();
     hb.setResponseId(1);
     hb.setHostname("mockcontainer_1___HBASE_MASTER");


[14/50] [abbrv] SLIDER-322 move all <> constructors to typed

Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index 6653dba..df4ab8e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -264,7 +264,7 @@ public final class RoleStatus implements Cloneable {
    * @return a map for use in statistics reports
    */
   public Map<String, Integer> buildStatistics() {
-    Map<String, Integer> stats = new HashMap<>();
+    Map<String, Integer> stats = new HashMap<String, Integer>();
     stats.put(StatusKeys.STATISTICS_CONTAINERS_ACTIVE_REQUESTS, getRequested());
     stats.put(StatusKeys.STATISTICS_CONTAINERS_COMPLETED, getCompleted());
     stats.put(StatusKeys.STATISTICS_CONTAINERS_DESIRED, getDesired());

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
index 4f290af..9192efe 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
@@ -96,7 +96,7 @@ public class SliderAMWebApp extends WebApp {
     String regex = "(?!/ws)";
     serveRegex(regex).with(SliderDefaultWrapperServlet.class);
 
-    Map<String, String> params = new HashMap<>();
+    Map<String, String> params = new HashMap<String, String>();
     params.put(ResourceConfig.FEATURE_IMPLICIT_VIEWABLES, "true");
     params.put(ServletContainer.FEATURE_FILTER_FORWARD_ON_404, "true");
     params.put(ResourceConfig.FEATURE_XMLROOTELEMENT_PROCESSING, "true");

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
index 606c05d..5fffa4a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
@@ -43,7 +43,7 @@ public class SliderAmFilterInitializer extends FilterInitializer {
   @Override
   public void initFilter(FilterContainer container, Configuration conf) {
     configuration = conf;
-    Map<String, String> params = new HashMap<>();
+    Map<String, String> params = new HashMap<String, String>();
     String proxy = WebAppUtils.getProxyHostAndPort(conf);
     String[] parts = proxy.split(":");
     params.put(SliderAmIpFilter.PROXY_HOST, parts[0]);

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
index ad5e219..aad443e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
@@ -70,7 +70,7 @@ public class SliderAmIpFilter implements Filter {
     synchronized(this) {
       if(proxyAddresses == null || (lastUpdate + updateInterval) >= now) {
         try {
-          proxyAddresses = new HashSet<>();
+          proxyAddresses = new HashSet<String>();
           for(InetAddress add : InetAddress.getAllByName(proxyHost)) {
             if (log.isDebugEnabled()) {
               log.debug("proxy address is: " + add.getHostAddress());

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
index 4eebd45..4d595a9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
@@ -126,7 +126,7 @@ public class WebAppApiImpl implements WebAppApi {
    */
   private TreeMap<String, RoleStatus> getRoleStatusesByName(Map<Integer, ProviderRole> rolesById,
       Map<Integer, RoleStatus> statusById) {
-    TreeMap<String, RoleStatus> statusByName = new TreeMap<>();
+    TreeMap<String, RoleStatus> statusByName = new TreeMap<String, RoleStatus>();
     for (Entry<Integer, ProviderRole> role : rolesById.entrySet()) {
       final RoleStatus status = statusById.get(role.getKey());
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
index a439d9b..5d8b657 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
@@ -110,7 +110,7 @@ public class PublisherResource {
   @Produces({MediaType.APPLICATION_JSON})
   public Set<URL> getAMClassPath() {
     URL[] urls = ((URLClassLoader) getClass().getClassLoader()).getURLs();
-    return new LinkedHashSet<>(Arrays.asList(urls));
+    return new LinkedHashSet<URL>(Arrays.asList(urls));
   }
 
   @GET
@@ -231,7 +231,7 @@ public class PublisherResource {
           propertyName, config);
       throw new NotFoundException("Property not found: " + propertyName);
     }
-    Map<String,String> rtnVal = new HashMap<>();
+    Map<String, String> rtnVal = new HashMap<String, String>();
     rtnVal.put(propertyName, propVal);
 
     return rtnVal;

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java b/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
index 2120be5..422ffeb 100644
--- a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
+++ b/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
@@ -90,7 +90,7 @@ public class RoleHistoryWriter {
     throws IOException {
     try {
       DatumWriter<RoleHistoryRecord> writer =
-        new SpecificDatumWriter<>(RoleHistoryRecord.class);
+        new SpecificDatumWriter<RoleHistoryRecord>(RoleHistoryRecord.class);
 
       int roles = history.getRoleSize();
       RoleHistoryHeader header = new RoleHistoryHeader();
@@ -184,7 +184,7 @@ public class RoleHistoryWriter {
                                                        BadConfigException {
     try {
       DatumReader<RoleHistoryRecord> reader =
-        new SpecificDatumReader<>(RoleHistoryRecord.class);
+        new SpecificDatumReader<RoleHistoryRecord>(RoleHistoryRecord.class);
       Decoder decoder =
         DecoderFactory.get().jsonDecoder(RoleHistoryRecord.getClassSchema(),
                                          in);
@@ -336,7 +336,7 @@ public class RoleHistoryWriter {
     
     PathFilter filter = new GlobFilter(SliderKeys.HISTORY_FILENAME_GLOB_PATTERN);
     FileStatus[] stats = fs.listStatus(dir, filter);
-    List<Path> paths = new ArrayList<>(stats.length);
+    List<Path> paths = new ArrayList<Path>(stats.length);
     for (FileStatus stat : stats) {
       log.debug("Possible entry: {}", stat.toString());
       if (stat.isFile() && (includeEmptyFiles || stat.getLen() > 0)) {

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorUtils.java b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorUtils.java
index 3b2c6d7..a4447e3 100644
--- a/slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorUtils.java
@@ -50,7 +50,7 @@ public final class MonitorUtils {
    * @return a list view with no empty strings
    */
   public static List<String> prepareArgs(String[] args) {
-    List<String> argsList = new ArrayList<>(args.length);
+    List<String> argsList = new ArrayList<String>(args.length);
     StringBuilder argsStr = new StringBuilder("Arguments: [");
     for (String arg : args) {
       argsStr.append('"').append(arg).append("\" ");

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
index 14b78a7..b3e2ff2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
@@ -54,10 +54,10 @@ public class RegistryBinderService<Payload> extends CuratorService {
   private final ServiceDiscovery<Payload> discovery;
 
   private final Map<String, ServiceInstance<Payload>> entries =
-    new HashMap<>();
+    new HashMap<String, ServiceInstance<Payload>>();
 
   private JsonSerDeser<CuratorServiceInstance<Payload>> deser =
-    new JsonSerDeser<>(CuratorServiceInstance.class);
+    new JsonSerDeser<CuratorServiceInstance<Payload>>(CuratorServiceInstance.class);
 
   /**
    * Create an instance
@@ -221,7 +221,7 @@ public class RegistryBinderService<Payload> extends CuratorService {
     try {
       List<String> instanceIDs = instanceIDs(servicetype);
       List<CuratorServiceInstance<Payload>> instances =
-        new ArrayList<>(instanceIDs.size());
+        new ArrayList<CuratorServiceInstance<Payload>>(instanceIDs.size());
       for (String instanceID : instanceIDs) {
         CuratorServiceInstance<Payload> instance =
           queryForInstance(servicetype, instanceID);

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryRestResources.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryRestResources.java b/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryRestResources.java
index 4938adf..e4e8523 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryRestResources.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryRestResources.java
@@ -100,7 +100,8 @@ public class RegistryRestResources extends DiscoveryResource<ServiceInstanceData
     try {
       List<CuratorServiceInstance<ServiceInstanceData>>
           instances = registry.listInstances(name);
-      return Response.ok(new CuratorServiceInstances<>(instances)).build();
+      return Response.ok(
+      new CuratorServiceInstances<ServiceInstanceData>(instances)).build();
     } catch (Exception e) {
       log.error("Error during generation of response", e);
       return Response.serverError().build();

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/services/registry/SliderRegistryService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/registry/SliderRegistryService.java b/slider-core/src/main/java/org/apache/slider/server/services/registry/SliderRegistryService.java
index 1904df5..ca4d180 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/registry/SliderRegistryService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/registry/SliderRegistryService.java
@@ -53,7 +53,7 @@ public class SliderRegistryService
       IOException {
     List<CuratorServiceInstance<ServiceInstanceData>> services =
         listInstances(serviceType);
-    List<ServiceInstanceData> payloads = new ArrayList<>(services.size());
+    List<ServiceInstanceData> payloads = new ArrayList<ServiceInstanceData>(services.size());
     for (CuratorServiceInstance<ServiceInstanceData> instance : services) {
       payloads.add(instance.payload);
     }


[12/50] [abbrv] git commit: SLIDER-318 rm slider-assembly module that isn't used

Posted by st...@apache.org.
SLIDER-318 rm slider-assembly module that isn't used


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/52f1f6fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/52f1f6fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/52f1f6fb

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 52f1f6fbe041a8bc604802e9eaaed2376b0e8ed5
Parents: 564462e
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 14 13:56:07 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 14 13:56:07 2014 +0100

----------------------------------------------------------------------
 pom.xml                                         |   1 -
 slider-assembly/src/main/bash/README.md         | 102 +++++++++++
 slider-assembly/src/main/bash/slider-client.xml |  83 +++++++++
 slider-assembly/src/main/bash/slider_destroy    |  64 +++++++
 slider-assembly/src/main/bash/slider_setup      | 173 +++++++++++++++++++
 slider-assembly/src/main/bash/slider_setup.conf |  60 +++++++
 slider-install/README.md                        | 102 -----------
 slider-install/pom.xml                          | 111 ------------
 slider-install/src/main/bash/slider-client.xml  |  83 ---------
 slider-install/src/main/bash/slider_destroy     |  64 -------
 slider-install/src/main/bash/slider_setup       | 173 -------------------
 slider-install/src/main/bash/slider_setup.conf  |  60 -------
 12 files changed, 482 insertions(+), 594 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ed823c4..0816479 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,7 +45,6 @@
     <module>slider-providers/hbase/hbase-funtests</module>
     <module>slider-providers/accumulo/slider-accumulo-provider</module>
     <module>slider-providers/accumulo/accumulo-funtests</module>
-    <module>slider-install</module>
   </modules>
 
   <licenses>

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-assembly/src/main/bash/README.md
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/bash/README.md b/slider-assembly/src/main/bash/README.md
new file mode 100644
index 0000000..a4b7b08
--- /dev/null
+++ b/slider-assembly/src/main/bash/README.md
@@ -0,0 +1,102 @@
+<!---
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+
+slider_setup
+============
+
+Tools for installing, starting, and destroying HBase, Accumulo, and Storm slider apps on YARN.
+
+**WARNING: This is intended for POC/sandbox testing, may not be idempotent so DO NOT use on an existing Production cluster!!!**
+
+Setup
+-----
+1. Clone the repo
+2. Set the necessary cluster variables in `slider_setup.conf`, it shouldn't be necessary to change any other values but thoe ones below
+
+
+    # Zookeeper nodes
+    ZK_QUORUM="zk1:2181,zk2:2181,zk3:2181"
+    
+    # Resource Manager address (yarn.resourcemanager.address)
+    RM_ADDRESS="rm1:8050"
+    
+    # Resource Manager scheduler address (yarn.resourcemanager.scheduler.address)
+    RM_SCHED_ADDRESS="rm1:8030"
+    
+    # Default FS (fs.defaultFS)
+    DEFAULT_FS="hdfs://nn1:8020"
+
+Running
+-------
+* slider_setup is the main script and handles the following
+  1. Pulls down slider and extracts the contents to the SLIDER_INST_DIR
+  2. Modifies slider-client.xml with cluster related info
+  3. Pulls down the slider enabled version of the specified product
+  4. Creates necessary directories and copies required files to HDFS
+  5. For HBase, creates the app dir in HDFS
+  6. Submits the slider base application to the YARN cluster
+
+* The following args are required
+  * -f - The path to the slider_setup.conf that has been modified with cluster info
+  * -p - The product to run (hbase, accumulo, or storm are all that are supported at this time)
+  * -w - The number of "worker" nodes. This has different meaning depending on product.
+    * HBase - number of region servers
+    * Accumulo - number of tablet servers
+    * Storm - number of supervisors
+  * -n - The name of the app, this will be the display name in the resource manager and is used by the teardown process
+
+* HBase Example:
+
+
+    ./slider_setup -f slider_setup.conf -p hbase -w 5 -n hbase-slider
+
+* Accumulo Example:
+
+
+    ./slider_setup -f slider_setup.conf -p accumulo -w 3 -n accumulo-slider
+
+* Storm Example:
+
+
+    ./slider_setup -f slider_setup.conf -p storm -w 3 -n storm-slider
+
+Tear Down
+---------
+
+* slider_destroy will do the following
+  1. Freeze the slider application based on provided name
+  2. Destory the slider application based on provided name
+
+* The following args are required
+  * `-f` - The path to the `slider_setup.conf` that has been modified with cluster info
+  * `-n` - The name of the app, this was provided to the slider_setup tool
+
+* HBase Example:
+
+
+    ./slider_destroy -f slider_setup.conf -n hbase-slider
+
+* Accumulo Example:
+
+
+    ./slider_destroy -f slider_setup.conf -n accumulo-slider
+
+* Storm Example:
+
+
+    ./slider_destroy -f slider_setup.conf -n storm-slider

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-assembly/src/main/bash/slider-client.xml
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/bash/slider-client.xml b/slider-assembly/src/main/bash/slider-client.xml
new file mode 100644
index 0000000..f7060c1
--- /dev/null
+++ b/slider-assembly/src/main/bash/slider-client.xml
@@ -0,0 +1,83 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<!--
+  Properties set here are picked up in the client.
+  They are not passed to the AM -though the filesystem
+  binding details (URL And principal) are added to the
+  hbase-site.xml file when a cluster is created.
+-->
+<configuration>
+
+  <property>
+    <name>yarn.log-aggregation-enable</name>
+    <value>true</value>
+  </property>
+  
+
+  <property>
+    <name>slider.yarn.queue</name>
+    <value>default</value>
+    <description>YARN queue for the Application Master</description>
+  </property>
+  
+  <property>
+    <name>yarn.resourcemanager.address</name>
+    <value>@@RM_ADDRESS@@</value>
+  </property>
+
+  <property>
+    <name>yarn.resourcemanager.scheduler.address</name>
+    <value>@@RM_SCHED_ADDRESS@@</value>
+  </property>
+
+  <property>
+    <name>fs.defaultFS</name>
+    <value>@@DEFAULT_FS@@</value>
+  </property>
+
+  <property>
+     <name>yarn.application.classpath</name>
+     <value>@@YARN_CP@@</value>
+  </property>
+
+  <property>
+     <name>slider.zookeeper.quorum</name>
+     <value>@@ZK_QUORUM@@</value>
+  </property>
+
+<!--
+  <property>
+    <name>yarn.resourcemanager.principal</name>
+    <value>yarn/master@MINICLUSTER</value>
+  </property>
+
+  <property>
+    <name>slider.security.enabled</name>
+    <value>true</value>
+  </property>
+
+  <property>
+    <name>dfs.namenode.kerberos.principal</name>
+    <value>hdfs/master@MINICLUSTER</value>
+  </property>
+-->
+
+
+</configuration>

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-assembly/src/main/bash/slider_destroy
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/bash/slider_destroy b/slider-assembly/src/main/bash/slider_destroy
new file mode 100755
index 0000000..9039751
--- /dev/null
+++ b/slider-assembly/src/main/bash/slider_destroy
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Base Vars
+#
+SCRIPT_NAME=`basename $0`
+SCRIPT_DIR=`cd $(dirname $0) && pwd`
+
+#
+# Functions
+#
+usage() { 
+    echo "Usage: $SCRIPT_NAME -f </path/to/config> -n <app name>"
+    exit 1
+}
+
+#
+# Parse cmd line args
+#
+while getopts "f:n:" opt; do
+    case "$opt" in
+        f) config=$OPTARG;;
+        n) app_name=$OPTARG;;
+        *) usage;;
+    esac
+done
+shift $((OPTIND-1))
+
+if [ -z "$config" ] || [ -z "$app_name" ]; then
+    usage
+fi
+
+#
+# Source the config
+#
+source $config
+
+#
+# Main
+#
+echo -e "\n## Freezing app $app_name"
+sudo -u yarn $SLIDER_INST_DIR/bin/slider freeze $app_name --manager $RM_ADDRESS || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Destroying app $app_name"
+sudo -u yarn $SLIDER_INST_DIR/bin/slider destroy $app_name --manager $RM_ADDRESS || exit 1
+echo "SUCCESS"
+
+exit 0

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-assembly/src/main/bash/slider_setup
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/bash/slider_setup b/slider-assembly/src/main/bash/slider_setup
new file mode 100755
index 0000000..2c15c95
--- /dev/null
+++ b/slider-assembly/src/main/bash/slider_setup
@@ -0,0 +1,173 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+#
+# Base Vars
+#
+SCRIPT_NAME=`basename $0`
+SCRIPT_DIR=`cd $(dirname $0) && pwd`
+
+#
+# Functions
+#
+usage() { 
+    echo "Usage: $SCRIPT_NAME -f </path/to/config> -p <product (hbase or storm)> -w <worker count> -n <app name>"
+    exit 1
+}
+
+#
+# Parse cmd line args
+#
+while getopts "f:p:w:n:" opt; do
+    case "$opt" in
+        f) config=$OPTARG;;
+        p) product=$OPTARG;;
+        w) worker_cnt=$OPTARG;;
+        n) app_name=$OPTARG;;
+        *) usage;;
+    esac
+done
+shift $((OPTIND-1))
+
+if [ -z "$config" ] || [ -z "$product" ] || [ -z "$worker_cnt" ] || [ -z "$app_name" ]; then
+    usage
+fi
+
+# Only support know products for now
+if [ $product != "hbase" ] && [ $product != "storm" ] && [ $product != "accumulo"]; then
+   echo "ERROR: Only HBase, Storm, and Accumulo are currently supported"
+   usage
+fi
+
+#
+# Source the config
+#
+source $config
+
+#
+# Product URLs
+#
+if [ $product = "hbase" ]; then
+    app_url=$HBASE_APP_URL
+elif [ $product = "storm" ]; then
+    app_url=$STORM_APP_URL
+elif [ $product = "accumulo" ]; then
+    app_url=$ACCUMULO_APP_URL
+fi
+
+#
+# Main
+#
+echo -e "\n## Creating slider install dir: $SLIDER_INST_DIR"
+mkdir -p $SLIDER_INST_DIR || exit 1
+chown yarn:hadoop $SLIDER_INST_DIR || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Downloading slider from: $SLIDER_URL"
+if [ -f /tmp/${SLIDER_VER}*tar ]; then
+   rm -f /tmp/${SLIDER_VER}*tar
+fi
+cd /tmp && wget $SLIDER_URL || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Extracting slider to $SLIDER_INST_DIR"
+tar -xf /tmp/${SLIDER_VER}*tar --strip-components=1 -C $SLIDER_INST_DIR || exit 1
+chown -R yarn:hadoop $SLIDER_INST_DIR || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Setting conf values"
+(cd $SCRIPT_DIR && cp slider-client.xml $SLIDER_INST_DIR/conf) || exit 1
+sed -i 's|@@RM_ADDRESS@@|'$RM_ADDRESS'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
+sed -i 's|@@RM_SCHED_ADDRESS@@|'$RM_SCHED_ADDRESS'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
+sed -i 's|@@DEFAULT_FS@@|'$DEFAULT_FS'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
+sed -i 's|@@YARN_CP@@|'$YARN_CP'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
+sed -i 's|@@ZK_QUORUM@@|'$ZK_QUORUM'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Setting PATH to include the JDK bin: $JDK_BIN"
+export PATH=$PATH:$JDK_BIN
+echo "SUCCESS"
+
+echo -e "\n## Checking version of Hadoop slider was compiled against"
+hadoop_compiled_ver=`$SLIDER_INST_DIR/bin/slider version | grep "Compiled against Hadoop" | awk '{print $NF}'`
+if [ "$hadoop_compiled_ver" != "2.4.0" ]; then
+   echo "ERROR: Compiled against Hadoop version $hadoop_compiled_ver instead of 2.4.0" && exit 1
+else
+   echo "Compiled against Hadoop version: $hadoop_compiled_ver"
+fi
+echo "SUCCESS"
+
+echo -e "\n## Setting up HDFS directories for slider"
+sudo -u hdfs hdfs dfs -mkdir -p /slider || exit 1
+sudo -u hdfs hdfs dfs -chown yarn:hdfs /slider || exit 1
+sudo -u hdfs hdfs dfs -mkdir -p /user/yarn || exit 1
+sudo -u hdfs hdfs dfs -chown yarn:hdfs /user/yarn || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Loading the Slider agent"
+sudo -u yarn hdfs dfs -mkdir -p /slider/agent/conf || exit 1
+sudo -u yarn hdfs dfs -copyFromLocal $SLIDER_INST_DIR/agent/* /slider/agent
+echo "SUCCESS"
+
+echo -e "\n## Downloading $product to $SLIDER_INST_DIR/apps/$product"
+sudo -u yarn mkdir -p $SLIDER_INST_DIR/apps/$product || exit 1
+(cd $SLIDER_INST_DIR/apps/$product && sudo -u yarn wget $app_url) || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Extracting $product in $SLIDER_INST_DIR/apps/$product"
+(cd $SLIDER_INST_DIR/apps/$product && sudo -u yarn unzip -o $product*zip) || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Adding $product to HDFS slider dir"
+sudo -u yarn hdfs dfs -copyFromLocal $SLIDER_INST_DIR/apps/$product/$product*zip /slider
+echo "SUCCESS"
+
+echo -e "\n## Setting number of workers in $SLIDER_INST_DIR/apps/$product/resources.json"
+if [ $product = "hbase" ]; then
+   component="HBASE_REGIONSERVER"
+elif [ $product = "storm" ]; then
+   component="SUPERVISOR"
+elif [ $product = "accumulo" ]; then
+   component="ACCUMULO_TSERVER"
+fi
+# Update the resource.json file with worker_cnt
+python << END
+import json
+with open("$SLIDER_INST_DIR/apps/$product/resources.json", "r+") as f:
+    data = json.load(f)
+    data["components"]["$component"]["yarn.component.instances"] = "$worker_cnt"
+
+with open("$SLIDER_INST_DIR/apps/$product/resources.json", "w+") as f:
+    f.write(json.dumps(data, sort_keys = False, indent = 4))
+END
+echo "SUCCESS"
+
+# Handle HBase HDFS dir needs
+if [ $product = "hbase" ]; then
+    echo -e "\n## Creating hbase HDFS dir /apps/hbase"
+    sudo -u hdfs hdfs dfs -mkdir -p /apps/hbase || exit 1
+    sudo -u hdfs hdfs dfs -chown yarn:hdfs /apps/hbase || exit 1
+    echo "SUCCESS"
+fi
+
+echo -e "\n##Starting app $product with $worker_cnt workers via slider"
+sudo -u yarn $SLIDER_INST_DIR/bin/slider create $app_name \
+    --image $DEFAULT_FS/slider/agent/slider-agent.tar.gz \
+    --template $SLIDER_INST_DIR/apps/$product/appConfig.json \
+    --resources $SLIDER_INST_DIR/apps/$product/resources.json || exit 1
+echo "SUCCESS"

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-assembly/src/main/bash/slider_setup.conf
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/bash/slider_setup.conf b/slider-assembly/src/main/bash/slider_setup.conf
new file mode 100644
index 0000000..fed3f4c
--- /dev/null
+++ b/slider-assembly/src/main/bash/slider_setup.conf
@@ -0,0 +1,60 @@
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Change values below to match the cluster being deployed on
+#
+
+# Zookeeper nodes
+ZK_QUORUM="zk1:2181,zk2:2181,zk3:2181"
+
+# Resource Manager address (yarn.resourcemanager.address)
+RM_ADDRESS="rm1:8050"
+
+# Resource Manager scheduler address (yarn.resourcemanager.scheduler.address)
+RM_SCHED_ADDRESS="rm1:8030"
+
+# Default FS (fs.defaultFS)
+DEFAULT_FS="hdfs://nn1:8020"
+
+
+#
+# Should not be necessary to change anything below except on new version releases
+#
+
+# Slider version
+SLIDER_VER="slider-0.30"
+
+# Slider URL
+SLIDER_URL="http://public-repo-1.hortonworks.com/slider/0.30/slider-0.30-all.tar"
+
+# HBase Slider App URL
+HBASE_APP_URL="http://public-repo-1.hortonworks.com/slider/0.30/apps/hbase_v096.zip"
+
+# Storm Slider App URL
+STORM_APP_URL="http://public-repo-1.hortonworks.com/slider/0.30/apps/storm_v091.zip"
+
+# Accumulo Slider App URL
+ACCUMULO_APP_URL="http://public-repo-1.hortonworks.com/slider/0.30/apps/accumulo_v151.zip"
+
+# Where to install Slider
+SLIDER_INST_DIR="/usr/lib/slider"
+
+# JDK bin to add to PATH
+JDK_BIN="/usr/jdk64/jdk1.7.0_45/bin"
+
+# Yarn container classpath (yarn.application.classpath)
+YARN_CP="/etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*"
+

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-install/README.md
----------------------------------------------------------------------
diff --git a/slider-install/README.md b/slider-install/README.md
deleted file mode 100644
index a4b7b08..0000000
--- a/slider-install/README.md
+++ /dev/null
@@ -1,102 +0,0 @@
-<!---
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-
-slider_setup
-============
-
-Tools for installing, starting, and destroying HBase, Accumulo, and Storm slider apps on YARN.
-
-**WARNING: This is intended for POC/sandbox testing, may not be idempotent so DO NOT use on an existing Production cluster!!!**
-
-Setup
------
-1. Clone the repo
-2. Set the necessary cluster variables in `slider_setup.conf`, it shouldn't be necessary to change any other values but thoe ones below
-
-
-    # Zookeeper nodes
-    ZK_QUORUM="zk1:2181,zk2:2181,zk3:2181"
-    
-    # Resource Manager address (yarn.resourcemanager.address)
-    RM_ADDRESS="rm1:8050"
-    
-    # Resource Manager scheduler address (yarn.resourcemanager.scheduler.address)
-    RM_SCHED_ADDRESS="rm1:8030"
-    
-    # Default FS (fs.defaultFS)
-    DEFAULT_FS="hdfs://nn1:8020"
-
-Running
--------
-* slider_setup is the main script and handles the following
-  1. Pulls down slider and extracts the contents to the SLIDER_INST_DIR
-  2. Modifies slider-client.xml with cluster related info
-  3. Pulls down the slider enabled version of the specified product
-  4. Creates necessary directories and copies required files to HDFS
-  5. For HBase, creates the app dir in HDFS
-  6. Submits the slider base application to the YARN cluster
-
-* The following args are required
-  * -f - The path to the slider_setup.conf that has been modified with cluster info
-  * -p - The product to run (hbase, accumulo, or storm are all that are supported at this time)
-  * -w - The number of "worker" nodes. This has different meaning depending on product.
-    * HBase - number of region servers
-    * Accumulo - number of tablet servers
-    * Storm - number of supervisors
-  * -n - The name of the app, this will be the display name in the resource manager and is used by the teardown process
-
-* HBase Example:
-
-
-    ./slider_setup -f slider_setup.conf -p hbase -w 5 -n hbase-slider
-
-* Accumulo Example:
-
-
-    ./slider_setup -f slider_setup.conf -p accumulo -w 3 -n accumulo-slider
-
-* Storm Example:
-
-
-    ./slider_setup -f slider_setup.conf -p storm -w 3 -n storm-slider
-
-Tear Down
----------
-
-* slider_destroy will do the following
-  1. Freeze the slider application based on provided name
-  2. Destory the slider application based on provided name
-
-* The following args are required
-  * `-f` - The path to the `slider_setup.conf` that has been modified with cluster info
-  * `-n` - The name of the app, this was provided to the slider_setup tool
-
-* HBase Example:
-
-
-    ./slider_destroy -f slider_setup.conf -n hbase-slider
-
-* Accumulo Example:
-
-
-    ./slider_destroy -f slider_setup.conf -n accumulo-slider
-
-* Storm Example:
-
-
-    ./slider_destroy -f slider_setup.conf -n storm-slider

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-install/pom.xml
----------------------------------------------------------------------
diff --git a/slider-install/pom.xml b/slider-install/pom.xml
deleted file mode 100644
index 8dd2ec6..0000000
--- a/slider-install/pom.xml
+++ /dev/null
@@ -1,111 +0,0 @@
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-  <artifactId>slider-install</artifactId>
-  <name>Slider Install</name>
-  <packaging>pom</packaging>
-  <description>
-    
-    Builds any RPMs and other install packaging for Slider.
-    This is downstream of the slider assembly and the app packages, so
-    that anything from these can be installed in the RPM.
-  </description>
-  <parent>
-    <groupId>org.apache.slider</groupId>
-    <artifactId>slider</artifactId>
-    <version>0.41.0-incubating-SNAPSHOT</version>
-  </parent>
-
-  <properties>
-    <basedir>/usr/local/slider</basedir>
-    <confdir>${basedir}/conf</confdir>
-    <bindir>${basedir}/bin</bindir>
-  </properties>
-  
-  <build>
-    <plugins>
-      <!--read in a build.properties file if defined-->
-      <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>properties-maven-plugin</artifactId>
-        <version>${maven.properties.version}</version>
-        <executions>
-          <execution>
-            <phase>initialize</phase>
-            <goals>
-              <goal>read-project-properties</goal>
-            </goals>
-            <configuration>
-              <quiet>true</quiet>
-              <files>
-                <file>build.properties</file>
-                <file>../build.properties</file>
-              </files>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-
-    </plugins>
-    
-    
-  </build>
-
-  <reporting>
-    <plugins>
- 
-
-
-    </plugins>
-  </reporting>
-
-  <dependencies>
-
-    <dependency>
-      <groupId>org.apache.slider</groupId>
-      <artifactId>slider-core</artifactId>
-      <version>${project.version}</version>
-    </dependency>
-
-    <!--
-     needed to order the build and ensure the agent tar is found
-     the test scope ensures that it isn't copied into the lib dir
-     -->
-    <dependency>
-      <groupId>org.apache.slider</groupId>
-      <artifactId>slider-agent</artifactId>
-      <version>${project.version}</version>
-      <scope>test</scope>
-      <type>tar.gz</type>
-    </dependency>
-
-    <dependency>
-      <groupId>com.beust</groupId>
-      <artifactId>jcommander</artifactId>
-    </dependency>
-
-
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-client</artifactId>
-    </dependency>
- 
-  </dependencies>
-
-
-</project>

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-install/src/main/bash/slider-client.xml
----------------------------------------------------------------------
diff --git a/slider-install/src/main/bash/slider-client.xml b/slider-install/src/main/bash/slider-client.xml
deleted file mode 100644
index f7060c1..0000000
--- a/slider-install/src/main/bash/slider-client.xml
+++ /dev/null
@@ -1,83 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<!--
-  Properties set here are picked up in the client.
-  They are not passed to the AM -though the filesystem
-  binding details (URL And principal) are added to the
-  hbase-site.xml file when a cluster is created.
--->
-<configuration>
-
-  <property>
-    <name>yarn.log-aggregation-enable</name>
-    <value>true</value>
-  </property>
-  
-
-  <property>
-    <name>slider.yarn.queue</name>
-    <value>default</value>
-    <description>YARN queue for the Application Master</description>
-  </property>
-  
-  <property>
-    <name>yarn.resourcemanager.address</name>
-    <value>@@RM_ADDRESS@@</value>
-  </property>
-
-  <property>
-    <name>yarn.resourcemanager.scheduler.address</name>
-    <value>@@RM_SCHED_ADDRESS@@</value>
-  </property>
-
-  <property>
-    <name>fs.defaultFS</name>
-    <value>@@DEFAULT_FS@@</value>
-  </property>
-
-  <property>
-     <name>yarn.application.classpath</name>
-     <value>@@YARN_CP@@</value>
-  </property>
-
-  <property>
-     <name>slider.zookeeper.quorum</name>
-     <value>@@ZK_QUORUM@@</value>
-  </property>
-
-<!--
-  <property>
-    <name>yarn.resourcemanager.principal</name>
-    <value>yarn/master@MINICLUSTER</value>
-  </property>
-
-  <property>
-    <name>slider.security.enabled</name>
-    <value>true</value>
-  </property>
-
-  <property>
-    <name>dfs.namenode.kerberos.principal</name>
-    <value>hdfs/master@MINICLUSTER</value>
-  </property>
--->
-
-
-</configuration>

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-install/src/main/bash/slider_destroy
----------------------------------------------------------------------
diff --git a/slider-install/src/main/bash/slider_destroy b/slider-install/src/main/bash/slider_destroy
deleted file mode 100755
index 9039751..0000000
--- a/slider-install/src/main/bash/slider_destroy
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#
-# Base Vars
-#
-SCRIPT_NAME=`basename $0`
-SCRIPT_DIR=`cd $(dirname $0) && pwd`
-
-#
-# Functions
-#
-usage() { 
-    echo "Usage: $SCRIPT_NAME -f </path/to/config> -n <app name>"
-    exit 1
-}
-
-#
-# Parse cmd line args
-#
-while getopts "f:n:" opt; do
-    case "$opt" in
-        f) config=$OPTARG;;
-        n) app_name=$OPTARG;;
-        *) usage;;
-    esac
-done
-shift $((OPTIND-1))
-
-if [ -z "$config" ] || [ -z "$app_name" ]; then
-    usage
-fi
-
-#
-# Source the config
-#
-source $config
-
-#
-# Main
-#
-echo -e "\n## Freezing app $app_name"
-sudo -u yarn $SLIDER_INST_DIR/bin/slider freeze $app_name --manager $RM_ADDRESS || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Destroying app $app_name"
-sudo -u yarn $SLIDER_INST_DIR/bin/slider destroy $app_name --manager $RM_ADDRESS || exit 1
-echo "SUCCESS"
-
-exit 0

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-install/src/main/bash/slider_setup
----------------------------------------------------------------------
diff --git a/slider-install/src/main/bash/slider_setup b/slider-install/src/main/bash/slider_setup
deleted file mode 100755
index 2c15c95..0000000
--- a/slider-install/src/main/bash/slider_setup
+++ /dev/null
@@ -1,173 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-#
-# Base Vars
-#
-SCRIPT_NAME=`basename $0`
-SCRIPT_DIR=`cd $(dirname $0) && pwd`
-
-#
-# Functions
-#
-usage() { 
-    echo "Usage: $SCRIPT_NAME -f </path/to/config> -p <product (hbase or storm)> -w <worker count> -n <app name>"
-    exit 1
-}
-
-#
-# Parse cmd line args
-#
-while getopts "f:p:w:n:" opt; do
-    case "$opt" in
-        f) config=$OPTARG;;
-        p) product=$OPTARG;;
-        w) worker_cnt=$OPTARG;;
-        n) app_name=$OPTARG;;
-        *) usage;;
-    esac
-done
-shift $((OPTIND-1))
-
-if [ -z "$config" ] || [ -z "$product" ] || [ -z "$worker_cnt" ] || [ -z "$app_name" ]; then
-    usage
-fi
-
-# Only support know products for now
-if [ $product != "hbase" ] && [ $product != "storm" ] && [ $product != "accumulo"]; then
-   echo "ERROR: Only HBase, Storm, and Accumulo are currently supported"
-   usage
-fi
-
-#
-# Source the config
-#
-source $config
-
-#
-# Product URLs
-#
-if [ $product = "hbase" ]; then
-    app_url=$HBASE_APP_URL
-elif [ $product = "storm" ]; then
-    app_url=$STORM_APP_URL
-elif [ $product = "accumulo" ]; then
-    app_url=$ACCUMULO_APP_URL
-fi
-
-#
-# Main
-#
-echo -e "\n## Creating slider install dir: $SLIDER_INST_DIR"
-mkdir -p $SLIDER_INST_DIR || exit 1
-chown yarn:hadoop $SLIDER_INST_DIR || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Downloading slider from: $SLIDER_URL"
-if [ -f /tmp/${SLIDER_VER}*tar ]; then
-   rm -f /tmp/${SLIDER_VER}*tar
-fi
-cd /tmp && wget $SLIDER_URL || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Extracting slider to $SLIDER_INST_DIR"
-tar -xf /tmp/${SLIDER_VER}*tar --strip-components=1 -C $SLIDER_INST_DIR || exit 1
-chown -R yarn:hadoop $SLIDER_INST_DIR || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Setting conf values"
-(cd $SCRIPT_DIR && cp slider-client.xml $SLIDER_INST_DIR/conf) || exit 1
-sed -i 's|@@RM_ADDRESS@@|'$RM_ADDRESS'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
-sed -i 's|@@RM_SCHED_ADDRESS@@|'$RM_SCHED_ADDRESS'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
-sed -i 's|@@DEFAULT_FS@@|'$DEFAULT_FS'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
-sed -i 's|@@YARN_CP@@|'$YARN_CP'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
-sed -i 's|@@ZK_QUORUM@@|'$ZK_QUORUM'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Setting PATH to include the JDK bin: $JDK_BIN"
-export PATH=$PATH:$JDK_BIN
-echo "SUCCESS"
-
-echo -e "\n## Checking version of Hadoop slider was compiled against"
-hadoop_compiled_ver=`$SLIDER_INST_DIR/bin/slider version | grep "Compiled against Hadoop" | awk '{print $NF}'`
-if [ "$hadoop_compiled_ver" != "2.4.0" ]; then
-   echo "ERROR: Compiled against Hadoop version $hadoop_compiled_ver instead of 2.4.0" && exit 1
-else
-   echo "Compiled against Hadoop version: $hadoop_compiled_ver"
-fi
-echo "SUCCESS"
-
-echo -e "\n## Setting up HDFS directories for slider"
-sudo -u hdfs hdfs dfs -mkdir -p /slider || exit 1
-sudo -u hdfs hdfs dfs -chown yarn:hdfs /slider || exit 1
-sudo -u hdfs hdfs dfs -mkdir -p /user/yarn || exit 1
-sudo -u hdfs hdfs dfs -chown yarn:hdfs /user/yarn || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Loading the Slider agent"
-sudo -u yarn hdfs dfs -mkdir -p /slider/agent/conf || exit 1
-sudo -u yarn hdfs dfs -copyFromLocal $SLIDER_INST_DIR/agent/* /slider/agent
-echo "SUCCESS"
-
-echo -e "\n## Downloading $product to $SLIDER_INST_DIR/apps/$product"
-sudo -u yarn mkdir -p $SLIDER_INST_DIR/apps/$product || exit 1
-(cd $SLIDER_INST_DIR/apps/$product && sudo -u yarn wget $app_url) || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Extracting $product in $SLIDER_INST_DIR/apps/$product"
-(cd $SLIDER_INST_DIR/apps/$product && sudo -u yarn unzip -o $product*zip) || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Adding $product to HDFS slider dir"
-sudo -u yarn hdfs dfs -copyFromLocal $SLIDER_INST_DIR/apps/$product/$product*zip /slider
-echo "SUCCESS"
-
-echo -e "\n## Setting number of workers in $SLIDER_INST_DIR/apps/$product/resources.json"
-if [ $product = "hbase" ]; then
-   component="HBASE_REGIONSERVER"
-elif [ $product = "storm" ]; then
-   component="SUPERVISOR"
-elif [ $product = "accumulo" ]; then
-   component="ACCUMULO_TSERVER"
-fi
-# Update the resource.json file with worker_cnt
-python << END
-import json
-with open("$SLIDER_INST_DIR/apps/$product/resources.json", "r+") as f:
-    data = json.load(f)
-    data["components"]["$component"]["yarn.component.instances"] = "$worker_cnt"
-
-with open("$SLIDER_INST_DIR/apps/$product/resources.json", "w+") as f:
-    f.write(json.dumps(data, sort_keys = False, indent = 4))
-END
-echo "SUCCESS"
-
-# Handle HBase HDFS dir needs
-if [ $product = "hbase" ]; then
-    echo -e "\n## Creating hbase HDFS dir /apps/hbase"
-    sudo -u hdfs hdfs dfs -mkdir -p /apps/hbase || exit 1
-    sudo -u hdfs hdfs dfs -chown yarn:hdfs /apps/hbase || exit 1
-    echo "SUCCESS"
-fi
-
-echo -e "\n##Starting app $product with $worker_cnt workers via slider"
-sudo -u yarn $SLIDER_INST_DIR/bin/slider create $app_name \
-    --image $DEFAULT_FS/slider/agent/slider-agent.tar.gz \
-    --template $SLIDER_INST_DIR/apps/$product/appConfig.json \
-    --resources $SLIDER_INST_DIR/apps/$product/resources.json || exit 1
-echo "SUCCESS"

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-install/src/main/bash/slider_setup.conf
----------------------------------------------------------------------
diff --git a/slider-install/src/main/bash/slider_setup.conf b/slider-install/src/main/bash/slider_setup.conf
deleted file mode 100644
index fed3f4c..0000000
--- a/slider-install/src/main/bash/slider_setup.conf
+++ /dev/null
@@ -1,60 +0,0 @@
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Change values below to match the cluster being deployed on
-#
-
-# Zookeeper nodes
-ZK_QUORUM="zk1:2181,zk2:2181,zk3:2181"
-
-# Resource Manager address (yarn.resourcemanager.address)
-RM_ADDRESS="rm1:8050"
-
-# Resource Manager scheduler address (yarn.resourcemanager.scheduler.address)
-RM_SCHED_ADDRESS="rm1:8030"
-
-# Default FS (fs.defaultFS)
-DEFAULT_FS="hdfs://nn1:8020"
-
-
-#
-# Should not be necessary to change anything below except on new version releases
-#
-
-# Slider version
-SLIDER_VER="slider-0.30"
-
-# Slider URL
-SLIDER_URL="http://public-repo-1.hortonworks.com/slider/0.30/slider-0.30-all.tar"
-
-# HBase Slider App URL
-HBASE_APP_URL="http://public-repo-1.hortonworks.com/slider/0.30/apps/hbase_v096.zip"
-
-# Storm Slider App URL
-STORM_APP_URL="http://public-repo-1.hortonworks.com/slider/0.30/apps/storm_v091.zip"
-
-# Accumulo Slider App URL
-ACCUMULO_APP_URL="http://public-repo-1.hortonworks.com/slider/0.30/apps/accumulo_v151.zip"
-
-# Where to install Slider
-SLIDER_INST_DIR="/usr/lib/slider"
-
-# JDK bin to add to PATH
-JDK_BIN="/usr/jdk64/jdk1.7.0_45/bin"
-
-# Yarn container classpath (yarn.application.classpath)
-YARN_CP="/etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*"
-


[37/50] [abbrv] git commit: Merge branch 'feature/SLIDER-320-endpoints-in-roleinstances' into develop

Posted by st...@apache.org.
Merge branch 'feature/SLIDER-320-endpoints-in-roleinstances' into develop


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8050466f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8050466f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8050466f

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 8050466fc3d395e8a31fc760c612b9140e94e3a6
Parents: 4ea66fa 4610c25
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 15 16:11:25 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 15 16:11:25 2014 +0100

----------------------------------------------------------------------
 .../providers/agent/AgentProviderService.java   |  32 +++++-
 .../server/appmaster/SliderAppMaster.java       |  46 ++++++--
 .../actions/RegisterComponentInstance.java      |  48 ++++++++
 .../actions/UnregisterComponentInstance.java    |  46 ++++++++
 .../slider/server/appmaster/state/AppState.java | 115 ++++++++++++++-----
 .../appmaster/state/ProviderAppState.java       |  17 ++-
 .../server/appmaster/state/RoleInstance.java    |  42 ++++++-
 .../state/StateAccessForProviders.java          |  13 ++-
 .../appmaster/web/view/ContainerStatsBlock.java |   2 +-
 .../server/appmaster/web/view/IndexBlock.java   |   3 +-
 .../slider/providers/agent/TestAgentEcho.groovy |   2 +-
 .../providers/hbase/HBaseProviderService.java   |  11 +-
 12 files changed, 312 insertions(+), 65 deletions(-)
----------------------------------------------------------------------



[22/50] [abbrv] git commit: SLIDER-262. Slider agent should provide process supervision such as auto-restart.

Posted by st...@apache.org.
SLIDER-262. Slider agent should provide process supervision such as auto-restart.


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/806ebb27
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/806ebb27
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/806ebb27

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 806ebb273f7b66027c771c2f13e2c8ef95cbe829
Parents: 506eb79
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Aug 13 07:53:31 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Aug 14 17:56:52 2014 -0700

----------------------------------------------------------------------
 .../src/main/python/agent/ActionQueue.py        |   1 +
 .../src/main/python/agent/CommandStatusDict.py  |   2 +
 .../src/main/python/agent/Controller.py         |  11 +-
 .../src/test/python/agent/TestActionQueue.py    |   7 +-
 .../test/python/agent/TestCommandStatusDict.py  |  94 +++++++++++++++
 .../src/test/python/agent/TestController.py     | 116 ++++++++++++++++++-
 6 files changed, 224 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/806ebb27/slider-agent/src/main/python/agent/ActionQueue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/ActionQueue.py b/slider-agent/src/main/python/agent/ActionQueue.py
index 774d1e6..fba9aa2 100644
--- a/slider-agent/src/main/python/agent/ActionQueue.py
+++ b/slider-agent/src/main/python/agent/ActionQueue.py
@@ -135,6 +135,7 @@ class ActionQueue(threading.Thread):
       'reportResult': reportResult
     })
     self.commandStatuses.put_command_status(command, in_progress_status, reportResult)
+
     store_config = False
     if ActionQueue.STORE_APPLIED_CONFIG in command['commandParams']:
       store_config = 'true' == command['commandParams'][ActionQueue.STORE_APPLIED_CONFIG]

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/806ebb27/slider-agent/src/main/python/agent/CommandStatusDict.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CommandStatusDict.py b/slider-agent/src/main/python/agent/CommandStatusDict.py
index 9261e29..f20e0fa 100644
--- a/slider-agent/src/main/python/agent/CommandStatusDict.py
+++ b/slider-agent/src/main/python/agent/CommandStatusDict.py
@@ -114,12 +114,14 @@ class CommandStatusDict():
     grep = Grep()
     output = grep.tail(tmpout, Grep.OUTPUT_LAST_LINES)
     inprogress = self.generate_report_template(command)
+    reportResult = not command[Constants.AUTO_GENERATED]
     inprogress.update({
       'stdout': grep.filterMarkup(output),
       'stderr': tmperr,
       'structuredOut': tmpstructuredout,
       Constants.EXIT_CODE: 777,
       'status': ActionQueue.IN_PROGRESS_STATUS,
+      'reportResult': reportResult
     })
     return inprogress
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/806ebb27/slider-agent/src/main/python/agent/Controller.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Controller.py b/slider-agent/src/main/python/agent/Controller.py
index 77dcafb..6bdfc26 100644
--- a/slider-agent/src/main/python/agent/Controller.py
+++ b/slider-agent/src/main/python/agent/Controller.py
@@ -268,7 +268,8 @@ class Controller(threading.Thread):
           if len(stored_command) > 0:
              auto_start_command = self.create_start_command(stored_command)
              if auto_start_command:
-               self.updateStateBasedOnCommand([auto_start_command])
+               logger.info("Automatically adding a start command.")
+               self.updateStateBasedOnCommand([auto_start_command], False)
                self.addToQueue([auto_start_command])
           pass
 
@@ -343,21 +344,25 @@ class Controller(threading.Thread):
     pass
     logger.info("Controller stopped heart-beating.")
 
+
   def create_start_command(self, stored_command):
     taskId = int(stored_command['taskId'])
     taskId = taskId + 1
     stored_command['taskId'] = taskId
     stored_command['commandId'] = "{0}-1".format(taskId)
     stored_command[Constants.AUTO_GENERATED] = True
+    return stored_command
     pass
 
-  def updateStateBasedOnCommand(self, commands):
+
+  def updateStateBasedOnCommand(self, commands, createStatus=True):
     for command in commands:
       if command["roleCommand"] == "START":
         self.componentExpectedState = State.STARTED
         self.componentActualState = State.STARTING
         self.failureCount = 0
-        self.statusCommand = self.createStatusCommand(command)
+        if createStatus:
+          self.statusCommand = self.createStatusCommand(command)
 
       if command["roleCommand"] == "INSTALL":
         self.componentExpectedState = State.INSTALLED

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/806ebb27/slider-agent/src/test/python/agent/TestActionQueue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestActionQueue.py b/slider-agent/src/test/python/agent/TestActionQueue.py
index f0935a9..4a53c2e 100644
--- a/slider-agent/src/test/python/agent/TestActionQueue.py
+++ b/slider-agent/src/test/python/agent/TestActionQueue.py
@@ -274,6 +274,7 @@ class TestActionQueue(TestCase):
   def test_execute_command(self, status_update_callback_mock, open_mock, json_load_mock,
                            resolve_script_path_mock):
 
+    self.assertEqual.__self__.maxDiff = None
     tempdir = tempfile.gettempdir()
     config = MagicMock()
     config.get.return_value = "something"
@@ -344,7 +345,8 @@ class TestActionQueue(TestCase):
                 'role': u'HBASE_MASTER',
                 'actionId': '1-1',
                 'taskId': 3,
-                'exitcode': 777}
+                'exitcode': 777,
+                'reportResult': True}
     self.assertEqual(report['reports'][0], expected)
     # Continue command execution
     unfreeze_flag.set()
@@ -406,7 +408,8 @@ class TestActionQueue(TestCase):
                 'actionId': '1-1',
                 'taskId': 3,
                 'structuredOut': '',
-                'exitcode': 13}
+                'exitcode': 13,
+                'reportResult': True}
     self.assertEqual(len(report['reports']), 1)
     self.assertEqual(report['reports'][0], expected)
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/806ebb27/slider-agent/src/test/python/agent/TestCommandStatusDict.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestCommandStatusDict.py b/slider-agent/src/test/python/agent/TestCommandStatusDict.py
new file mode 100644
index 0000000..ee91de6
--- /dev/null
+++ b/slider-agent/src/test/python/agent/TestCommandStatusDict.py
@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import StringIO
+import ssl
+import unittest, threading
+import sys
+from CommandStatusDict import CommandStatusDict
+from mock.mock import patch, MagicMock, call, Mock
+import logging
+from threading import Event
+
+class TestCommandStatusDict(unittest.TestCase):
+
+  logger = logging.getLogger()
+
+  auto_hbase_install_command = {
+    'commandType': 'EXECUTION_COMMAND',
+    'role': u'HBASE',
+    'roleCommand': u'INSTALL',
+    'commandId': '1-1',
+    'taskId': 7,
+    "componentName": "HBASE_MASTER",
+    'clusterName': u'cc',
+    'serviceName': u'HDFS',
+    'auto_generated': True
+  }
+
+  @patch("__builtin__.open")
+  def test_generate_progress_report(self, open_mock):
+    csd = CommandStatusDict(None)
+    report = {}
+    report['tmpout'] = None
+    report['tmperr'] = None
+    report['structuredOut'] = None
+
+    # Make file read calls visible
+    def open_side_effect(file, mode):
+      if mode == 'r':
+        file_mock = MagicMock()
+        file_mock.read.return_value = "Read from " + str(file)
+        return file_mock
+      else:
+        return self.original_open(file, mode)
+
+    open_mock.side_effect = open_side_effect
+
+    inprogress = csd.generate_in_progress_report(self.auto_hbase_install_command, report)
+    expected = {
+      'status': 'IN_PROGRESS',
+      'stderr': 'Read from None',
+      'stdout': 'Read from None',
+      'clusterName': u'cc',
+      'structuredOut': '{}',
+      'reportResult': False,
+      'roleCommand': u'INSTALL',
+      'serviceName': u'HDFS',
+      'role': u'HBASE',
+      'actionId': '1-1',
+      'taskId': 7,
+      'exitcode': 777}
+    self.assertEqual(inprogress, expected)
+
+    self.auto_hbase_install_command['auto_generated'] = False
+    inprogress = csd.generate_in_progress_report(self.auto_hbase_install_command, report)
+    expected['reportResult'] = True
+    self.assertEqual(inprogress, expected)
+    pass
+
+if __name__ == "__main__":
+  logging.basicConfig(format='%(asctime)s %(message)s',level=logging.DEBUG)
+  unittest.main()
+
+
+
+

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/806ebb27/slider-agent/src/test/python/agent/TestController.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestController.py b/slider-agent/src/test/python/agent/TestController.py
index af62c67..91e12f1 100644
--- a/slider-agent/src/test/python/agent/TestController.py
+++ b/slider-agent/src/test/python/agent/TestController.py
@@ -42,7 +42,7 @@ class TestController(unittest.TestCase):
   @patch.object(hostname, "hostname")
   def setUp(self, hostname_method, NetUtil_mock, lockMock, threadMock):
 
-    Controller.logger = MagicMock()
+    #Controller.logger = MagicMock()
     lockMock.return_value = MagicMock()
     NetUtil_mock.return_value = MagicMock()
     hostname_method.return_value = "test_hostname"
@@ -589,9 +589,121 @@ class TestController(unittest.TestCase):
     self.controller.config = original_value
     pass
 
+  def test_create_start_command(self):
+    stored_command = {
+      'commandType': 'EXECUTION_COMMAND',
+      'role': u'HBASE_MASTER',
+      "componentName": "HBASE_MASTER",
+      'roleCommand': u'INSTALL',
+      'commandId': '1-1',
+      'taskId': 3,
+      'clusterName': u'cc',
+      'serviceName': u'HBASE',
+      'configurations': {'global': {}},
+      'configurationTags': {'global': {'tag': 'v1'}},
+      'auto_generated': False,
+      'roleParams': {'auto_restart':'false'},
+      'commandParams': {'script_type': 'PYTHON',
+                        'script': 'scripts/abc.py',
+                        'command_timeout': '600'}
+    }
+
+    expected = {
+      'commandType': 'EXECUTION_COMMAND',
+      'role': u'HBASE_MASTER',
+      "componentName": "HBASE_MASTER",
+      'roleCommand': u'INSTALL',
+      'commandId': '4-1',
+      'taskId': 4,
+      'clusterName': u'cc',
+      'serviceName': u'HBASE',
+      'configurations': {'global': {}},
+      'configurationTags': {'global': {'tag': 'v1'}},
+      'auto_generated': False,
+      'roleParams': {'auto_restart':'false'},
+      'commandParams': {'script_type': 'PYTHON',
+                        'script': 'scripts/abc.py',
+                        'command_timeout': '600'},
+      'auto_generated': True
+    }
+
+    modified_command = self.controller.create_start_command(stored_command)
+    self.assertEqual.__self__.maxDiff = None
+    self.assertEqual(modified_command, expected)
+
+  @patch.object(Controller.Controller, "createStatusCommand")
+  @patch.object(threading._Event, "wait")
+  @patch("time.sleep")
+  @patch("json.loads")
+  @patch("json.dumps")
+  def test_auto_start(self, dumpsMock, loadsMock, timeMock, waitMock, mock_createStatusCommand):
+    original_value = self.controller.config
+    self.controller.config = AgentConfig("", "")
+    out = StringIO.StringIO()
+    sys.stdout = out
+
+    heartbeat = MagicMock()
+    self.controller.heartbeat = heartbeat
+
+    dumpsMock.return_value = "data"
+
+    sendRequest = MagicMock(name="sendRequest")
+    self.controller.sendRequest = sendRequest
+
+    self.controller.responseId = 1
+    response = {"responseId":"2", "restartAgent":"false", "restartEnabled":'true'}
+    loadsMock.return_value = response
+
+    def one_heartbeat(*args, **kwargs):
+      self.controller.DEBUG_STOP_HEARTBEATING = True
+      return "data"
+
+    sendRequest.side_effect = one_heartbeat
+
+    actionQueue = MagicMock()
+    actionQueue.isIdle.return_value = True
+
+    # one successful request, after stop
+    self.controller.actionQueue = actionQueue
+    self.controller.componentActualState = State.INSTALLED
+    self.controller.componentExpectedState = State.STARTED
+    self.assertTrue(self.controller.componentActualState, State.INSTALLED)
+    self.controller.actionQueue.customServiceOrchestrator.stored_command = {
+      'commandType': 'EXECUTION_COMMAND',
+      'role': u'HBASE',
+      'roleCommand': u'START',
+      'commandId': '7-1',
+      'taskId': 7,
+      "componentName": "HBASE_MASTER",
+      'clusterName': u'cc',
+      'serviceName': u'HDFS'
+    }
+    addToQueue = MagicMock(name="addToQueue")
+    self.controller.addToQueue = addToQueue
+
+    self.controller.heartbeatWithServer()
+    self.assertTrue(sendRequest.called)
+
+    self.assertTrue(self.controller.componentActualState, State.STARTING)
+    self.assertTrue(self.controller.componentExpectedState, State.STARTED)
+    self.assertEquals(self.controller.failureCount, 0)
+    self.assertFalse(mock_createStatusCommand.called)
+    addToQueue.assert_has_calls([call([{
+      'commandType': 'EXECUTION_COMMAND',
+      'clusterName': u'cc',
+      'serviceName': u'HDFS',
+      'role': u'HBASE',
+      'taskId': 8,
+      'roleCommand': u'START',
+      'componentName': 'HBASE_MASTER',
+      'commandId': '8-1',
+      'auto_generated': True}])])
+    self.controller.config = original_value
+    pass
+
 
 if __name__ == "__main__":
-  logging.basicConfig(format='%(asctime)s %(message)s',level=logging.DEBUG)
+  logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)
   unittest.main()
 
 


[13/50] [abbrv] git commit: SLIDER-320 Action PublishEndpointDetails

Posted by st...@apache.org.
SLIDER-320 Action PublishEndpointDetails


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/74cca8a9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/74cca8a9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/74cca8a9

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 74cca8a9f351fbc25e7a617eacce8a330224e4a9
Parents: 52f1f6f
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 14 16:19:34 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 14 16:19:34 2014 +0100

----------------------------------------------------------------------
 .../actions/PublishEndpointDetails.java         | 45 ++++++++++++++++++++
 1 file changed, 45 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/74cca8a9/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishEndpointDetails.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishEndpointDetails.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishEndpointDetails.java
new file mode 100644
index 0000000..67198a2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishEndpointDetails.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.state.AppState;
+
+import java.util.concurrent.TimeUnit;
+
+public class PublishEndpointDetails extends AsyncAction {
+  
+
+  public final ContainerId containerId;
+
+  public PublishEndpointDetails(ContainerId containerId, long delay,
+      TimeUnit timeUnit) {
+    super("PublishEndpointDetails :" + containerId.toString(),
+        delay, timeUnit);
+    this.containerId = containerId;
+  }
+
+  @Override
+  public void execute(SliderAppMaster appMaster,
+      QueueAccess queueService,
+      AppState appState) throws Exception {
+    
+  }
+}


[33/50] [abbrv] git commit: SLIDER-320 rename activeContainers in appState (and all related methods) to ownedContainers ... differentiates from livecontainers better

Posted by st...@apache.org.
SLIDER-320 rename activeContainers in appState (and all related methods) to ownedContainers ... differentiates from livecontainers better


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7e2ce276
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7e2ce276
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7e2ce276

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 7e2ce2764153fea9a586d809fcf1ebd4f593255e
Parents: 41e0c5b
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 15 14:29:32 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 15 14:29:32 2014 +0100

----------------------------------------------------------------------
 .../server/appmaster/SliderAppMaster.java       |  2 +-
 .../slider/server/appmaster/state/AppState.java | 72 +++++++++++++-------
 .../appmaster/state/ProviderAppState.java       | 12 ++--
 .../state/StateAccessForProviders.java          |  6 +-
 .../appmaster/web/view/ContainerStatsBlock.java |  2 +-
 .../server/appmaster/web/view/IndexBlock.java   |  3 +-
 6 files changed, 61 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e2ce276/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index acd6870..9a16188 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -1565,7 +1565,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
       throws SliderException {
     log.info("containerLostContactWithProvider: container {} lost",
         containerId);
-    RoleInstance activeContainer = appState.getActiveContainer(containerId);
+    RoleInstance activeContainer = appState.getOwnedContainer(containerId);
     if (activeContainer != null) {
       executeRMOperations(appState.releaseContainer(containerId));
       // ask for more containers if needed

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e2ce276/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 0d2afcc..2f2eadd 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -163,7 +163,7 @@ public class AppState {
    * Hash map of the containers we have. This includes things that have
    * been allocated but are not live; it is a superset of the live list
    */
-  private final ConcurrentMap<ContainerId, RoleInstance> activeContainers =
+  private final ConcurrentMap<ContainerId, RoleInstance> ownedContainers =
     new ConcurrentHashMap<ContainerId, RoleInstance>();
 
   /**
@@ -792,23 +792,49 @@ public class AppState {
   }
 
 
-  public synchronized List<RoleInstance> cloneActiveContainerList() {
-    Collection<RoleInstance> values = activeContainers.values();
+  /**
+   * Clone the list of active (==owned) containers
+   * @return the list of role instances representing all owned containers
+   */
+  public synchronized List<RoleInstance> cloneOwnedContainerList() {
+    Collection<RoleInstance> values = ownedContainers.values();
     return new ArrayList<RoleInstance>(values);
   }
-  
 
-  public int getNumActiveContainers() {
-    return activeContainers.size();
+  /**
+   * Get the number of active (==owned) containers
+   * @return
+   */
+  public int getNumOwnedContainers() {
+    return ownedContainers.size();
   }
   
+  /**
+   * Look up an active container: any container that the AM has, even
+   * if it is not currently running/live
+   */
+  public RoleInstance getOwnedContainer(ContainerId id) {
+    return ownedContainers.get(id);
+  }
 
-  public RoleInstance getActiveContainer(ContainerId id) {
-    return activeContainers.get(id);
+  /**
+   * Remove an owned container
+   * @param id container ID
+   * @return the instance removed
+   */
+  private RoleInstance removeOwnedContainer(ContainerId id) {
+    return ownedContainers.remove(id);
   }
-  
-  private RoleInstance removeActiveContainer(ContainerId id) {
-    return activeContainers.remove(id);
+
+  /**
+   * set/update an owned container
+   * @param id container ID
+   * @param instance
+   * @return
+   */
+  private RoleInstance putOwnedContainer(ContainerId id,
+      RoleInstance instance) {
+    return ownedContainers.put(id, instance);
   }
 
   /**
@@ -885,7 +911,7 @@ public class AppState {
       boolean active) {
     List<RoleInstance> nodes = new ArrayList<RoleInstance>();
     Collection<RoleInstance> allRoleInstances;
-    allRoleInstances = active? activeContainers.values() : liveNodes.values();
+    allRoleInstances = active? ownedContainers.values() : liveNodes.values();
     for (RoleInstance node : allRoleInstances) {
       if (node.roleId == roleId) {
         nodes.add(node);
@@ -943,7 +969,7 @@ public class AppState {
     instance.container = container;
     instance.createTime = now();
     getStartingNodes().put(container.getId(), instance);
-    activeContainers.put(container.getId(), instance);
+    putOwnedContainer(container.getId(), instance);
     roleHistory.onContainerStartSubmitted(container, instance);
   }
 
@@ -960,7 +986,7 @@ public class AppState {
       throws SliderInternalStateException {
     ContainerId id = container.getId();
     //look up the container
-    RoleInstance instance = getActiveContainer(id);
+    RoleInstance instance = getOwnedContainer(id);
     if (instance == null) {
       throw new SliderInternalStateException(
         "No active container with ID " + id);
@@ -1110,7 +1136,7 @@ public class AppState {
   @VisibleForTesting
   public RoleInstance innerOnNodeManagerContainerStarted(ContainerId containerId) {
     incStartedCountainerCount();
-    RoleInstance instance = activeContainers.get(containerId);
+    RoleInstance instance = getOwnedContainer(containerId);
     if (instance == null) {
       //serious problem
       throw new YarnRuntimeException("Container not in active containers start "+
@@ -1146,7 +1172,7 @@ public class AppState {
    */
   public synchronized void onNodeManagerContainerStartFailed(ContainerId containerId,
                                                              Throwable thrown) {
-    activeContainers.remove(containerId);
+    removeOwnedContainer(containerId);
     incFailedCountainerCount();
     incStartFailedCountainerCount();
     RoleInstance instance = getStartingNodes().remove(containerId);
@@ -1246,7 +1272,7 @@ public class AppState {
     } else {
       //a container has failed 
       result.containerFailed = true;
-      roleInstance = activeContainers.remove(containerId);
+      roleInstance = removeOwnedContainer(containerId);
       if (roleInstance != null) {
         //it was active, move it to failed 
         incFailedCountainerCount();
@@ -1320,14 +1346,14 @@ public class AppState {
     }
     
     // and the active node list if present
-    removeActiveContainer(containerId);
+    removeOwnedContainer(containerId);
     
     // finally, verify the node doesn't exist any more
     assert !containersBeingReleased.containsKey(
         containerId) : "container still in release queue";
     assert !getLiveNodes().containsKey(
         containerId) : " container still in live nodes";
-    assert getActiveContainer(containerId) ==
+    assert getOwnedContainer(containerId) ==
            null : "Container still in active container list";
 
     return result;
@@ -1623,7 +1649,7 @@ public class AppState {
   public List<AbstractRMOperation> releaseContainer(ContainerId containerId)
       throws SliderInternalStateException {
     List<AbstractRMOperation> operations = new ArrayList<AbstractRMOperation>();
-    List<RoleInstance> activeRoleInstances = cloneActiveContainerList();
+    List<RoleInstance> activeRoleInstances = cloneOwnedContainerList();
     for (RoleInstance role : activeRoleInstances) {
       if (role.container.getId().equals(containerId)) {
         containerReleaseSubmitted(role.container);
@@ -1645,7 +1671,7 @@ public class AppState {
    * that can be released.
    */
   private RoleInstance findRoleInstanceOnHost(NodeInstance node, int roleId) {
-    Collection<RoleInstance> targets = cloneActiveContainerList();
+    Collection<RoleInstance> targets = cloneOwnedContainerList();
     String hostname = node.hostname;
     for (RoleInstance ri : targets) {
       if (hostname.equals(RoleHistoryUtils.hostnameOf(ri.container))
@@ -1663,7 +1689,7 @@ public class AppState {
    */
   public synchronized List<AbstractRMOperation> releaseAllContainers() {
 
-    Collection<RoleInstance> targets = cloneActiveContainerList();
+    Collection<RoleInstance> targets = cloneOwnedContainerList();
     log.info("Releasing {} containers", targets.size());
     List<AbstractRMOperation> operations =
       new ArrayList<AbstractRMOperation>(targets.size());
@@ -1817,7 +1843,7 @@ public class AppState {
     instance.container = container;
     instance.createTime = now();
     instance.state = ClusterDescription.STATE_LIVE;
-    activeContainers.put(cid, instance);
+    putOwnedContainer(cid, instance);
     //role history gets told
     roleHistory.onContainerAssigned(container);
     // pretend the container has just had its start actions submitted

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e2ce276/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
index a5b36d4..e7384c8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
@@ -164,18 +164,18 @@ public class ProviderAppState implements StateAccessForProviders {
   }
 
   @Override
-  public List<RoleInstance> cloneActiveContainerList() {
-    return appState.cloneActiveContainerList();
+  public List<RoleInstance> cloneOwnedContainerList() {
+    return appState.cloneOwnedContainerList();
   }
 
   @Override
-  public int getNumActiveContainers() {
-    return appState.getNumActiveContainers();
+  public int getNumOwnedContainers() {
+    return appState.getNumOwnedContainers();
   }
 
   @Override
-  public RoleInstance getActiveContainer(ContainerId id) {
-    return appState.getActiveContainer(id);
+  public RoleInstance getOwnedContainer(ContainerId id) {
+    return appState.getOwnedContainer(id);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e2ce276/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
index acba8cc..b236dfc 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
@@ -144,20 +144,20 @@ public interface StateAccessForProviders {
    * @return the active containers at the time
    * the call was made
    */
-  List<RoleInstance> cloneActiveContainerList();
+  List<RoleInstance> cloneOwnedContainerList();
 
   /**
    * Get the number of active containers
    * @return the number of active containers the time the call was made
    */
-  int getNumActiveContainers();
+  int getNumOwnedContainers();
 
   /**
    * Get any active container with the given ID
    * @param id container Id
    * @return the active container or null if it is not found
    */
-  RoleInstance getActiveContainer(ContainerId id);
+  RoleInstance getOwnedContainer(ContainerId id);
 
   /**
    * Create a clone of the list of live cluster nodes.

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e2ce276/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
index 5645e0e..95f0417 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
@@ -96,7 +96,7 @@ public class ContainerStatsBlock extends HtmlBlock {
   protected void render(Block html) {
     // TODO Probably better to just get a copy of this list for us to avoid the repeated synchronization?
     // does this change if we have 50 node, 100node, 500 node clusters?
-    final Map<String,RoleInstance> containerInstances = getContainerInstances(slider.getAppState().cloneActiveContainerList());
+    final Map<String,RoleInstance> containerInstances = getContainerInstances(slider.getAppState().cloneOwnedContainerList());
 
     for (Entry<String,RoleStatus> entry : slider.getRoleStatusByName().entrySet()) {
       final String name = entry.getKey();

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e2ce276/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
index 90a3ee7..54bdb09 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
@@ -30,7 +30,6 @@ import org.apache.slider.server.appmaster.web.WebAppApi;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.net.URL;
 import java.util.Map;
 import java.util.Map.Entry;
 
@@ -63,7 +62,7 @@ public class IndexBlock extends HtmlBlock {
 
     UL<DIV<Hamlet>> ul = div.ul();
 
-    ul.li("Total number of containers for cluster: " + appState.getNumActiveContainers());
+    ul.li("Total number of containers for cluster: " + appState.getNumOwnedContainers());
     ul.li("Cluster created: " + getInfoAvoidingNulls(StatusKeys.INFO_CREATE_TIME_HUMAN));
     ul.li("Cluster last flexed: " + getInfoAvoidingNulls(StatusKeys.INFO_FLEX_TIME_HUMAN));
     ul.li("Cluster running since: " + getInfoAvoidingNulls(StatusKeys.INFO_LIVE_TIME_HUMAN));


[29/50] [abbrv] git commit: SLIDER-326 regression: integration tests failing in/after freeze

Posted by st...@apache.org.
SLIDER-326 regression: integration tests failing in/after freeze


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b8c27622
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b8c27622
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b8c27622

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: b8c27622820a688e02c197f66b5e3725c4f18919
Parents: 15da279
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 15 13:46:35 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 15 13:46:35 2014 +0100

----------------------------------------------------------------------
 .../slider/funtest/framework/CommandTestBase.groovy | 16 +++++++++++++++-
 .../lifecycle/AgentClusterLifecycleIT.groovy        | 11 ++++++++---
 .../hbase/funtest/HBaseClusterLifecycleIT.groovy    | 16 ++++++++++++----
 .../hbase/funtest/HBaseIntegrationIT.groovy         |  5 ++++-
 .../providers/hbase/funtest/HBaseLoadIT.groovy      |  7 +++++++
 src/test/clusters/sandbox/slider/slider-client.xml  |  2 +-
 6 files changed, 47 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b8c27622/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index ba7be53..278bd2b 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -211,6 +211,20 @@ abstract class CommandTestBase extends SliderTestUtils {
     ])
   }
 
+
+  static SliderShell freeze(
+      int exitCode,
+      String name,
+      Collection<String> args) {
+    slider(exitCode, [ACTION_FREEZE, name] + args)
+  }
+
+  /**
+   * Freeze cluster: no exit code checking
+   * @param name
+   * @param args
+   * @return
+   */
   static SliderShell freeze(String name, Collection<String> args) {
     slider([ACTION_FREEZE, name] + args)
   }
@@ -288,7 +302,7 @@ abstract class CommandTestBase extends SliderTestUtils {
   }
 
   static SliderShell thaw(String name, Collection<String> args) {
-    slider([ACTION_THAW, name] + args)
+    slider(0, [ACTION_THAW, name] + args)
   }
 
   static SliderShell registry(int result, Collection<String> commands) {

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b8c27622/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
index b30d670..6b0f2bd 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
@@ -126,7 +126,7 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
       log.info("Connected via Client {}", sliderClient.toString())
 
       //freeze
-      freeze(CLUSTER, [
+      freeze(0, CLUSTER, [
           ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
           ARG_MESSAGE, "freeze-in-test-cluster-lifecycle"
       ])
@@ -145,7 +145,7 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
       exists(0, CLUSTER)
       describe " >>> Cluster is now thawed."
 
-      freeze(CLUSTER,
+      freeze(0, CLUSTER,
           [
               ARG_FORCE,
               ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
@@ -178,7 +178,12 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
           StatusKeys.INFO_CONTAINERS_AM_RESTART)
       assert restarted != null
       assert Integer.parseInt(restarted) == 0
-      freeze(CLUSTER)
+      freeze(0, CLUSTER,
+          [
+              ARG_FORCE,
+              ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
+              ARG_MESSAGE, "final-shutdown"
+          ])
 
       destroy(0, CLUSTER)
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b8c27622/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy
index aca12ef..63b5fb6 100644
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy
@@ -118,11 +118,13 @@ public class HBaseClusterLifecycleIT extends HBaseCommandTestBase
       log.info("Connected via Client {}", sliderClient.toString())
 
       //freeze
-      freeze(CLUSTER, [
+      def frozen = freeze(0, CLUSTER, [
           ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
-          ARG_MESSAGE, "freeze-in-test cluster lifecycle"
+          ARG_MESSAGE, "freeze-in-test-cluster-lifecycle"
       ])
+      frozen.assertExitCode(0)
 
+//      sleep(FREEZE_WAIT_TIME)
       //cluster exists if you don't want it to be live
       exists(0, CLUSTER, false)
       // condition returns false if it is required to be live
@@ -136,7 +138,7 @@ public class HBaseClusterLifecycleIT extends HBaseCommandTestBase
               ARG_WAIT, Integer.toString(THAW_WAIT_TIME),
           ])
       exists(0, CLUSTER)
-      freeze(CLUSTER,
+      freeze(0, CLUSTER,
           [
               ARG_FORCE,
               ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
@@ -165,7 +167,13 @@ public class HBaseClusterLifecycleIT extends HBaseCommandTestBase
           StatusKeys.INFO_CONTAINERS_AM_RESTART)
       assert restarted != null
       assert Integer.parseInt(restarted) == 0
-      freeze(CLUSTER)
+      freeze(0, CLUSTER,
+          [
+              ARG_FORCE,
+              ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
+              ARG_MESSAGE, "teardown-freeze"
+          ])
+      
 
       destroy(0, CLUSTER)
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b8c27622/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseIntegrationIT.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseIntegrationIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseIntegrationIT.groovy
index 69da02a..21a7494 100644
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseIntegrationIT.groovy
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseIntegrationIT.groovy
@@ -24,13 +24,16 @@ import org.apache.hadoop.hbase.IntegrationTestingUtility
 import org.apache.hadoop.util.ToolRunner
 import org.apache.slider.api.ClusterDescription
 import org.apache.slider.client.SliderClient
-import org.apache.slider.providers.hbase.HBaseConfigFileOptions;
+import org.apache.slider.providers.hbase.HBaseConfigFileOptions
+import org.junit.Ignore;
 
 /* Runs IntegrationTestIngest on cluster
  *
  * Note: this test runs for about 20 minutes
  * please set slider.test.timeout.seconds accordingly
  */
+
+@Ignore("appears localhost only")
 class HBaseIntegrationIT extends FunctionalHBaseClusterIT {
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b8c27622/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseLoadIT.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseLoadIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseLoadIT.groovy
index 7393186..61bcc70 100644
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseLoadIT.groovy
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseLoadIT.groovy
@@ -23,6 +23,8 @@ import org.apache.hadoop.hbase.util.LoadTestTool
 import org.apache.slider.api.ClusterDescription
 import org.apache.slider.client.SliderClient
 import org.apache.slider.providers.hbase.HBaseConfigFileOptions
+import org.junit.Assume
+import org.junit.Ignore
 
 class HBaseLoadIT extends FunctionalHBaseClusterIT {
 
@@ -32,6 +34,11 @@ class HBaseLoadIT extends FunctionalHBaseClusterIT {
   }
 
   @Override
+  void testHBaseCreateCluster() throws Throwable {
+    super.testHBaseCreateCluster()
+  }
+
+  @Override
   void clusterOperations(
       String clustername,
       SliderClient sliderClient,

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b8c27622/src/test/clusters/sandbox/slider/slider-client.xml
----------------------------------------------------------------------
diff --git a/src/test/clusters/sandbox/slider/slider-client.xml b/src/test/clusters/sandbox/slider/slider-client.xml
index 30937ec..5ac5d59 100644
--- a/src/test/clusters/sandbox/slider/slider-client.xml
+++ b/src/test/clusters/sandbox/slider/slider-client.xml
@@ -52,7 +52,7 @@
   <property>
     <name>slider.test.agent.enabled</name>
     <description>Flag to enable/disable Agent tests</description>
-    <value>false</value>
+    <value>true</value>
   </property>
 
 


[44/50] [abbrv] git commit: SLIDER-332 fixed a couple of license headers (Gour Saha via billie)

Posted by st...@apache.org.
SLIDER-332 fixed a couple of license headers (Gour Saha via billie)


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/624cafe7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/624cafe7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/624cafe7

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 624cafe78b4bbacd4ccb769edc06a4319b82ef26
Parents: 80eec9b
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Mon Aug 18 18:25:47 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Mon Aug 18 18:25:47 2014 -0700

----------------------------------------------------------------------
 slider-agent/src/main/python/kazoo/handlers/gevent.py    | 7 +++++--
 slider-agent/src/main/python/kazoo/handlers/threading.py | 5 +++--
 slider-agent/src/main/python/kazoo/tests/__init__.py     | 1 +
 3 files changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/624cafe7/slider-agent/src/main/python/kazoo/handlers/gevent.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/handlers/gevent.py b/slider-agent/src/main/python/kazoo/handlers/gevent.py
index 5e07143..060320c 100644
--- a/slider-agent/src/main/python/kazoo/handlers/gevent.py
+++ b/slider-agent/src/main/python/kazoo/handlers/gevent.py
@@ -1,5 +1,8 @@
-"""license: Apache License 2.0, see LICENSE for more details."""
-"""A gevent based handler."""
+"""
+license: Apache License 2.0, see LICENSE for more details.
+A gevent based handler.
+"""
+
 from __future__ import absolute_import
 
 import atexit

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/624cafe7/slider-agent/src/main/python/kazoo/handlers/threading.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/handlers/threading.py b/slider-agent/src/main/python/kazoo/handlers/threading.py
index b2e355a..3ca9a8f 100644
--- a/slider-agent/src/main/python/kazoo/handlers/threading.py
+++ b/slider-agent/src/main/python/kazoo/handlers/threading.py
@@ -1,5 +1,6 @@
-"""license: Apache License 2.0, see LICENSE for more details."""
-"""A threading based handler.
+"""
+license: Apache License 2.0, see LICENSE for more details.
+A threading based handler.
 
 The :class:`SequentialThreadingHandler` is intended for regular Python
 environments that use threads.

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/624cafe7/slider-agent/src/main/python/kazoo/tests/__init__.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/__init__.py b/slider-agent/src/main/python/kazoo/tests/__init__.py
index e69de29..901253b 100644
--- a/slider-agent/src/main/python/kazoo/tests/__init__.py
+++ b/slider-agent/src/main/python/kazoo/tests/__init__.py
@@ -0,0 +1 @@
+"""license: Apache License 2.0, see LICENSE for more details."""


[05/50] [abbrv] git commit: SLIDER-314 factor out internal keys into class InternalKeys

Posted by st...@apache.org.
SLIDER-314 factor out internal keys into class InternalKeys


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1f6490d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1f6490d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1f6490d5

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 1f6490d5f6ec8d73f5cdb6f04600aa180be25b81
Parents: b111268
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 13 15:03:34 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 13 15:03:34 2014 +0100

----------------------------------------------------------------------
 .../api/ClusterDescriptionOperations.java       |  12 +-
 .../org/apache/slider/api/InternalKeys.java     | 114 +++++++++++++++++++
 .../java/org/apache/slider/api/OptionKeys.java  |  69 +----------
 .../org/apache/slider/client/SliderClient.java  |   7 +-
 .../slider/core/build/InstanceBuilder.java      |   3 +-
 .../apache/slider/providers/ProviderUtils.java  |   9 +-
 .../slideram/SliderAMClientProvider.java        |   4 +-
 .../server/appmaster/SliderAppMaster.java       |  22 ++--
 .../agent/TestAgentProviderService.java         |   9 +-
 .../accumulo/AccumuloClientProvider.java        |   3 +-
 .../accumulo/AccumuloProviderService.java       |   9 +-
 .../providers/hbase/HBaseClientProvider.java    |   3 +-
 .../providers/hbase/HBaseProviderService.java   |   4 +-
 ...reezeReconfigureThawLiveRegionService.groovy |   4 +-
 14 files changed, 166 insertions(+), 106 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionOperations.java b/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionOperations.java
index 7e73a92..21ece2b 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionOperations.java
@@ -64,15 +64,15 @@ public class ClusterDescriptionOperations {
     MapOperations appOptions =
       aggregateConf.getAppConfOperations().getGlobalOptions();
 
-    cd.type = internalOptions.getOption(OptionKeys.INTERNAL_PROVIDER_NAME,
+    cd.type = internalOptions.getOption(InternalKeys.INTERNAL_PROVIDER_NAME,
                                 SliderProviderFactory.DEFAULT_CLUSTER_TYPE);
 
-    cd.dataPath = internalOptions.get(OptionKeys.INTERNAL_DATA_DIR_PATH);
+    cd.dataPath = internalOptions.get(InternalKeys.INTERNAL_DATA_DIR_PATH);
     cd.name = internalOptions.get(OptionKeys.APPLICATION_NAME);
-    cd.originConfigurationPath = internalOptions.get(OptionKeys.INTERNAL_SNAPSHOT_CONF_PATH);
-    cd.generatedConfigurationPath = internalOptions.get(OptionKeys.INTERNAL_GENERATED_CONF_PATH);
-    cd.setImagePath(internalOptions.get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH));
-    cd.setApplicationHome(internalOptions.get(OptionKeys.INTERNAL_APPLICATION_HOME));
+    cd.originConfigurationPath = internalOptions.get(InternalKeys.INTERNAL_SNAPSHOT_CONF_PATH);
+    cd.generatedConfigurationPath = internalOptions.get(InternalKeys.INTERNAL_GENERATED_CONF_PATH);
+    cd.setImagePath(internalOptions.get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH));
+    cd.setApplicationHome(internalOptions.get(InternalKeys.INTERNAL_APPLICATION_HOME));
     cd.setZkPath(appOptions.get(ZOOKEEPER_PATH));
     cd.setZkHosts(appOptions.get(ZOOKEEPER_QUORUM));
     

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
new file mode 100644
index 0000000..4045c0e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api;
+
+/**
+ * Keys for internal use, go into `internal.json` and not intended for normal
+ * use except when tuning Slider AM operations
+ */
+public interface InternalKeys {
+
+
+  /**
+   * Home dir of the app: {@value}
+   * If set, implies there is a home dir to use
+   */
+  String INTERNAL_APPLICATION_HOME = "internal.application.home";
+  /**
+   * Path to an image file containing the app: {@value}
+   */
+  String INTERNAL_APPLICATION_IMAGE_PATH = "internal.application.image.path";
+  /**
+   * Time in milliseconds to wait after forking any in-AM 
+   * process before attempting to start up the containers: {@value}
+   * 
+   * A shorter value brings the cluster up faster, but means that if the
+   * in AM process fails (due to a bad configuration), then time
+   * is wasted starting containers on a cluster that isn't going to come
+   * up
+   */
+  String INTERNAL_CONTAINER_STARTUP_DELAY = "internal.container.startup.delay";
+  /**
+   * internal temp directory: {@value}
+   */
+  String INTERNAL_AM_TMP_DIR = "internal.tmp.dir";
+  /**
+   * where a snapshot of the original conf dir is: {@value}
+   */
+  String INTERNAL_SNAPSHOT_CONF_PATH = "internal.snapshot.conf.path";
+  /**
+   * where a snapshot of the original conf dir is: {@value}
+   */
+  String INTERNAL_GENERATED_CONF_PATH = "internal.generated.conf.path";
+  /**
+   * where a snapshot of the original conf dir is: {@value}
+   */
+  String INTERNAL_PROVIDER_NAME = "internal.provider.name";
+  /**
+   * where a snapshot of the original conf dir is: {@value}
+   */
+  String INTERNAL_DATA_DIR_PATH = "internal.data.dir.path";
+  /**
+   * Time in milliseconds to wait after forking any in-AM 
+   * process before attempting to start up the containers: {@value}
+   *
+   * A shorter value brings the cluster up faster, but means that if the
+   * in AM process fails (due to a bad configuration), then time
+   * is wasted starting containers on a cluster that isn't going to come
+   * up
+   */
+  int DEFAULT_INTERNAL_CONTAINER_STARTUP_DELAY = 5000;
+  /**
+   * Version of the app: {@value}
+   */
+  String KEYTAB_LOCATION = "internal.keytab.location";
+
+
+  /**
+   * Flag to indicate whether or not the chaos monkey is enabled:
+   * {@value}
+   */
+  String INTERNAL_CHAOS_MONKEY_ENABLED = "internal.chaos.monkey.enabled";
+  boolean DEFAULT_INTERNAL_CHAOS_MONKEY_ENABLED = false;
+
+
+  /**
+   * Rate
+   */
+
+  String INTERNAL_CHAOS_MONKEY_RATE = "internal.chaos.monkey.rate";
+
+  int DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_DAYS = 0;
+  int DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_HOURS = 1;
+  int DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_MINUTES = 0;
+  
+  String INTERNAL_CHAOS_MONKEY_PROBABILITY =
+      "internal.chaos.monkey.probability";
+  /**
+   * Probabilies are out of 10000 ; 100==1%
+   */
+
+  String INTERNAL_CHAOS_MONKEY_PROBABILITY_AM_FAILURE = INTERNAL_CHAOS_MONKEY_PROBABILITY +".amfailure";
+  int DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE = 10;
+  String INTERNAL_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE =
+      INTERNAL_CHAOS_MONKEY_PROBABILITY + ".containerfailure";
+  int DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE = 100;
+
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java b/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
index 006e280..320cc4f 100644
--- a/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
@@ -22,18 +22,7 @@ package org.apache.slider.api;
  *  Keys for entries in the <code>options</code> section
  *  of a cluster description.
  */
-public interface OptionKeys {
-
-  /**
-   * Home dir of the app: {@value}
-   * If set, implies there is a home dir to use
-   */
-  String INTERNAL_APPLICATION_HOME = "internal.application.home";
-  
-  /**
-   * Path to an image file containing the app: {@value}
-   */
-  String INTERNAL_APPLICATION_IMAGE_PATH = "internal.application.image.path";
+public interface OptionKeys extends InternalKeys {
 
   /**
    * Time in milliseconds to wait after forking any in-AM 
@@ -44,72 +33,18 @@ public interface OptionKeys {
    * is wasted starting containers on a cluster that isn't going to come
    * up
    */
-  String INTERNAL_CONTAINER_STARTUP_DELAY = "internal.container.startup.delay";
-  
-  /**
-   * Time in milliseconds to wait after forking any in-AM 
-   * process before attempting to start up the containers: {@value}
-   * 
-   * A shorter value brings the cluster up faster, but means that if the
-   * in AM process fails (due to a bad configuration), then time
-   * is wasted starting containers on a cluster that isn't going to come
-   * up
-   */
   String APPLICATION_TYPE = "application.type";
   
-  /**
-   * Time in milliseconds to wait after forking any in-AM 
-   * process before attempting to start up the containers: {@value}
-   * 
-   * A shorter value brings the cluster up faster, but means that if the
-   * in AM process fails (due to a bad configuration), then time
-   * is wasted starting containers on a cluster that isn't going to come
-   * up
-   */
-  String APPLICATION_NAME = "application.name";
-
-  /**
-   * delay for container startup:{@value}
-   */
-  int DEFAULT_CONTAINER_STARTUP_DELAY = 5000;
 
-  /**
-   * Version of the app: {@value}
-   */
-  String KEYTAB_LOCATION = "internal.keytab.location";
+  String APPLICATION_NAME = "application.name";
 
   /**
    * Prefix for site.xml options: {@value}
    */
   String SITE_XML_PREFIX = "site.";
 
-  /**
-   * internal temp directory: {@value}
-   */
-  String INTERNAL_AM_TMP_DIR = "internal.tmp.dir";
 
   /**
-   * where a snapshot of the original conf dir is: {@value}
-   */
-  String INTERNAL_SNAPSHOT_CONF_PATH = "internal.snapshot.conf.path";
-  
-  /**
-   * where a snapshot of the original conf dir is: {@value}
-   */
-  String INTERNAL_GENERATED_CONF_PATH = "internal.generated.conf.path";
-    
-  /**
-   * where a snapshot of the original conf dir is: {@value}
-   */
-  String INTERNAL_PROVIDER_NAME = "internal.provider.name";
-  
-    
-  /**
-   * where a snapshot of the original conf dir is: {@value}
-   */
-  String INTERNAL_DATA_DIR_PATH = "internal.data.dir.path";
-  
-  /**
    * Zookeeper quorum host list: {@value}
    */
   String ZOOKEEPER_QUORUM = "zookeeper.quorum";

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 03f78be..1337570 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.slider.api.ClusterDescription;
 import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.InternalKeys;
 import org.apache.slider.api.OptionKeys;
 import org.apache.slider.api.ResourceKeys;
 import org.apache.slider.api.SliderClusterProtocol;
@@ -793,16 +794,16 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
       instanceDefinition.getAppConfOperations();
     Path generatedConfDirPath =
       createPathThatMustExist(internalOptions.getMandatoryOption(
-        OptionKeys.INTERNAL_GENERATED_CONF_PATH));
+        InternalKeys.INTERNAL_GENERATED_CONF_PATH));
     Path snapshotConfPath =
       createPathThatMustExist(internalOptions.getMandatoryOption(
-        OptionKeys.INTERNAL_SNAPSHOT_CONF_PATH));
+        InternalKeys.INTERNAL_SNAPSHOT_CONF_PATH));
 
 
     // cluster Provider
     AbstractClientProvider provider = createClientProvider(
       internalOptions.getMandatoryOption(
-        OptionKeys.INTERNAL_PROVIDER_NAME));
+        InternalKeys.INTERNAL_PROVIDER_NAME));
     // make sure the conf dir is valid;
     
     // now build up the image path

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
index 0a33c36..937b777 100644
--- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.slider.api.InternalKeys;
 import org.apache.slider.api.OptionKeys;
 import org.apache.slider.api.StatusKeys;
 import org.apache.slider.common.SliderXmlConfKeys;
@@ -139,7 +140,7 @@ public class InstanceBuilder {
                     instancePaths.dataPath.toUri());
 
 
-    internalOps.set(OptionKeys.INTERNAL_PROVIDER_NAME, provider);
+    internalOps.set(InternalKeys.INTERNAL_PROVIDER_NAME, provider);
     internalOps.set(OptionKeys.APPLICATION_NAME, clustername);
 
   }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index cb7d27a..db9f494 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.InternalKeys;
 import org.apache.slider.api.OptionKeys;
 import org.apache.slider.api.ResourceKeys;
 import org.apache.slider.api.RoleKeys;
@@ -282,9 +283,9 @@ public class ProviderUtils implements RoleKeys {
     MapOperations globalOptions =
       instanceDefinition.getInternalOperations().getGlobalOptions();
     String applicationHome =
-      globalOptions.get(OptionKeys.INTERNAL_APPLICATION_HOME);
+      globalOptions.get(InternalKeys.INTERNAL_APPLICATION_HOME);
     String imagePath =
-      globalOptions.get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+      globalOptions.get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
     return buildPathToHomeDir(imagePath, applicationHome, bindir, script);
   }
 
@@ -350,8 +351,8 @@ public class ProviderUtils implements RoleKeys {
                                 String script) throws FileNotFoundException {
     
     String homedir = buildPathToHomeDir(
-      internal.get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH),
-      internal.get(OptionKeys.INTERNAL_APPLICATION_HOME),
+      internal.get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH),
+      internal.get(InternalKeys.INTERNAL_APPLICATION_HOME),
       bindir,
       script);
     return buildScriptPath(bindir, script, homedir);

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
index 6aeb801..c3870b8 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.InternalKeys;
 import org.apache.slider.api.ResourceKeys;
 import org.apache.slider.api.RoleKeys;
 import org.apache.slider.common.SliderKeys;
@@ -123,7 +123,7 @@ public class SliderAMClientProvider extends AbstractClientProvider implements
       dataPath = instanceDefinition.getInternalOperations()
                                    .getGlobalOptions()
                                    .getMandatoryOption(
-                                     OptionKeys.INTERNAL_DATA_DIR_PATH);
+                                     InternalKeys.INTERNAL_DATA_DIR_PATH);
 
     Path path = new Path(dataPath);
     sliderFileSystem.verifyDirectoryWriteAccess(path);

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index ee0e59e..e6c7aba 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -56,7 +56,7 @@ import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.webapp.WebApps;
 import org.apache.slider.api.ClusterDescription;
-import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.InternalKeys;
 import org.apache.slider.api.ResourceKeys;
 import org.apache.slider.api.RoleKeys;
 import org.apache.slider.api.SliderClusterProtocol;
@@ -529,10 +529,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     serviceConf.set(SliderAmIpFilter.WS_CONTEXT_ROOT, WS_CONTEXT_ROOT + "|" + WS_AGENT_CONTEXT_ROOT);
     
     //get our provider
-    MapOperations globalInternalOptions =
-      instanceDefinition.getInternalOperations().getGlobalOptions();
+    MapOperations globalInternalOptions = getGlobalInternalOptions();
     String providerType = globalInternalOptions.getMandatoryOption(
-      OptionKeys.INTERNAL_PROVIDER_NAME);
+      InternalKeys.INTERNAL_PROVIDER_NAME);
     log.info("Cluster provider type is {}", providerType);
     SliderProviderFactory factory =
       SliderProviderFactory.createSliderProviderFactory(
@@ -751,7 +750,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     }
     String rolesTmpSubdir = appMasterContainerID.toString() + "/roles";
 
-    String amTmpDir = globalInternalOptions.getMandatoryOption(OptionKeys.INTERNAL_AM_TMP_DIR);
+    String amTmpDir = globalInternalOptions.getMandatoryOption(InternalKeys.INTERNAL_AM_TMP_DIR);
 
     Path tmpDirPath = new Path(amTmpDir);
     Path launcherTmpDirPath = new Path(tmpDirPath, rolesTmpSubdir);
@@ -927,9 +926,18 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
    * @return the generated configuration dir
    */
   public String getGeneratedConfDir() {
+    return getGlobalInternalOptions().get(
+        InternalKeys.INTERNAL_GENERATED_CONF_PATH);
+  }
+
+  /**
+   * Get the global internal options for the AM
+   * @return a map to access the internals
+   */
+  public MapOperations getGlobalInternalOptions() {
     return getInstanceDefinition()
       .getInternalOperations().
-      getGlobalOptions().get(OptionKeys.INTERNAL_GENERATED_CONF_PATH);
+      getGlobalOptions();
   }
 
   /**
@@ -1179,7 +1187,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
 
   /**
    * Schedule the failure window
-   * @param resources 
+   * @param resources the resource tree
    * @throws BadConfigException if the window is out of range
    */
   private void scheduleFailureWindowResets(ConfTree resources) throws

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index a9b9e99..22de0d2 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -27,13 +27,10 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.Token;
 import org.apache.slider.api.ClusterDescription;
 import org.apache.slider.api.ClusterDescriptionKeys;
 import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.InternalKeys;
 import org.apache.slider.api.OptionKeys;
 import org.apache.slider.common.SliderXmlConfKeys;
 import org.apache.slider.common.tools.SliderFileSystem;
@@ -218,7 +215,7 @@ public class TestAgentProviderService {
   public void testRegistration() throws IOException {
 
     ConfTree tree = new ConfTree();
-    tree.global.put(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");
+    tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");
 
     AgentProviderService aps = new AgentProviderService();
     ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class);
@@ -585,7 +582,7 @@ public class TestAgentProviderService {
     InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
     Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
     ConfTree tree = new ConfTree();
-    tree.global.put(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");
+    tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");
 
     AgentProviderService aps = new AgentProviderService();
     ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class);

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java
index db99360..7f99573 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.slider.api.InternalKeys;
 import org.apache.slider.common.SliderKeys;
 import org.apache.slider.common.SliderXmlConfKeys;
 import org.apache.slider.api.OptionKeys;
@@ -135,7 +136,7 @@ public class AccumuloClientProvider extends AbstractClientProvider implements
 
     propagateClientFSBinding(sitexml);
     setDatabasePath(sitexml,
-                    globalInstanceOptions.getMandatoryOption(OptionKeys.INTERNAL_DATA_DIR_PATH));
+                    globalInstanceOptions.getMandatoryOption(InternalKeys.INTERNAL_DATA_DIR_PATH));
 
 
     String quorum =

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
index c511efb..7dedd2c 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.service.Service;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.slider.api.InternalKeys;
 import org.apache.slider.common.SliderKeys;
 import org.apache.slider.api.ClusterDescription;
 import org.apache.slider.api.OptionKeys;
@@ -161,7 +162,7 @@ public class AccumuloProviderService extends AbstractProviderService implements
     //Add binaries
     //now add the image if it was set
     String imageURI = instanceDefinition.getInternalOperations()
-                                        .get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+                                        .get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
     fileSystem.maybeAddImagePath(launcher.getLocalResources(), imageURI);
 
     CommandLineBuilder commandLine = new CommandLineBuilder();
@@ -333,8 +334,8 @@ public class AccumuloProviderService extends AbstractProviderService implements
     //callback to AM to trigger cluster review is set up to happen after
     //the init/verify action has succeeded
     int delay = internalOperations.getGlobalOptions().getOptionInt(
-        OptionKeys.INTERNAL_CONTAINER_STARTUP_DELAY,
-        OptionKeys.DEFAULT_CONTAINER_STARTUP_DELAY);
+        InternalKeys.INTERNAL_CONTAINER_STARTUP_DELAY,
+        InternalKeys.DEFAULT_INTERNAL_CONTAINER_STARTUP_DELAY);
     ProviderCompletedCallable completedCallable =
         new ProviderCompletedCallable(execInProgress, null);
     Service notifier = new WorkflowCallbackService<>(
@@ -364,7 +365,7 @@ public class AccumuloProviderService extends AbstractProviderService implements
     String dataDir = cd.getInternalOperations()
                                .getGlobalOptions()
                                .getMandatoryOption(
-                                 OptionKeys.INTERNAL_DATA_DIR_PATH);
+                                 InternalKeys.INTERNAL_DATA_DIR_PATH);
     Path accumuloInited = new Path(dataDir, INSTANCE_ID);
     FileSystem fs2 = FileSystem.get(accumuloInited.toUri(), getConf());
     return fs2.exists(accumuloInited);

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
index cd104a2..76bf73a 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
@@ -21,6 +21,7 @@ package org.apache.slider.providers.hbase;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.slider.api.InternalKeys;
 import org.apache.slider.common.SliderKeys;
 import org.apache.slider.common.SliderXmlConfKeys;
 import org.apache.slider.api.OptionKeys;
@@ -122,7 +123,7 @@ public class HBaseClientProvider extends AbstractClientProvider implements
 
     sitexml.put(KEY_HBASE_ROOTDIR,
         globalInstanceOptions.getMandatoryOption(
-            OptionKeys.INTERNAL_DATA_DIR_PATH)
+            InternalKeys.INTERNAL_DATA_DIR_PATH)
     );
     providerUtils.propagateOption(globalAppOptions, OptionKeys.ZOOKEEPER_PATH,
                                   sitexml, KEY_ZNODE_PARENT);

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index f72967f..feffb08 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -22,9 +22,9 @@ package org.apache.slider.providers.hbase;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.slider.api.InternalKeys;
 import org.apache.slider.common.SliderKeys;
 import org.apache.slider.api.ClusterDescription;
-import org.apache.slider.api.OptionKeys;
 import org.apache.slider.api.RoleKeys;
 import org.apache.slider.api.StatusKeys;
 import org.apache.slider.core.conf.AggregateConf;
@@ -152,7 +152,7 @@ public class HBaseProviderService extends AbstractProviderService implements
     //Add binaries
     //now add the image if it was set
     String imageURI = instanceDefinition.getInternalOperations()
-                  .get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+                  .get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
     coreFS.maybeAddImagePath(launcher.getLocalResources(), imageURI);
 
     CommandLineBuilder cli = new CommandLineBuilder();

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeReconfigureThawLiveRegionService.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeReconfigureThawLiveRegionService.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeReconfigureThawLiveRegionService.groovy
index d50517c..f6748e0 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeReconfigureThawLiveRegionService.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeReconfigureThawLiveRegionService.groovy
@@ -25,9 +25,9 @@ import org.apache.hadoop.fs.FileSystem as HadoopFS
 import org.apache.hadoop.fs.Path
 import org.apache.hadoop.hbase.ClusterStatus
 import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.api.InternalKeys
 import org.apache.slider.core.main.ServiceLauncher
 import org.apache.slider.api.ClusterDescription
-import org.apache.slider.api.OptionKeys
 import org.apache.slider.core.build.InstanceIO
 import org.apache.slider.providers.hbase.HBaseKeys
 import org.apache.slider.common.tools.ConfigHelper
@@ -93,7 +93,7 @@ class TestFreezeReconfigureThawLiveRegionService
         clusterDir)
 
     def snapshotPath = instanceDefinition.internalOperations.get(
-        OptionKeys.INTERNAL_SNAPSHOT_CONF_PATH)
+        InternalKeys.INTERNAL_SNAPSHOT_CONF_PATH)
     assert snapshotPath != null
 
     Path confdir = new Path(snapshotPath);


[18/50] [abbrv] git commit: SLIDER-322 hbase and accumulo -> java6 language

Posted by st...@apache.org.
SLIDER-322 hbase and accumulo -> java6 language


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/280afd88
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/280afd88
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/280afd88

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 280afd88960457864e214da9254242c57a023fc6
Parents: 397585a
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 14 17:47:01 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 14 17:47:01 2014 +0100

----------------------------------------------------------------------
 .../accumulo/AccumuloProviderService.java       | 19 ++++++++++--
 .../providers/hbase/HBaseClientProvider.java    |  6 ++--
 .../providers/hbase/HBaseProviderService.java   | 31 +++++++++++++++++++-
 3 files changed, 49 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/280afd88/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
index 7dedd2c..b8f4c00 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
@@ -170,6 +170,8 @@ public class AccumuloProviderService extends AbstractProviderService implements
     String heap = "-Xmx" + appComponent.getOption(RoleKeys.JVM_HEAP, DEFAULT_JVM_HEAP);
     String opt = "ACCUMULO_OTHER_OPTS";
     if (SliderUtils.isSet(heap)) {
+/* JDK7
+
       switch (role) {
         case AccumuloKeys.ROLE_MASTER:
           opt = "ACCUMULO_MASTER_OPTS";
@@ -184,6 +186,16 @@ public class AccumuloProviderService extends AbstractProviderService implements
           opt = "ACCUMULO_GC_OPTS";
           break;
       }
+*/
+      if (AccumuloKeys.ROLE_MASTER.equals(role)) {
+        opt = "ACCUMULO_MASTER_OPTS";
+      } else if (AccumuloKeys.ROLE_TABLET.equals(role)) {
+        opt = "ACCUMULO_TSERVER_OPTS";
+      } else if (AccumuloKeys.ROLE_MONITOR.equals(role)) {
+        opt = "ACCUMULO_MONITOR_OPTS";
+      } else if (AccumuloKeys.ROLE_GARBAGE_COLLECTOR.equals(role)) {
+        opt = "ACCUMULO_GC_OPTS";
+      }
       launcher.setEnv(opt, heap);
     }
 
@@ -239,7 +251,7 @@ public class AccumuloProviderService extends AbstractProviderService implements
 
 
     String accumuloScript = AccumuloClientProvider.buildScriptBinPath(instance);
-    List<String> launchSequence = new ArrayList<>(8);
+    List<String> launchSequence = new ArrayList<String>(8);
     launchSequence.add(0, accumuloScript);
     Collections.addAll(launchSequence, commands);
     return launchSequence;
@@ -338,7 +350,8 @@ public class AccumuloProviderService extends AbstractProviderService implements
         InternalKeys.DEFAULT_INTERNAL_CONTAINER_STARTUP_DELAY);
     ProviderCompletedCallable completedCallable =
         new ProviderCompletedCallable(execInProgress, null);
-    Service notifier = new WorkflowCallbackService<>(
+    // JDK7
+    Service notifier = new WorkflowCallbackService(
         "accumulo notifier",
         completedCallable,
         delay,
@@ -389,7 +402,7 @@ public class AccumuloProviderService extends AbstractProviderService implements
   @Override
   public Map<String, String> buildProviderStatus() {
     
-    Map<String,String> status = new HashMap<>();
+    Map<String,String> status = new HashMap<String, String>();
     
     
     return status;

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/280afd88/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
index 76bf73a..9ad872f 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
@@ -108,7 +108,7 @@ public class HBaseClientProvider extends AbstractClientProvider implements
 
     MapOperations worker = appconf.getMandatoryComponent(HBaseKeys.ROLE_WORKER);
     
-    Map<String, String> sitexml = new HashMap<>();
+    Map<String, String> sitexml = new HashMap<String, String>();
 
     //map all cluster-wide site. options
     providerUtils.propagateSiteOptions(globalAppOptions, sitexml);
@@ -210,7 +210,7 @@ public class HBaseClientProvider extends AbstractClientProvider implements
     }
   }
 
-  private static Set<String> knownRoleNames = new HashSet<>();
+  private static Set<String> knownRoleNames = new HashSet<String>();
   static {
     List<ProviderRole> roles = HBaseRoles.getRoles();
     knownRoleNames.add(SliderKeys.COMPONENT_AM);
@@ -263,7 +263,7 @@ public class HBaseClientProvider extends AbstractClientProvider implements
 
     // add any and all dependency files
     Map<String, LocalResource> providerResources =
-        new HashMap<>();
+        new HashMap<String, LocalResource>();
 
     ProviderUtils.addProviderJar(providerResources,
         this,

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/280afd88/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index feffb08..1fb392c 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -180,6 +180,8 @@ public class HBaseProviderService extends AbstractProviderService implements
     String roleCommand;
     String logfile;
     //now look at the role
+
+/* JDK7
     switch (role) {
       case ROLE_WORKER:
         //role is region server
@@ -210,6 +212,33 @@ public class HBaseProviderService extends AbstractProviderService implements
         throw new SliderInternalStateException("Cannot start role %s", role);
     }
 
+*/
+    if (ROLE_WORKER.equals(role)) {
+      //role is region server
+      roleCommand = REGION_SERVER;
+      logfile = "/region-server.txt";
+      
+    } else if (ROLE_MASTER.equals(role)) {
+      roleCommand = MASTER;
+      logfile = "/master.txt";
+
+    } else if (ROLE_REST_GATEWAY.equals(role)) {
+      roleCommand = REST_GATEWAY;
+      logfile = "/rest-gateway.txt";
+
+    } else if (ROLE_THRIFT_GATEWAY.equals(role)) {
+      roleCommand = THRIFT_GATEWAY;
+      logfile = "/thrift-gateway.txt";
+
+    } else if (ROLE_THRIFT2_GATEWAY.equals(role)) {
+      roleCommand = THRIFT2_GATEWAY;
+      logfile = "/thrift2-gateway.txt";
+    }
+    
+    else {
+      throw new SliderInternalStateException("Cannot start role %s", role);
+    }
+    
     cli.add(roleCommand);
     cli.add(ACTION_START);
     //log details
@@ -319,7 +348,7 @@ public class HBaseProviderService extends AbstractProviderService implements
    * @return the provider status - map of entries to add to the info section
    */
   public Map<String, String> buildProviderStatus() {
-    Map<String, String> stats = new HashMap<>();
+    Map<String, String> stats = new HashMap<String, String>();
 
     return stats;
   }


[06/50] [abbrv] git commit: SLIDER-202: tweak mock tests to automatically get name from test method unless overridden.

Posted by st...@apache.org.
SLIDER-202: tweak mock tests to automatically get name from test method unless overridden.


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8ed52a90
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8ed52a90
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8ed52a90

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 8ed52a90ef413931bf48e23a6d4565694a95bdb2
Parents: 1f6490d
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 13 15:04:36 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 13 15:04:36 2014 +0100

----------------------------------------------------------------------
 .../model/appstate/TestMockAppStateContainerFailure.groovy   | 2 +-
 .../model/appstate/TestMockAppStateDynamicRoles.groovy       | 2 +-
 .../model/appstate/TestMockAppStateFlexDynamicRoles.groovy   | 2 +-
 .../appmaster/model/appstate/TestMockAppStateFlexing.groovy  | 2 +-
 .../model/appstate/TestMockAppStateRMOperations.groovy       | 2 +-
 .../model/appstate/TestMockAppStateRebuildOnAMRestart.groovy | 2 +-
 .../model/appstate/TestMockAppStateRolePlacement.groovy      | 2 +-
 .../model/appstate/TestMockAppStateRoleRelease.groovy        | 2 +-
 .../appstate/TestMockContainerResourceAllocations.groovy     | 2 +-
 .../server/appmaster/model/mock/BaseMockAppStateTest.groovy  | 8 +++++++-
 10 files changed, 16 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ed52a90/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
index c0f41f4..068b876 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
@@ -43,7 +43,7 @@ class TestMockAppStateContainerFailure extends BaseMockAppStateTest
 
   @Override
   String getTestName() {
-    return "TestAppStateContainerFailure"
+    return "TestMockAppStateContainerFailure"
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ed52a90/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
index 1b01f76..136e1ea 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
@@ -42,7 +42,7 @@ class TestMockAppStateDynamicRoles extends BaseMockAppStateTest
 
   @Override
   String getTestName() {
-    return "TestAppStateDynamicRoles"
+    return "TestMockAppStateDynamicRoles"
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ed52a90/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
index aa330f8..5c9dce9 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
@@ -44,7 +44,7 @@ class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
 
   @Override
   String getTestName() {
-    return "TestAppStateDynamicRoles"
+    return "TestMockAppStateFlexDynamicRoles"
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ed52a90/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
index 80937dc..a7bf068 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
@@ -33,7 +33,7 @@ class TestMockAppStateFlexing extends BaseMockAppStateTest implements MockRoles
 
   @Override
   String getTestName() {
-    return "TestMockFlexing"
+    return "TestMockAppStateFlexing"
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ed52a90/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
index 6a5df5c..f8e852e 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
@@ -40,7 +40,7 @@ class TestMockAppStateRMOperations extends BaseMockAppStateTest implements MockR
 
   @Override
   String getTestName() {
-    return "TestMockRMOperations"
+    return "TestMockAppStateRMOperations"
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ed52a90/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
index c142c77..c2783f3 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
@@ -40,7 +40,7 @@ class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest
 
   @Override
   String getTestName() {
-    return "TestAppStateRebuildOnAMRestart"
+    return "TestMockAppStateRebuildOnAMRestart"
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ed52a90/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
index e7ddf79..17ebc31 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
@@ -43,7 +43,7 @@ class TestMockAppStateRolePlacement extends BaseMockAppStateTest
 
   @Override
   String getTestName() {
-    return "TestAppStateRolePlacement"
+    return "TestMockAppStateRolePlacement"
   }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ed52a90/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.groovy
index de4d493..addfaa5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.groovy
@@ -38,7 +38,7 @@ class TestMockAppStateRoleRelease extends BaseMockAppStateTest
 
   @Override
   String getTestName() {
-    return "TestAppStateRolePlacement"
+    return "TestMockAppStateRoleRelease"
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ed52a90/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
index 0bf9a90..73d40ee 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
@@ -39,7 +39,7 @@ class TestMockContainerResourceAllocations extends BaseMockAppStateTest {
 
   @Override
   String getTestName() {
-    "TestContainerResourceAllocations"
+    "TestMockContainerResourceAllocations"
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ed52a90/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
index 289b7c5..f96a238 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -101,7 +101,13 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
     factory.newInstanceDefinition(0, 0, 0)
   }
 
-  abstract String getTestName();
+  /**
+   * Get the test name ... defaults to method name
+   * @return
+   */
+  String getTestName() {
+    methodName.methodName;
+  }
 
   public RoleStatus getRole0Status() {
     return appState.lookupRoleStatus(ROLE0)


[38/50] [abbrv] git commit: SLIDER-324. Exports should be attributed to individual master components rather than only one master component

Posted by st...@apache.org.
SLIDER-324. Exports should be attributed to individual master components rather than only one master component


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bc9561d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bc9561d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bc9561d5

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: bc9561d59d3c0a68a6abaa47e8a91b87a36a1287
Parents: 599e972
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Fri Aug 15 13:32:12 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Fri Aug 15 13:32:12 2014 -0700

----------------------------------------------------------------------
 app-packages/storm/metainfo.xml                 | 28 ++++----
 .../agent/application/metadata/Component.java   | 21 ++++--
 .../application/metadata/ComponentExport.java   | 54 ++++++++++++++
 .../application/metadata/MetainfoParser.java    |  9 +--
 .../agent/TestAgentProviderService.java         | 23 +++---
 .../agent/application/metadata/metainfo.xml     | 74 ++++++++++++++++++--
 6 files changed, 171 insertions(+), 38 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc9561d5/app-packages/storm/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index e94bc62..d9004ad 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -29,21 +29,25 @@
         <name>QuickLinks</name>
         <exports>
           <export>
-            <name>org.apache.slider.jmx</name>
+            <name>app.jmx</name>
             <value>http://${STORM_REST_API_HOST}:${site.global.rest_api_port}/api/cluster/summary</value>
           </export>
           <export>
-            <name>org.apache.slider.monitor</name>
+            <name>app.monitor</name>
             <value>http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}</value>
           </export>
           <export>
-            <name>org.apache.slider.metrics</name>
+            <name>app.metrics</name>
             <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value>
           </export>
           <export>
-            <name>org.apache.slider.ganglia</name>
+            <name>ganglia.ui</name>
             <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value>
           </export>
+          <export>
+            <name>nimbus.url</name>
+            <value>http://${NIMBUS_HOST}:${site.storm-site.nimbus.thrift.port}</value>
+          </export>
         </exports>
       </exportGroup>
     </exportGroups>
@@ -78,6 +82,7 @@
         <name>NIMBUS</name>
         <category>MASTER</category>
         <autoStartOnFailure>true</autoStartOnFailure>
+        <appExports>QuickLinks-nimbus.url,QuickLinks-ganglia.ui,QuickLinks-app.metrics</appExports>
         <commandScript>
           <script>scripts/nimbus.py</script>
           <scriptType>PYTHON</scriptType>
@@ -89,6 +94,7 @@
         <name>STORM_REST_API</name>
         <category>MASTER</category>
         <autoStartOnFailure>true</autoStartOnFailure>
+        <appExports>QuickLinks-app.jmx</appExports>
         <commandScript>
           <script>scripts/rest_api.py</script>
           <scriptType>PYTHON</scriptType>
@@ -100,12 +106,12 @@
         <name>SUPERVISOR</name>
         <category>SLAVE</category>
         <autoStartOnFailure>true</autoStartOnFailure>
-        <exports>
-          <export>
+        <componentExports>
+          <componentExport>
             <name>log_viewer_port</name>
             <value>${THIS_HOST}:${site.storm-site.logviewer.port}</value>
-          </export>
-        </exports>
+          </componentExport>
+        </componentExports>
         <commandScript>
           <script>scripts/supervisor.py</script>
           <scriptType>PYTHON</scriptType>
@@ -117,6 +123,7 @@
         <name>STORM_UI_SERVER</name>
         <category>MASTER</category>
         <publishConfig>true</publishConfig>
+        <appExports>QuickLinks-app.monitor</appExports>
         <autoStartOnFailure>true</autoStartOnFailure>
         <commandScript>
           <script>scripts/ui_server.py</script>
@@ -148,10 +155,5 @@
         </packages>
       </osSpecific>
     </osSpecifics>
-
-    <configuration-dependencies>
-      <config-type>storm-site</config-type>
-      <config-type>global</config-type>
-    </configuration-dependencies>
   </application>
 </metainfo>

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc9561d5/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
index acf6ecb..03c64d4 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
@@ -29,12 +29,13 @@ public class Component {
   String minInstanceCount;
   String maxInstanceCount;
   String autoStartOnFailure;
+  String appExports;
   CommandScript commandScript;
-  List<Export> exports;
+  List<ComponentExport> componentExports;
 
   public Component() {
     publishConfig = Boolean.FALSE.toString();
-    exports = new ArrayList<Export>();
+    componentExports = new ArrayList<ComponentExport>();
   }
 
   public String getName() {
@@ -69,6 +70,14 @@ public class Component {
     this.autoStartOnFailure = autoStartOnFailure;
   }
 
+  public String getAppExports() {
+    return appExports;
+  }
+
+  public void setAppExports(String appExports) {
+    this.appExports = appExports;
+  }
+
   public String getMinInstanceCount() {
     return minInstanceCount;
   }
@@ -93,12 +102,12 @@ public class Component {
     this.commandScript = commandScript;
   }
 
-  public void addExport(Export export) {
-    exports.add(export);
+  public void addComponentExport(ComponentExport export) {
+    componentExports.add(export);
   }
 
-  public List<Export> getExports() {
-    return exports;
+  public List<ComponentExport> getComponentExports() {
+    return componentExports;
   }
 
   public Boolean getRequiresAutoRestart() {

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc9561d5/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java
new file mode 100644
index 0000000..a18854c
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+/**
+ *
+ */
+public class ComponentExport {
+  String name;
+  String value;
+
+  public ComponentExport() {
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public void setName(String name) {
+    this.name = name;
+  }
+
+  public String getValue() {
+    return value;
+  }
+
+  public void setValue(String value) {
+    this.value = value;
+  }
+
+  @Override
+  public String toString() {
+    final StringBuilder sb =
+        new StringBuilder("{");
+    sb.append(",\n\"name\": ").append(name);
+    sb.append(",\n\"value\": ").append(value);
+    sb.append('}');
+    return sb.toString();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc9561d5/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
index ada5b83..4b91ff7 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
@@ -60,10 +60,11 @@ public class MetainfoParser {
     digester.addBeanPropertySetter("*/component/minInstanceCount");
     digester.addBeanPropertySetter("*/component/maxInstanceCount");
     digester.addBeanPropertySetter("*/component/autoStartOnFailure");
-    digester.addObjectCreate("*/component/export", Export.class);
-    digester.addBeanPropertySetter("*/component/export/name");
-    digester.addBeanPropertySetter("*/component/export/value");
-    digester.addSetNext("*/component/export", "addExport");
+    digester.addBeanPropertySetter("*/component/appExports");
+    digester.addObjectCreate("*/component/componentExport", ComponentExport.class);
+    digester.addBeanPropertySetter("*/component/componentExport/name");
+    digester.addBeanPropertySetter("*/component/componentExport/value");
+    digester.addSetNext("*/component/componentExport", "addComponentExport");
     digester.addSetNext("*/component", "addComponent");
 
     digester.addObjectCreate("*/commandScript", CommandScript.class);

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc9561d5/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 20c16ff..1ea11a6 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -43,6 +43,7 @@ import org.apache.slider.core.launch.ContainerLauncher;
 import org.apache.slider.providers.agent.application.metadata.Application;
 import org.apache.slider.providers.agent.application.metadata.CommandOrder;
 import org.apache.slider.providers.agent.application.metadata.Component;
+import org.apache.slider.providers.agent.application.metadata.ComponentExport;
 import org.apache.slider.providers.agent.application.metadata.Export;
 import org.apache.slider.providers.agent.application.metadata.ExportGroup;
 import org.apache.slider.providers.agent.application.metadata.Metainfo;
@@ -138,6 +139,7 @@ public class TestAgentProviderService {
                                                + "          <name>HBASE_MASTER</name>\n"
                                                + "          <category>MASTER</category>\n"
                                                + "          <autoStartOnFailure>true</autoStartOnFailure>\n"
+                                               + "          <appExports>QuickLinks-JMX_Endpoint,QuickLinks-Master_Status</appExports>\n"
                                                + "          <minInstanceCount>1</minInstanceCount>\n"
                                                + "          <maxInstanceCount>2</maxInstanceCount>\n"
                                                + "          <commandScript>\n"
@@ -155,16 +157,16 @@ public class TestAgentProviderService {
                                                + "            <script>scripts/hbase_regionserver.py</script>\n"
                                                + "            <scriptType>PYTHON</scriptType>\n"
                                                + "          </commandScript>\n"
-                                               + "          <exports>\n"
-                                               + "            <export>\n"
+                                               + "          <componentExports>\n"
+                                               + "            <componentExport>\n"
                                                + "              <name>PropertyA</name>\n"
                                                + "              <value>${THIS_HOST}:${site.global.listen_port}</value>\n"
-                                               + "            </export>\n"
-                                               + "            <export>\n"
+                                               + "            </componentExport>\n"
+                                               + "            <componentExport>\n"
                                                + "              <name>PropertyB</name>\n"
                                                + "              <value>AConstant</value>\n"
-                                               + "            </export>\n"
-                                               + "          </exports>\n"
+                                               + "            </componentExport>\n"
+                                               + "          </componentExport>\n"
                                                + "        </component>\n"
                                                + "      </components>\n"
                                                + "      <osSpecifics>\n"
@@ -496,7 +498,8 @@ public class TestAgentProviderService {
         Assert.assertEquals(component.getMaxInstanceCount(), "2");
         Assert.assertEquals(component.getCommandScript().getScript(), "scripts/hbase_master.py");
         Assert.assertEquals(component.getCategory(), "MASTER");
-        Assert.assertEquals(component.getExports().size(), 0);
+        Assert.assertEquals(component.getComponentExports().size(), 0);
+        Assert.assertEquals(component.getAppExports(), "QuickLinks-JMX_Endpoint,QuickLinks-Master_Status");
         found++;
       }
       if (component.getName().equals("HBASE_REGIONSERVER")) {
@@ -506,9 +509,9 @@ public class TestAgentProviderService {
         Assert.assertNull(component.getMaxInstanceCount());
         Assert.assertEquals(component.getCommandScript().getScript(), "scripts/hbase_regionserver.py");
         Assert.assertEquals(component.getCategory(), "SLAVE");
-        Assert.assertEquals(component.getExports().size(), 2);
-        List<Export> es = component.getExports();
-        Export e = es.get(0);
+        Assert.assertEquals(component.getComponentExports().size(), 2);
+        List<ComponentExport> es = component.getComponentExports();
+        ComponentExport e = es.get(0);
         Assert.assertEquals(e.getName(), "PropertyA");
         Assert.assertEquals(e.getValue(), "${THIS_HOST}:${site.global.listen_port}");
         e = es.get(1);

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc9561d5/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml b/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml
index 3d24f96..d9004ad 100644
--- a/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml
+++ b/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml
@@ -22,11 +22,67 @@
     <name>STORM</name>
     <comment>Apache Hadoop Stream processing framework</comment>
     <version>0.9.1.2.1</version>
+    <exportedConfigs>storm-site</exportedConfigs>
+
+    <exportGroups>
+      <exportGroup>
+        <name>QuickLinks</name>
+        <exports>
+          <export>
+            <name>app.jmx</name>
+            <value>http://${STORM_REST_API_HOST}:${site.global.rest_api_port}/api/cluster/summary</value>
+          </export>
+          <export>
+            <name>app.monitor</name>
+            <value>http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}</value>
+          </export>
+          <export>
+            <name>app.metrics</name>
+            <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value>
+          </export>
+          <export>
+            <name>ganglia.ui</name>
+            <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value>
+          </export>
+          <export>
+            <name>nimbus.url</name>
+            <value>http://${NIMBUS_HOST}:${site.storm-site.nimbus.thrift.port}</value>
+          </export>
+        </exports>
+      </exportGroup>
+    </exportGroups>
+
+    <commandOrders>
+      <commandOrder>
+        <command>NIMBUS-START</command>
+        <requires>SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED,STORM_REST_API-INSTALLED
+        </requires>
+      </commandOrder>
+      <commandOrder>
+        <command>SUPERVISOR-START</command>
+        <requires>NIMBUS-STARTED</requires>
+      </commandOrder>
+      <commandOrder>
+        <command>DRPC_SERVER-START</command>
+        <requires>NIMBUS-STARTED</requires>
+      </commandOrder>
+      <commandOrder>
+        <command>STORM_REST_API-START</command>
+        <requires>NIMBUS-STARTED,DRPC_SERVER-STARTED,STORM_UI_SERVER-STARTED</requires>
+      </commandOrder>
+      <commandOrder>
+        <command>STORM_UI_SERVER-START</command>
+        <requires>NIMBUS-STARTED</requires>
+      </commandOrder>
+    </commandOrders>
+
     <components>
 
       <component>
         <name>NIMBUS</name>
         <category>MASTER</category>
+        <autoStartOnFailure>true</autoStartOnFailure>
+        <appExports>QuickLinks-nimbus.url,QuickLinks-ganglia.ui,QuickLinks-app.metrics</appExports>
         <commandScript>
           <script>scripts/nimbus.py</script>
           <scriptType>PYTHON</scriptType>
@@ -37,6 +93,8 @@
       <component>
         <name>STORM_REST_API</name>
         <category>MASTER</category>
+        <autoStartOnFailure>true</autoStartOnFailure>
+        <appExports>QuickLinks-app.jmx</appExports>
         <commandScript>
           <script>scripts/rest_api.py</script>
           <scriptType>PYTHON</scriptType>
@@ -47,6 +105,13 @@
       <component>
         <name>SUPERVISOR</name>
         <category>SLAVE</category>
+        <autoStartOnFailure>true</autoStartOnFailure>
+        <componentExports>
+          <componentExport>
+            <name>log_viewer_port</name>
+            <value>${THIS_HOST}:${site.storm-site.logviewer.port}</value>
+          </componentExport>
+        </componentExports>
         <commandScript>
           <script>scripts/supervisor.py</script>
           <scriptType>PYTHON</scriptType>
@@ -57,6 +122,9 @@
       <component>
         <name>STORM_UI_SERVER</name>
         <category>MASTER</category>
+        <publishConfig>true</publishConfig>
+        <appExports>QuickLinks-app.monitor</appExports>
+        <autoStartOnFailure>true</autoStartOnFailure>
         <commandScript>
           <script>scripts/ui_server.py</script>
           <scriptType>PYTHON</scriptType>
@@ -67,6 +135,7 @@
       <component>
         <name>DRPC_SERVER</name>
         <category>MASTER</category>
+        <autoStartOnFailure>true</autoStartOnFailure>
         <commandScript>
           <script>scripts/drpc_server.py</script>
           <scriptType>PYTHON</scriptType>
@@ -86,10 +155,5 @@
         </packages>
       </osSpecific>
     </osSpecifics>
-
-    <configuration-dependencies>
-      <config-type>storm-site</config-type>
-      <config-type>global</config-type>
-    </configuration-dependencies>
   </application>
 </metainfo>


[30/50] [abbrv] git commit: Merge branch 'feature/SLIDER-325_codahale_metrics_not_being_uploaded_to_AM_classpath' into develop

Posted by st...@apache.org.
Merge branch 'feature/SLIDER-325_codahale_metrics_not_being_uploaded_to_AM_classpath' into develop


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4ea66fad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4ea66fad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4ea66fad

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 4ea66faddb24564d7efc65367d6834db18b4d1fa
Parents: b8c2762 388805a
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 15 13:47:02 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 15 13:47:02 2014 +0100

----------------------------------------------------------------------
 .../slideram/SliderAMClientProvider.java        | 28 ++++++++++----------
 1 file changed, 14 insertions(+), 14 deletions(-)
----------------------------------------------------------------------



[45/50] [abbrv] git commit: SLIDER-302: add -live and --history flags to the list action

Posted by st...@apache.org.
SLIDER-302: add -live and --history flags to the list action


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5d59454c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5d59454c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5d59454c

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 5d59454cf726ea8d626c9906220a43379c442a42
Parents: 624cafe
Author: Steve Loughran <st...@apache.org>
Authored: Mon Aug 11 11:06:52 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 19 10:53:45 2014 +0100

----------------------------------------------------------------------
 .../org/apache/slider/client/SliderClient.java  | 50 +++++++++++++++-----
 .../slider/common/params/ActionListArgs.java    |  9 ++++
 .../apache/slider/common/params/Arguments.java  |  1 +
 .../apache/slider/common/tools/SliderUtils.java | 49 +++++++++++++++++++
 4 files changed, 98 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d59454c/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 93f6207..fb06454 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -51,6 +51,7 @@ import org.apache.slider.common.params.ActionFlexArgs;
 import org.apache.slider.common.params.ActionFreezeArgs;
 import org.apache.slider.common.params.ActionGetConfArgs;
 import org.apache.slider.common.params.ActionKillContainerArgs;
+import org.apache.slider.common.params.ActionListArgs;
 import org.apache.slider.common.params.ActionRegistryArgs;
 import org.apache.slider.common.params.ActionStatusArgs;
 import org.apache.slider.common.params.ActionThawArgs;
@@ -259,7 +260,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
             serviceArgs.getActionAMSuicideArgs());
         break;
       case ACTION_LIST:
-        exitCode = actionList(clusterName);
+        exitCode = actionList(clusterName, serviceArgs.getActionListArgs());
         break;
       case ACTION_REGISTRY:
         exitCode = actionRegistry(
@@ -1400,30 +1401,48 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
 
   /**
    * Implement the list action: list all nodes
+   *
+   * live: List out only live instances
+   * history: List out only history instances
+   *
+   * If arguments are not given then list out both finished and
+   * running instances
+   *
+   * @param clustername List out specific cluster
+   * @param args Action list arguments
    * @return exit code of 0 if a list was created
    */
   @VisibleForTesting
-  public int actionList(String clustername) throws IOException, YarnException {
+  public int actionList(String clustername, ActionListArgs args)
+      throws IOException, YarnException {
     verifyBindingsDefined();
 
     String user = UserGroupInformation.getCurrentUser().getUserName();
     List<ApplicationReport> instances = listSliderInstances(user);
-
+    SliderUtils.sortApplicationReport(instances);
     if (isUnset(clustername)) {
       log.info("Instances for {}: {}",
-               (user != null ? user : "all users"),
-               instances.size());
+          (user != null ? user : "all users"),
+          instances.size());
       for (ApplicationReport report : instances) {
-        logAppReport(report);
+        logAppReport(report, args.live, args.history);
       }
       return EXIT_SUCCESS;
     } else {
       SliderUtils.validateClusterName(clustername);
       log.debug("Listing cluster named {}", clustername);
-      ApplicationReport report =
-        findClusterInInstanceList(instances, clustername);
+      if (args.history) {
+        for (ApplicationReport report : instances) {
+          if (report.getName().equals(clustername)) {
+            logAppReport(report, args.live, args.history);
+          }
+        }
+        return EXIT_SUCCESS;
+      }
+      ApplicationReport report = findClusterInInstanceList(instances,
+          clustername);
       if (report != null) {
-        logAppReport(report);
+        logAppReport(report, true, false);
         return EXIT_SUCCESS;
       } else {
         throw unknownClusterException(clustername);
@@ -1435,8 +1454,17 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
    * Log the application report at INFO
    * @param report report to log
    */
-  public void logAppReport(ApplicationReport report) {
-    log.info(SliderUtils.appReportToString(report, "\n"));
+  public void logAppReport(ApplicationReport report, boolean live, boolean history) {
+    boolean active = report.getYarnApplicationState() == YarnApplicationState.RUNNING
+                || report.getYarnApplicationState() ==
+                   YarnApplicationState.ACCEPTED;
+    if (active && live && !history) {
+      log.info(SliderUtils.appReportToString(report, "\n"));
+    } else if (!active && history && !live) {
+      log.info(SliderUtils.appReportToString(report, "\n"));
+    } else {
+      log.info(SliderUtils.appReportToString(report, "\n"));
+    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d59454c/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
index 8f42b69..586f0a1 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
@@ -18,6 +18,7 @@
 
 package org.apache.slider.common.params;
 
+import com.beust.jcommander.Parameter;
 import com.beust.jcommander.Parameters;
 
 @Parameters(commandNames = {SliderActions.ACTION_LIST},
@@ -29,6 +30,14 @@ public class ActionListArgs extends AbstractActionArgs {
     return SliderActions.ACTION_LIST;
   }
 
+  @Parameter(names = {ARG_LIVE},
+          description = "List only live application instances")
+  public boolean live;
+
+  @Parameter(names = {ARG_HISTORY},
+          description = "List only historical application instances")
+  public boolean history;
+
   /**
    * Get the min #of params expected
    * @return the min number of params in the {@link #parameters} field

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d59454c/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index 764ffca..a16bf98 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -46,6 +46,7 @@ public interface Arguments {
   String ARG_GETCONF = "--getconf";
   String ARG_GETFILES = "--getfiles";
   String ARG_HELP = "--help";
+  String ARG_HISTORY = "--history";
   String ARG_ID = "--id";
   String ARG_IMAGE = "--image";
   String ARG_INTERNAL = "--internal";

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d59454c/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 188b7d9..6aff38a 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -75,6 +75,8 @@ import java.net.URLDecoder;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.Date;
 import java.util.Enumeration;
 import java.util.HashMap;
@@ -566,6 +568,53 @@ public final class SliderUtils {
   }
 
   /**
+   * Sorts the given list of application reports
+   * Finished instances are ordered by finished time and running/accepted instances are
+   * ordered by start time
+   * Finally Instance are order by finished instances and running instances
+   *
+   * @param instances
+   */
+  public static void sortApplicationReport(List<ApplicationReport> instances) {
+    if (instances.size() <= 1) {
+      return;
+    }
+    List<ApplicationReport> nonLiveInstance = new ArrayList<>(instances.size());
+    List<ApplicationReport> liveInstance = new ArrayList<>(instances.size());
+
+    for (ApplicationReport report : instances) {
+      if (report.getYarnApplicationState() == YarnApplicationState.RUNNING
+          || report.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
+        liveInstance.add(report);
+      } else {
+        nonLiveInstance.add(report);
+      }
+    }
+
+    if (liveInstance.size() > 1) {
+      Comparator<ApplicationReport> liveInstanceComparator = new Comparator<ApplicationReport>() {
+        @Override
+        public int compare(ApplicationReport r1, ApplicationReport r2) {
+          return Long.compare(r1.getStartTime(), r2.getStartTime());
+        }
+      };
+      Collections.sort(liveInstance, liveInstanceComparator);
+    }
+    if (nonLiveInstance.size() > 1) {
+      Comparator<ApplicationReport> nonLiveInstanceComparator = new Comparator<ApplicationReport>() {
+        @Override
+        public int compare(ApplicationReport r1, ApplicationReport r2) {
+          return Long.compare(r1.getFinishTime(), r2.getFinishTime());
+        }
+      };
+      Collections.sort(nonLiveInstance, nonLiveInstanceComparator);
+    }
+    instances.clear();
+    instances.addAll(nonLiveInstance);
+    instances.addAll(liveInstance);
+  }
+
+  /**
    * Merge in one map to another -all entries in the second map are
    * merged into the first -overwriting any duplicate keys.
    * @param first first map -the updated one.


[21/50] [abbrv] git commit: SLIDER-181. Slider support for Storm application package SLIDER-262. Slider agent should provide process supervision such as auto-restart SLIDER-123. Status check for Storm on YARN needs to be robust SLIDER-301. Add support fo

Posted by st...@apache.org.
SLIDER-181. Slider support for Storm application package
SLIDER-262. Slider agent should provide process supervision such as auto-restart
SLIDER-123. Status check for Storm on YARN needs to be robust
SLIDER-301. Add support for default status check through pid file


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/004fc8df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/004fc8df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/004fc8df

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 004fc8dfdfde58c04297d4001e861a0f8e93d69e
Parents: 966c8c8
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Aug 12 14:37:12 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Aug 14 17:54:57 2014 -0700

----------------------------------------------------------------------
 app-packages/memcached-win/appConfig.json       |  2 +-
 .../memcached-win/package/scripts/memcached.py  |  5 +++-
 .../memcached-win/package/scripts/params.py     |  1 +
 app-packages/memcached/appConfig.json           |  1 +
 .../memcached/package/scripts/memcached.py      |  5 +++-
 .../memcached/package/scripts/params.py         |  2 ++
 app-packages/storm/appConfig.json               |  6 ++--
 app-packages/storm/metainfo.xml                 | 12 ++++++++
 app-packages/storm/package/scripts/service.py   | 24 ++++++++-------
 .../storm/package/scripts/status_params.py      |  1 +
 .../src/main/python/agent/Controller.py         |  3 +-
 .../python/agent/CustomServiceOrchestrator.py   | 31 +++++++++++++++++++-
 .../core/providers/system.py                    |  3 +-
 .../core/providers/windows/system.py            | 11 +++++--
 .../core/resources/system.py                    |  4 +++
 .../python/resource_management/core/shell.py    | 14 +++++----
 .../agent/TestCustomServiceOrchestrator.py      |  8 +++++
 .../java/org/apache/slider/api/OptionKeys.java  |  1 -
 .../slider/common/tools/CoreFileSystem.java     |  3 --
 .../providers/agent/AgentClientProvider.java    |  6 ----
 .../providers/agent/AgentProviderService.java   | 24 +++++++++------
 .../agent/application/metadata/Component.java   | 13 ++++++++
 .../application/metadata/MetainfoParser.java    |  1 +
 .../agent/TestAgentProviderService.java         |  6 ++++
 24 files changed, 140 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/app-packages/memcached-win/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/appConfig.json b/app-packages/memcached-win/appConfig.json
index 63835ec..b76ecde 100644
--- a/app-packages/memcached-win/appConfig.json
+++ b/app-packages/memcached-win/appConfig.json
@@ -8,7 +8,7 @@
 
     "site.global.app_user": "hadoop",
     "site.global.app_root": "${AGENT_WORK_ROOT}\\app\\install",
-
+    "site.global.pid_file": "${AGENT_WORK_ROOT}\\app\\run\\component.pid",
     "site.global.additional_cp": "C:\\hdp\\hadoop-2.4.0.2.1.3.0-1990\\share\\hadoop\\common\\lib\\*",
     "site.global.xmx_val": "256m",
     "site.global.xms_val": "128m",

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/app-packages/memcached-win/package/scripts/memcached.py
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/package/scripts/memcached.py b/app-packages/memcached-win/package/scripts/memcached.py
index 1bb05dc..bc9905d 100644
--- a/app-packages/memcached-win/package/scripts/memcached.py
+++ b/app-packages/memcached-win/package/scripts/memcached.py
@@ -39,7 +39,8 @@ class Memcached(Script):
     Execute(process_cmd,
         user=params.app_user,
         logoutput=False,
-        wait_for_finish=False
+        wait_for_finish=False,
+        pid_file=params.pid_file
     )
 
   def stop(self, env):
@@ -49,6 +50,8 @@ class Memcached(Script):
   def status(self, env):
     import params
     env.set_params(params)
+    #Check process status need to be changed for Windows
+    #check_process_status(params.pid_file)
 
 if __name__ == "__main__":
   Memcached().execute()

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/app-packages/memcached-win/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/package/scripts/params.py b/app-packages/memcached-win/package/scripts/params.py
index f8e64da..fab3714 100644
--- a/app-packages/memcached-win/package/scripts/params.py
+++ b/app-packages/memcached-win/package/scripts/params.py
@@ -26,6 +26,7 @@ config = Script.get_config()
 app_root = config['configurations']['global']['app_root']
 java64_home = config['hostLevelParams']['java_home']
 app_user = config['configurations']['global']['app_user']
+pid_file = config['configurations']['global']['pid_file']
 additional_cp = config['configurations']['global']['additional_cp']
 xmx_val = config['configurations']['global']['xmx_val']
 xms_val = config['configurations']['global']['xms_val']

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/app-packages/memcached/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached/appConfig.json b/app-packages/memcached/appConfig.json
index b179723..5f32030 100644
--- a/app-packages/memcached/appConfig.json
+++ b/app-packages/memcached/appConfig.json
@@ -8,6 +8,7 @@
 
     "site.global.app_user": "yarn",
     "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/jmemcached-1.0.0",
+    "site.global.pid_file": "${AGENT_WORK_ROOT}/app/run/component.pid",
 
     "site.global.additional_cp": "/usr/lib/hadoop/lib/*",
     "site.global.xmx_val": "256m",

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/app-packages/memcached/package/scripts/memcached.py
----------------------------------------------------------------------
diff --git a/app-packages/memcached/package/scripts/memcached.py b/app-packages/memcached/package/scripts/memcached.py
index cf04cf1..6e14e86 100644
--- a/app-packages/memcached/package/scripts/memcached.py
+++ b/app-packages/memcached/package/scripts/memcached.py
@@ -24,6 +24,7 @@ from resource_management import *
 class Memcached(Script):
   def install(self, env):
     self.install_packages(env)
+    pass
 
   def configure(self, env):
     import params
@@ -38,7 +39,8 @@ class Memcached(Script):
     Execute(process_cmd,
         user=params.app_user,
         logoutput=False,
-        wait_for_finish=False
+        wait_for_finish=False,
+        pid_file=params.pid_file
     )
 
   def stop(self, env):
@@ -48,6 +50,7 @@ class Memcached(Script):
   def status(self, env):
     import params
     env.set_params(params)
+    check_process_status(params.pid_file)
 
 if __name__ == "__main__":
   Memcached().execute()

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/app-packages/memcached/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/memcached/package/scripts/params.py b/app-packages/memcached/package/scripts/params.py
index f8e64da..25b4055 100644
--- a/app-packages/memcached/package/scripts/params.py
+++ b/app-packages/memcached/package/scripts/params.py
@@ -26,6 +26,8 @@ config = Script.get_config()
 app_root = config['configurations']['global']['app_root']
 java64_home = config['hostLevelParams']['java_home']
 app_user = config['configurations']['global']['app_user']
+pid_file = config['configurations']['global']['pid_file']
+
 additional_cp = config['configurations']['global']['additional_cp']
 xmx_val = config['configurations']['global']['xmx_val']
 xms_val = config['configurations']['global']['xms_val']

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/app-packages/storm/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json
index 6d6aa3a..24078cf 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -3,7 +3,7 @@
   "metadata": {
   },
   "global": {
-    "application.def": "/slider/storm_v091.zip",
+    "application.def": "package/storm_v091.zip",
     "config_types": "storm-site",
     "java_home": "/usr/jdk64/jdk1.7.0_45",
     "package_list": "files/apache-storm-0.9.1.2.1.1.0-237.tar.gz",
@@ -68,7 +68,7 @@
     "site.storm-site.logviewer.appender.name": "A1",
     "site.storm-site.nimbus.host": "${NIMBUS_HOST}",
     "site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
-    "site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}]",
+    "site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
     "site.storm-site.nimbus.file.copy.expiration.secs": "600",
     "site.storm-site.supervisor.monitor.frequency.secs": "3",
     "site.storm-site.transactional.zookeeper.servers": "null",
@@ -105,7 +105,7 @@
     "site.storm-site.topology.trident.batch.emit.interval.millis": "500",
     "site.storm-site.topology.builtin.metrics.bucket.size.secs": "60",
     "site.storm-site.storm.thrift.transport": "backtype.storm.security.auth.SimpleTransportPlugin",
-    "site.storm-site.logviewer.port": "0",
+    "site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}",
     "site.storm-site.topology.debug": "false"
   },
   "components": {

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/app-packages/storm/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index 7edd794..e94bc62 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -22,6 +22,7 @@
     <name>STORM</name>
     <comment>Apache Hadoop Stream processing framework</comment>
     <version>0.9.1.2.1</version>
+    <exportedConfigs>storm-site</exportedConfigs>
 
     <exportGroups>
       <exportGroup>
@@ -76,6 +77,7 @@
       <component>
         <name>NIMBUS</name>
         <category>MASTER</category>
+        <autoStartOnFailure>true</autoStartOnFailure>
         <commandScript>
           <script>scripts/nimbus.py</script>
           <scriptType>PYTHON</scriptType>
@@ -86,6 +88,7 @@
       <component>
         <name>STORM_REST_API</name>
         <category>MASTER</category>
+        <autoStartOnFailure>true</autoStartOnFailure>
         <commandScript>
           <script>scripts/rest_api.py</script>
           <scriptType>PYTHON</scriptType>
@@ -96,6 +99,13 @@
       <component>
         <name>SUPERVISOR</name>
         <category>SLAVE</category>
+        <autoStartOnFailure>true</autoStartOnFailure>
+        <exports>
+          <export>
+            <name>log_viewer_port</name>
+            <value>${THIS_HOST}:${site.storm-site.logviewer.port}</value>
+          </export>
+        </exports>
         <commandScript>
           <script>scripts/supervisor.py</script>
           <scriptType>PYTHON</scriptType>
@@ -107,6 +117,7 @@
         <name>STORM_UI_SERVER</name>
         <category>MASTER</category>
         <publishConfig>true</publishConfig>
+        <autoStartOnFailure>true</autoStartOnFailure>
         <commandScript>
           <script>scripts/ui_server.py</script>
           <scriptType>PYTHON</scriptType>
@@ -117,6 +128,7 @@
       <component>
         <name>DRPC_SERVER</name>
         <category>MASTER</category>
+        <autoStartOnFailure>true</autoStartOnFailure>
         <commandScript>
           <script>scripts/drpc_server.py</script>
           <scriptType>PYTHON</scriptType>

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/app-packages/storm/package/scripts/service.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/scripts/service.py b/app-packages/storm/package/scripts/service.py
index 10fa5b9..13fcef2 100644
--- a/app-packages/storm/package/scripts/service.py
+++ b/app-packages/storm/package/scripts/service.py
@@ -22,7 +22,9 @@ limitations under the License.
 from resource_management import *
 import time
 
-
+"""
+Slider package uses jps as pgrep does not list the whole process start command
+"""
 def service(
     name,
     action='start'):
@@ -30,25 +32,25 @@ def service(
   import status_params
 
   pid_file = status_params.pid_files[name]
+  container_id = status_params.container_id
   no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
+
   jps_path = format("{java64_home}/bin/jps")
-  grep_and_awk = "| grep -v grep | awk '{print $1}'"
+  grep_and_awk = format("| grep {container_id}") + " | awk '{print $1}'"
 
   if name == 'ui':
-    #process_cmd = "^java.+backtype.storm.ui.core$"
-    pid_chk_cmd = format("{jps_path} -vl | grep \"^[0-9 ]*backtype.storm.ui.core\" {grep_and_awk}  > {pid_file}")
+    crt_pid_cmd = format("{jps_path} -vl | grep \"^[0-9 ]*backtype.storm.ui.core\" {grep_and_awk}  > {pid_file}")
   elif name == "rest_api":
-    process_cmd = format("{java64_home}/bin/java -jar {rest_lib_dir}/`ls {rest_lib_dir} | grep -wE storm-rest-[0-9.-]+\.jar` server")
-    crt_pid_cmd = format("pgrep -f \"{process_cmd}\" && pgrep -f \"{process_cmd}\" > {pid_file}")
+    rest_process_cmd = format("{java64_home}/bin/java -jar {rest_lib_dir}/`ls {rest_lib_dir} | grep -wE storm-rest-[0-9.-]+\.jar` server")
+    crt_pid_cmd = format("pgrep -f \"{rest_process_cmd}\" > {pid_file}")
   else:
-    #process_cmd = format("^java.+backtype.storm.daemon.{name}$")
-    pid_chk_cmd = format("{jps_path} -vl | grep \"^[0-9 ]*backtype.storm.daemon.{name}\" {grep_and_awk}  > {pid_file}")
+    crt_pid_cmd = format("{jps_path} -vl | grep \"^[0-9 ]*backtype.storm.daemon.{name}\" {grep_and_awk}  > {pid_file}")
 
   if action == "start":
     if name == "rest_api":
-      cmd = format("{process_cmd} {rest_api_conf_file} > {log_dir}/restapi.log")
+      cmd = format("{rest_process_cmd} {rest_api_conf_file} > {log_dir}/restapi.log")
     else:
-      cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {storm_bin} {name}")
+      cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {storm_bin} {name} > {log_dir}/{name}.out 2>&1")
 
     Execute(cmd,
             not_if=no_op_test,
@@ -67,7 +69,7 @@ def service(
     else:
       content = None
       for i in xrange(12):
-        Execute(pid_chk_cmd,
+        Execute(crt_pid_cmd,
                 user=params.storm_user,
                 logoutput=True
         )

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/app-packages/storm/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/scripts/status_params.py b/app-packages/storm/package/scripts/status_params.py
index eab83cf..5907446 100644
--- a/app-packages/storm/package/scripts/status_params.py
+++ b/app-packages/storm/package/scripts/status_params.py
@@ -21,6 +21,7 @@ from resource_management import *
 
 config = Script.get_config()
 
+container_id = config['configurations']['global']['app_container_id']
 pid_dir = config['configurations']['global']['app_pid_dir']
 pid_nimbus = format("{pid_dir}/nimbus.pid")
 pid_supervisor = format("{pid_dir}/supervisor.pid")

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-agent/src/main/python/agent/Controller.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Controller.py b/slider-agent/src/main/python/agent/Controller.py
index 4a103cb..1a01bf8 100644
--- a/slider-agent/src/main/python/agent/Controller.py
+++ b/slider-agent/src/main/python/agent/Controller.py
@@ -237,7 +237,8 @@ class Controller(threading.Thread):
             return
 
         if serverId != self.responseId + 1:
-          logger.error("Error in responseId sequence - restarting")
+          logger.error("Error in responseId sequence expected " + str(self.responseId + 1)
+                       + " but got " + str(serverId) + " - restarting")
           self.restartAgent()
         else:
           self.responseId = serverId

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 0f4acc6..0078c9c 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -259,7 +259,7 @@ class CustomServiceOrchestrator():
               elif allocated_for_any in value:
                 ## All unallocated ports should be set to 0
                 logger.info("Assigning port 0 " + "for " + value)
-                value = "0"
+                value = self.set_all_unallocated_ports(value)
               command['configurations'][key][k] = value
               pass
             pass
@@ -274,6 +274,35 @@ class CustomServiceOrchestrator():
   pass
 
   """
+  All unallocated ports should be set to 0
+  Look for "${SOME_COMPONENT_NAME.ALLOCATED_PORT}"
+        or "${component.ALLOCATED_PORT}{DEFAULT_port}"
+        or "${component.ALLOCATED_PORT}{DEFAULT_port}{DO_NOT_PROPAGATE}"
+  """
+
+  def set_all_unallocated_ports(self, value):
+    pattern_start = "${"
+    sub_section_start = "}{"
+    pattern_end = "}"
+    index = value.find(pattern_start)
+    while index != -1:
+      replace_index_start = index
+      replace_index_end = value.find(pattern_end, replace_index_start)
+      next_pattern_start = value.find(sub_section_start, replace_index_start)
+      while next_pattern_start == replace_index_end:
+        replace_index_end = value.find(pattern_end, replace_index_end + 1)
+        next_pattern_start = value.find(sub_section_start, next_pattern_start + 1)
+        pass
+
+      value = value[:replace_index_start] + "0" + value[replace_index_end + 1:]
+
+      # look for the next
+      index = value.find(pattern_start)
+
+    return value
+    pass
+
+  """
   Port allocation can asks for multiple dynamic ports
   port_req_pattern is of type ${component_name.ALLOCATED_PORT}
     append {DEFAULT_ and find the default value

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-agent/src/main/python/resource_management/core/providers/system.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/providers/system.py b/slider-agent/src/main/python/resource_management/core/providers/system.py
index e273498..6969c62 100644
--- a/slider-agent/src/main/python/resource_management/core/providers/system.py
+++ b/slider-agent/src/main/python/resource_management/core/providers/system.py
@@ -242,7 +242,8 @@ class ExecuteProvider(Provider):
         shell.checked_call(self.resource.command, logoutput=self.resource.logoutput,
                             cwd=self.resource.cwd, env=self.resource.environment,
                             preexec_fn=_preexec_fn(self.resource), user=self.resource.user,
-                            wait_for_finish=self.resource.wait_for_finish, timeout=self.resource.timeout)
+                            wait_for_finish=self.resource.wait_for_finish, timeout=self.resource.timeout,
+                            pid_file=self.resource.pid_file)
         break
       except Fail as ex:
         if i == self.resource.tries-1: # last try

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
index fa29d7d..f0d4825 100644
--- a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
+++ b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
@@ -30,12 +30,16 @@ import subprocess
 import shutil
 
 
-def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish=True, timeout=None):
-  # TODO implement timeout, logoutput, wait_for_finish
+def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish=True, timeout=None, pid_file_name=None):
+  # TODO implement logoutput
   Logger.info("Executing %s" % (command))
   proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                           cwd=cwd, env=env, shell=False)
   if not wait_for_finish:
+    if pid_file_name:
+      pidfile = open(pid_file_name, 'w')
+      pidfile.write(str(proc.pid))
+      pidfile.close()
     return None, None
 
   if timeout:
@@ -135,7 +139,8 @@ class ExecuteProvider(Provider):
       try:
         _call_command(self.resource.command, logoutput=self.resource.logoutput,
                       cwd=self.resource.cwd, env=self.resource.environment,
-                      wait_for_finish=self.resource.wait_for_finish, timeout=self.resource.timeout)
+                      wait_for_finish=self.resource.wait_for_finish, timeout=self.resource.timeout,
+                      pid_file_name=self.resource.pid_file)
         break
       except Fail as ex:
         if i == self.resource.tries - 1:  # last try

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-agent/src/main/python/resource_management/core/resources/system.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/resources/system.py b/slider-agent/src/main/python/resource_management/core/resources/system.py
index 2c832a4..a63d993 100644
--- a/slider-agent/src/main/python/resource_management/core/resources/system.py
+++ b/slider-agent/src/main/python/resource_management/core/resources/system.py
@@ -101,6 +101,10 @@ class Execute(Resource):
   - try_sleep
   """
   wait_for_finish = BooleanArgument(default=True)
+  """
+  if wait_for_finish is True then optionally the caller can ask for the pid to be written
+  """
+  pid_file = ResourceArgument()
 
 
 class ExecuteScript(Resource):

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-agent/src/main/python/resource_management/core/shell.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/shell.py b/slider-agent/src/main/python/resource_management/core/shell.py
index 92312d5..fb2c946 100644
--- a/slider-agent/src/main/python/resource_management/core/shell.py
+++ b/slider-agent/src/main/python/resource_management/core/shell.py
@@ -31,15 +31,15 @@ from exceptions import ExecuteTimeoutException
 from resource_management.core.logger import Logger
 
 def checked_call(command, logoutput=False, 
-         cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None):
-  return _call(command, logoutput, True, cwd, env, preexec_fn, user, wait_for_finish, timeout)
+         cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, pid_file=None):
+  return _call(command, logoutput, True, cwd, env, preexec_fn, user, wait_for_finish, timeout, pid_file)
 
 def call(command, logoutput=False, 
-         cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None):
-  return _call(command, logoutput, False, cwd, env, preexec_fn, user, wait_for_finish, timeout)
+         cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, pid_file=None):
+  return _call(command, logoutput, False, cwd, env, preexec_fn, user, wait_for_finish, timeout, pid_file)
             
 def _call(command, logoutput=False, throw_on_failure=True, 
-         cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None):
+         cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, pid_file_name=None):
   """
   Execute shell command
   
@@ -67,6 +67,10 @@ def _call(command, logoutput=False, throw_on_failure=True,
                           preexec_fn=preexec_fn)
 
   if not wait_for_finish:
+    if pid_file_name:
+      pidfile = open(pid_file_name, 'w')
+      pidfile.write(str(proc.pid))
+      pidfile.close()
     return None, None
   
   if timeout:

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index c07f05f..48e4df4 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -484,12 +484,20 @@ class TestCustomServiceOrchestrator(TestCase):
     command['configurations']['oozie-site'] = {}
     command['configurations']['oozie-site']['log_root'] = "${AGENT_LOG_ROOT}"
     command['configurations']['oozie-site']['a_port'] = "${HBASE_MASTER.ALLOCATED_PORT}"
+    command['configurations']['oozie-site']['ignore_port1'] = "[${HBASE_RS.ALLOCATED_PORT}]"
+    command['configurations']['oozie-site']['ignore_port2'] = "[${HBASE_RS.ALLOCATED_PORT},${HBASE_REST.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]"
+    command['configurations']['oozie-site']['ignore_port3'] = "[${HBASE_RS.ALLOCATED_PORT}{a}{b}{c},${A.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${A.ALLOCATED_PORT}{DEFAULT_3}{DO_NOT_PROPAGATE}]"
+    command['configurations']['oozie-site']['ignore_port4'] = "${HBASE_RS}{a}{b}{c}"
 
     allocated_ports = {}
     orchestrator.finalize_command(command, False, allocated_ports)
     self.assertEqual(command['configurations']['hbase-site']['work_root'], tempWorkDir)
     self.assertEqual(command['configurations']['oozie-site']['log_root'], tempdir)
     self.assertEqual(command['configurations']['oozie-site']['a_port'], "10023")
+    self.assertEqual(command['configurations']['oozie-site']['ignore_port1'], "[0]")
+    self.assertEqual(command['configurations']['oozie-site']['ignore_port2'], "[0,0]")
+    self.assertEqual(command['configurations']['oozie-site']['ignore_port3'], "[0,0,0]")
+    self.assertEqual(command['configurations']['oozie-site']['ignore_port4'], "${HBASE_RS}{a}{b}{c}")
     self.assertEqual(orchestrator.applied_configs, {})
     self.assertEqual(len(allocated_ports), 1)
     self.assertTrue('oozie-site.a_port' in allocated_ports)

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java b/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
index 320cc4f..a035a99 100644
--- a/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
@@ -35,7 +35,6 @@ public interface OptionKeys extends InternalKeys {
    */
   String APPLICATION_TYPE = "application.type";
   
-
   String APPLICATION_NAME = "application.name";
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index 2381a82..a145b8f 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -48,11 +48,8 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.nio.charset.Charset;
-import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipFile;
 
 import static org.apache.slider.common.SliderXmlConfKeys.CLUSTER_DIRECTORY_PERMISSIONS;
 import static org.apache.slider.common.SliderXmlConfKeys.DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS;

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 8f92ffa..3a1ee76 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -92,12 +92,6 @@ public class AgentClientProvider extends AbstractClientProvider
         getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
     Path appDefPath = new Path(appDef);
     sliderFileSystem.verifyFileExists(appDefPath);
-    
-    String pkgList = instanceDefinition.getAppConfOperations().
-        getGlobalOptions().getOption(AgentKeys.PACKAGE_LIST, null);
-    if (pkgList != null) {
-      sliderFileSystem.verifyFileExistsInZip(appDefPath, pkgList);
-    }
 
     String agentConf = instanceDefinition.getAppConfOperations().
         getGlobalOptions().getOption(AgentKeys.AGENT_CONF, "");

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 3225066..559c4e7 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -131,6 +131,7 @@ public class AgentProviderService extends AbstractProviderService implements
   private HeartbeatMonitor monitor;
   private Boolean canAnyMasterPublish = null;
   private AgentLaunchParameter agentLaunchParameter = null;
+  private String clusterName = null;
 
   private final Map<String, ComponentInstanceState> componentStatuses =
       new ConcurrentHashMap<String, ComponentInstanceState>();
@@ -629,6 +630,13 @@ public class AgentProviderService extends AbstractProviderService implements
     return this.heartbeatMonitorInterval;
   }
 
+  private String getClusterName() {
+    if (clusterName == null || clusterName.length() == 0) {
+      clusterName = getAmState().getInternalsSnapshot().get(OptionKeys.APPLICATION_NAME);
+    }
+    return clusterName;
+  }
+
   /**
    * Publish a named property bag that may contain name-value pairs for app configurations such as hbase-site
    * @param name
@@ -973,11 +981,10 @@ public class AgentProviderService extends AbstractProviderService implements
       throws SliderException {
     assert getAmState().isApplicationLive();
     ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
-    ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot();
 
     ExecutionCommand cmd = new ExecutionCommand(AgentCommandType.EXECUTION_COMMAND);
     prepareExecutionCommand(cmd);
-    String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME);
+    String clusterName = getClusterName();
     cmd.setClusterName(clusterName);
     cmd.setRoleCommand(Command.INSTALL.toString());
     cmd.setServiceName(clusterName);
@@ -1050,10 +1057,9 @@ public class AgentProviderService extends AbstractProviderService implements
       throws SliderException {
     assert getAmState().isApplicationLive();
     ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
-    ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot();
 
     StatusCommand cmd = new StatusCommand();
-    String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME);
+    String clusterName = getClusterName();
 
     cmd.setCommandType(AgentCommandType.STATUS_COMMAND);
     cmd.setComponentName(roleName);
@@ -1079,10 +1085,9 @@ public class AgentProviderService extends AbstractProviderService implements
   protected void addGetConfigCommand(String roleName, String containerId, HeartBeatResponse response)
       throws SliderException {
     assert getAmState().isApplicationLive();
-    ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot();
 
     StatusCommand cmd = new StatusCommand();
-    String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME);
+    String clusterName = getClusterName();
 
     cmd.setCommandType(AgentCommandType.STATUS_COMMAND);
     cmd.setComponentName(roleName);
@@ -1203,7 +1208,7 @@ public class AgentProviderService extends AbstractProviderService implements
                                      Map<String, String> tokens, String containerId) {
     Map<String, String> config = new HashMap<String, String>();
     if (configName.equals(GLOBAL_CONFIG_TAG)) {
-      addDefaultGlobalConfig(config);
+      addDefaultGlobalConfig(config, containerId);
     }
     // add role hosts to tokens
     addRoleRelatedTokens(tokens);
@@ -1248,10 +1253,11 @@ public class AgentProviderService extends AbstractProviderService implements
     return hosts;
   }
 
-  private void addDefaultGlobalConfig(Map<String, String> config) {
-    config.put("app_log_dir", "${AGENT_LOG_ROOT}/app/log");
+  private void addDefaultGlobalConfig(Map<String, String> config, String containerId) {
+    config.put("app_log_dir", "${AGENT_LOG_ROOT}");
     config.put("app_pid_dir", "${AGENT_WORK_ROOT}/app/run");
     config.put("app_install_dir", "${AGENT_WORK_ROOT}/app/install");
+    config.put("app_container_id", containerId);
   }
 
   private void buildRoleHostDetails(Map<String, String> details) {

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
index f7b77ff..acf6ecb 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
@@ -28,6 +28,7 @@ public class Component {
   String publishConfig;
   String minInstanceCount;
   String maxInstanceCount;
+  String autoStartOnFailure;
   CommandScript commandScript;
   List<Export> exports;
 
@@ -60,6 +61,14 @@ public class Component {
     this.publishConfig = publishConfig;
   }
 
+  public String getAutoStartOnFailure() {
+    return autoStartOnFailure;
+  }
+
+  public void setAutoStartOnFailure(String autoStartOnFailure) {
+    this.autoStartOnFailure = autoStartOnFailure;
+  }
+
   public String getMinInstanceCount() {
     return minInstanceCount;
   }
@@ -92,6 +101,10 @@ public class Component {
     return exports;
   }
 
+  public Boolean getRequiresAutoRestart() {
+    return Boolean.parseBoolean(this.autoStartOnFailure);
+  }
+
   @Override
   public String toString() {
     final StringBuilder sb =

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
index deb7a6a..23134b6 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
@@ -39,6 +39,7 @@ public class MetainfoParser {
     digester.addBeanPropertySetter("*/application/comment");
     digester.addBeanPropertySetter("*/application/version");
     digester.addBeanPropertySetter("*/application/exportedConfigs");
+    digester.addBeanPropertySetter("*/application/autoStartOnFailure");
 
     digester.addObjectCreate("*/commandOrder", CommandOrder.class);
     digester.addBeanPropertySetter("*/commandOrder/command");

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index ee61129..a68e770 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -134,6 +134,7 @@ public class TestAgentProviderService {
                                                + "        <component>\n"
                                                + "          <name>HBASE_MASTER</name>\n"
                                                + "          <category>MASTER</category>\n"
+                                               + "          <autoStartOnFailure>true</autoStartOnFailure>\n"
                                                + "          <minInstanceCount>1</minInstanceCount>\n"
                                                + "          <maxInstanceCount>2</maxInstanceCount>\n"
                                                + "          <commandScript>\n"
@@ -146,6 +147,7 @@ public class TestAgentProviderService {
                                                + "          <name>HBASE_REGIONSERVER</name>\n"
                                                + "          <category>SLAVE</category>\n"
                                                + "          <minInstanceCount>1</minInstanceCount>\n"
+                                               + "          <autoStartOnFailure>Falsee</autoStartOnFailure>\n"
                                                + "          <commandScript>\n"
                                                + "            <script>scripts/hbase_regionserver.py</script>\n"
                                                + "            <scriptType>PYTHON</scriptType>\n"
@@ -469,6 +471,8 @@ public class TestAgentProviderService {
     int found = 0;
     for (Component component : components) {
       if (component.getName().equals("HBASE_MASTER")) {
+        Assert.assertEquals(component.getAutoStartOnFailure(), "true");
+        Assert.assertEquals(component.getRequiresAutoRestart(), Boolean.TRUE);
         Assert.assertEquals(component.getMinInstanceCount(), "1");
         Assert.assertEquals(component.getMaxInstanceCount(), "2");
         Assert.assertEquals(component.getCommandScript().getScript(), "scripts/hbase_master.py");
@@ -477,6 +481,8 @@ public class TestAgentProviderService {
         found++;
       }
       if (component.getName().equals("HBASE_REGIONSERVER")) {
+        Assert.assertEquals(component.getAutoStartOnFailure(), "Falsee");
+        Assert.assertEquals(component.getRequiresAutoRestart(), Boolean.FALSE);
         Assert.assertEquals(component.getMinInstanceCount(), "1");
         Assert.assertNull(component.getMaxInstanceCount());
         Assert.assertEquals(component.getCommandScript().getScript(), "scripts/hbase_regionserver.py");


[42/50] [abbrv] git commit: SLIDER-332. Add LICENSE file and license headers to kazoo scripts (Gour Saha via smohanty)

Posted by st...@apache.org.
SLIDER-332. Add LICENSE file and license headers to kazoo scripts (Gour Saha via smohanty)


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a34cca8d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a34cca8d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a34cca8d

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: a34cca8dcdf8cf4918a5a54f5c472befc69ed922
Parents: 72daf37
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Mon Aug 18 15:46:49 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Mon Aug 18 15:46:49 2014 -0700

----------------------------------------------------------------------
 slider-agent/src/main/python/kazoo/LICENSE      | 176 +++++++++++++++++++
 slider-agent/src/main/python/kazoo/__init__.py  |   1 +
 slider-agent/src/main/python/kazoo/client.py    |   1 +
 .../src/main/python/kazoo/exceptions.py         |   1 +
 .../src/main/python/kazoo/handlers/__init__.py  |   1 +
 .../src/main/python/kazoo/handlers/gevent.py    |   1 +
 .../src/main/python/kazoo/handlers/threading.py |   1 +
 .../src/main/python/kazoo/handlers/utils.py     |   1 +
 slider-agent/src/main/python/kazoo/hosts.py     |   1 +
 .../src/main/python/kazoo/interfaces.py         |   1 +
 .../src/main/python/kazoo/loggingsupport.py     |   1 +
 .../src/main/python/kazoo/protocol/__init__.py  |   1 +
 .../main/python/kazoo/protocol/connection.py    |   1 +
 .../src/main/python/kazoo/protocol/paths.py     |   1 +
 .../main/python/kazoo/protocol/serialization.py |   1 +
 .../src/main/python/kazoo/protocol/states.py    |   1 +
 .../src/main/python/kazoo/recipe/__init__.py    |   1 +
 .../src/main/python/kazoo/recipe/barrier.py     |   1 +
 .../src/main/python/kazoo/recipe/counter.py     |   1 +
 .../src/main/python/kazoo/recipe/election.py    |   1 +
 .../src/main/python/kazoo/recipe/lock.py        |   1 +
 .../src/main/python/kazoo/recipe/partitioner.py |   1 +
 .../src/main/python/kazoo/recipe/party.py       |   1 +
 .../src/main/python/kazoo/recipe/queue.py       |   1 +
 .../src/main/python/kazoo/recipe/watchers.py    |   1 +
 slider-agent/src/main/python/kazoo/retry.py     |   1 +
 slider-agent/src/main/python/kazoo/security.py  |   1 +
 .../src/main/python/kazoo/testing/__init__.py   |   1 +
 .../src/main/python/kazoo/testing/common.py     |   1 +
 .../src/main/python/kazoo/testing/harness.py    |   1 +
 .../src/main/python/kazoo/tests/test_barrier.py |   1 +
 .../src/main/python/kazoo/tests/test_build.py   |   1 +
 .../src/main/python/kazoo/tests/test_client.py  |   1 +
 .../main/python/kazoo/tests/test_connection.py  |   1 +
 .../src/main/python/kazoo/tests/test_counter.py |   1 +
 .../main/python/kazoo/tests/test_election.py    |   1 +
 .../main/python/kazoo/tests/test_exceptions.py  |   1 +
 .../python/kazoo/tests/test_gevent_handler.py   |   1 +
 .../src/main/python/kazoo/tests/test_lock.py    |   1 +
 .../main/python/kazoo/tests/test_partitioner.py |   1 +
 .../src/main/python/kazoo/tests/test_party.py   |   1 +
 .../src/main/python/kazoo/tests/test_paths.py   |   1 +
 .../src/main/python/kazoo/tests/test_queue.py   |   1 +
 .../src/main/python/kazoo/tests/test_retry.py   |   1 +
 .../main/python/kazoo/tests/test_security.py    |   1 +
 .../kazoo/tests/test_threading_handler.py       |   1 +
 .../main/python/kazoo/tests/test_watchers.py    |   1 +
 .../src/main/python/kazoo/tests/util.py         |   1 +
 48 files changed, 223 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/LICENSE
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/LICENSE b/slider-agent/src/main/python/kazoo/LICENSE
new file mode 100644
index 0000000..68c771a
--- /dev/null
+++ b/slider-agent/src/main/python/kazoo/LICENSE
@@ -0,0 +1,176 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/__init__.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/__init__.py b/slider-agent/src/main/python/kazoo/__init__.py
index 792d600..a7bacf3 100644
--- a/slider-agent/src/main/python/kazoo/__init__.py
+++ b/slider-agent/src/main/python/kazoo/__init__.py
@@ -1 +1,2 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 #

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/client.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/client.py b/slider-agent/src/main/python/kazoo/client.py
index 6e3a219..a315489 100644
--- a/slider-agent/src/main/python/kazoo/client.py
+++ b/slider-agent/src/main/python/kazoo/client.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 """Kazoo Zookeeper Client"""
 import inspect
 import logging

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/exceptions.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/exceptions.py b/slider-agent/src/main/python/kazoo/exceptions.py
index 8d4f0f3..9c9e71d 100644
--- a/slider-agent/src/main/python/kazoo/exceptions.py
+++ b/slider-agent/src/main/python/kazoo/exceptions.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 """Kazoo Exceptions"""
 from collections import defaultdict
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/handlers/__init__.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/handlers/__init__.py b/slider-agent/src/main/python/kazoo/handlers/__init__.py
index 792d600..a7bacf3 100644
--- a/slider-agent/src/main/python/kazoo/handlers/__init__.py
+++ b/slider-agent/src/main/python/kazoo/handlers/__init__.py
@@ -1 +1,2 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 #

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/handlers/gevent.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/handlers/gevent.py b/slider-agent/src/main/python/kazoo/handlers/gevent.py
index 6e40cae..5e07143 100644
--- a/slider-agent/src/main/python/kazoo/handlers/gevent.py
+++ b/slider-agent/src/main/python/kazoo/handlers/gevent.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 """A gevent based handler."""
 from __future__ import absolute_import
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/handlers/threading.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/handlers/threading.py b/slider-agent/src/main/python/kazoo/handlers/threading.py
index 8e6648d..b2e355a 100644
--- a/slider-agent/src/main/python/kazoo/handlers/threading.py
+++ b/slider-agent/src/main/python/kazoo/handlers/threading.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 """A threading based handler.
 
 The :class:`SequentialThreadingHandler` is intended for regular Python

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/handlers/utils.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/handlers/utils.py b/slider-agent/src/main/python/kazoo/handlers/utils.py
index ad204b5..60d6404 100644
--- a/slider-agent/src/main/python/kazoo/handlers/utils.py
+++ b/slider-agent/src/main/python/kazoo/handlers/utils.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 """Kazoo handler helpers"""
 
 HAS_FNCTL = True

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/hosts.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/hosts.py b/slider-agent/src/main/python/kazoo/hosts.py
index 15ce447..ca0dd35 100644
--- a/slider-agent/src/main/python/kazoo/hosts.py
+++ b/slider-agent/src/main/python/kazoo/hosts.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 import random
 
 try:

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/interfaces.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/interfaces.py b/slider-agent/src/main/python/kazoo/interfaces.py
index 351f1fd..7aff561 100644
--- a/slider-agent/src/main/python/kazoo/interfaces.py
+++ b/slider-agent/src/main/python/kazoo/interfaces.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 """Kazoo Interfaces
 
 .. versionchanged:: 1.4

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/loggingsupport.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/loggingsupport.py b/slider-agent/src/main/python/kazoo/loggingsupport.py
index 5ed2f8f..91ae632 100644
--- a/slider-agent/src/main/python/kazoo/loggingsupport.py
+++ b/slider-agent/src/main/python/kazoo/loggingsupport.py
@@ -1,2 +1,3 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 BLATHER = 5 # log level for low-level debugging
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/protocol/__init__.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/protocol/__init__.py b/slider-agent/src/main/python/kazoo/protocol/__init__.py
index 792d600..a7bacf3 100644
--- a/slider-agent/src/main/python/kazoo/protocol/__init__.py
+++ b/slider-agent/src/main/python/kazoo/protocol/__init__.py
@@ -1 +1,2 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 #

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/protocol/connection.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/protocol/connection.py b/slider-agent/src/main/python/kazoo/protocol/connection.py
index e6ad63d..3cbb87f 100644
--- a/slider-agent/src/main/python/kazoo/protocol/connection.py
+++ b/slider-agent/src/main/python/kazoo/protocol/connection.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 """Zookeeper Protocol Connection Handler"""
 import logging
 import os

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/protocol/paths.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/protocol/paths.py b/slider-agent/src/main/python/kazoo/protocol/paths.py
index 52b2d6b..e37c1a7 100644
--- a/slider-agent/src/main/python/kazoo/protocol/paths.py
+++ b/slider-agent/src/main/python/kazoo/protocol/paths.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 def normpath(path, trailing=False):
     """Normalize path, eliminating double slashes, etc."""
     comps = path.split('/')

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/protocol/serialization.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/protocol/serialization.py b/slider-agent/src/main/python/kazoo/protocol/serialization.py
index f44f49a..3b5df6c 100644
--- a/slider-agent/src/main/python/kazoo/protocol/serialization.py
+++ b/slider-agent/src/main/python/kazoo/protocol/serialization.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 """Zookeeper Serializers, Deserializers, and NamedTuple objects"""
 from collections import namedtuple
 import struct

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/protocol/states.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/protocol/states.py b/slider-agent/src/main/python/kazoo/protocol/states.py
index 395c013..d09c2c6 100644
--- a/slider-agent/src/main/python/kazoo/protocol/states.py
+++ b/slider-agent/src/main/python/kazoo/protocol/states.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 """Kazoo State and Event objects"""
 from collections import namedtuple
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/recipe/__init__.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/recipe/__init__.py b/slider-agent/src/main/python/kazoo/recipe/__init__.py
index 792d600..a7bacf3 100644
--- a/slider-agent/src/main/python/kazoo/recipe/__init__.py
+++ b/slider-agent/src/main/python/kazoo/recipe/__init__.py
@@ -1 +1,2 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 #

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/recipe/barrier.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/recipe/barrier.py b/slider-agent/src/main/python/kazoo/recipe/barrier.py
index 05addb4..729ac07 100644
--- a/slider-agent/src/main/python/kazoo/recipe/barrier.py
+++ b/slider-agent/src/main/python/kazoo/recipe/barrier.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 """Zookeeper Barriers
 
 :Maintainer: None

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/recipe/counter.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/recipe/counter.py b/slider-agent/src/main/python/kazoo/recipe/counter.py
index ed80f51..026d724 100644
--- a/slider-agent/src/main/python/kazoo/recipe/counter.py
+++ b/slider-agent/src/main/python/kazoo/recipe/counter.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 """Zookeeper Counter
 
 :Maintainer: None

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/recipe/election.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/recipe/election.py b/slider-agent/src/main/python/kazoo/recipe/election.py
index 3089fa6..5349e26 100644
--- a/slider-agent/src/main/python/kazoo/recipe/election.py
+++ b/slider-agent/src/main/python/kazoo/recipe/election.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 """ZooKeeper Leader Elections
 
 :Maintainer: None

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/recipe/lock.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/recipe/lock.py b/slider-agent/src/main/python/kazoo/recipe/lock.py
index 8c944b7..8f14847 100644
--- a/slider-agent/src/main/python/kazoo/recipe/lock.py
+++ b/slider-agent/src/main/python/kazoo/recipe/lock.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 """Zookeeper Locking Implementations
 
 :Maintainer: None

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/recipe/partitioner.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/recipe/partitioner.py b/slider-agent/src/main/python/kazoo/recipe/partitioner.py
index 8de4231..0d92661 100644
--- a/slider-agent/src/main/python/kazoo/recipe/partitioner.py
+++ b/slider-agent/src/main/python/kazoo/recipe/partitioner.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 """Zookeeper Partitioner Implementation
 
 :Maintainer: None

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/recipe/party.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/recipe/party.py b/slider-agent/src/main/python/kazoo/recipe/party.py
index 7186c10..4fd873e 100644
--- a/slider-agent/src/main/python/kazoo/recipe/party.py
+++ b/slider-agent/src/main/python/kazoo/recipe/party.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 """Party
 
 :Maintainer: Ben Bangert <be...@groovie.org>

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/recipe/queue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/recipe/queue.py b/slider-agent/src/main/python/kazoo/recipe/queue.py
index e2e30dd..81289b9 100644
--- a/slider-agent/src/main/python/kazoo/recipe/queue.py
+++ b/slider-agent/src/main/python/kazoo/recipe/queue.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 """Zookeeper based queue implementations.
 
 :Maintainer: None

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/recipe/watchers.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/recipe/watchers.py b/slider-agent/src/main/python/kazoo/recipe/watchers.py
index 0624153..f9667f8 100644
--- a/slider-agent/src/main/python/kazoo/recipe/watchers.py
+++ b/slider-agent/src/main/python/kazoo/recipe/watchers.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 """Higher level child and data watching API's.
 
 :Maintainer: Ben Bangert <be...@groovie.org>

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/retry.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/retry.py b/slider-agent/src/main/python/kazoo/retry.py
index 152e9a6..229b99d 100644
--- a/slider-agent/src/main/python/kazoo/retry.py
+++ b/slider-agent/src/main/python/kazoo/retry.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 import logging
 import random
 import time

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/security.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/security.py b/slider-agent/src/main/python/kazoo/security.py
index 1bd989d..4532489 100644
--- a/slider-agent/src/main/python/kazoo/security.py
+++ b/slider-agent/src/main/python/kazoo/security.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 """Kazoo Security"""
 from base64 import b64encode
 from collections import namedtuple

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/testing/__init__.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/testing/__init__.py b/slider-agent/src/main/python/kazoo/testing/__init__.py
index bf8f149..660546b 100644
--- a/slider-agent/src/main/python/kazoo/testing/__init__.py
+++ b/slider-agent/src/main/python/kazoo/testing/__init__.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 from kazoo.testing.harness import KazooTestCase
 from kazoo.testing.harness import KazooTestHarness
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/testing/common.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/testing/common.py b/slider-agent/src/main/python/kazoo/testing/common.py
index 8e1e9b4..b497a8e 100644
--- a/slider-agent/src/main/python/kazoo/testing/common.py
+++ b/slider-agent/src/main/python/kazoo/testing/common.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 #
 #  Copyright (C) 2010-2011, 2011 Canonical Ltd. All Rights Reserved
 #

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/testing/harness.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/testing/harness.py b/slider-agent/src/main/python/kazoo/testing/harness.py
index 0a9079a..93cc744 100644
--- a/slider-agent/src/main/python/kazoo/testing/harness.py
+++ b/slider-agent/src/main/python/kazoo/testing/harness.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 """Kazoo testing harnesses"""
 import atexit
 import logging

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_barrier.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_barrier.py b/slider-agent/src/main/python/kazoo/tests/test_barrier.py
index 94b6e92..461664f 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_barrier.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_barrier.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 import threading
 
 from nose.tools import eq_

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_build.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_build.py b/slider-agent/src/main/python/kazoo/tests/test_build.py
index 1f9316f..0f75d7c 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_build.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_build.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 import os
 
 from nose import SkipTest

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_client.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_client.py b/slider-agent/src/main/python/kazoo/tests/test_client.py
index b99ee36..eb19ef5 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_client.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_client.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 import socket
 import sys
 import threading

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_connection.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_connection.py b/slider-agent/src/main/python/kazoo/tests/test_connection.py
index faa121e..c764b03 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_connection.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_connection.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 from collections import namedtuple
 import os
 import errno

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_counter.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_counter.py b/slider-agent/src/main/python/kazoo/tests/test_counter.py
index d582a45..b0361d0 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_counter.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_counter.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 import uuid
 
 from nose.tools import eq_

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_election.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_election.py b/slider-agent/src/main/python/kazoo/tests/test_election.py
index 12a04a2..a9610bf 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_election.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_election.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 import uuid
 import sys
 import threading

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_exceptions.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_exceptions.py b/slider-agent/src/main/python/kazoo/tests/test_exceptions.py
index 154bcb7..e469089 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_exceptions.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_exceptions.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 from unittest import TestCase
 
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_gevent_handler.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_gevent_handler.py b/slider-agent/src/main/python/kazoo/tests/test_gevent_handler.py
index 67ccb11..71d9727 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_gevent_handler.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_gevent_handler.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 import unittest
 
 from nose import SkipTest

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_lock.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_lock.py b/slider-agent/src/main/python/kazoo/tests/test_lock.py
index 36b6d42..6dd15b0 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_lock.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_lock.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 import uuid
 import threading
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_partitioner.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_partitioner.py b/slider-agent/src/main/python/kazoo/tests/test_partitioner.py
index a323d07..1a4f205 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_partitioner.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_partitioner.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 import uuid
 import time
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_party.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_party.py b/slider-agent/src/main/python/kazoo/tests/test_party.py
index d44eec7..61400ae 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_party.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_party.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 import uuid
 
 from nose.tools import eq_

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_paths.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_paths.py b/slider-agent/src/main/python/kazoo/tests/test_paths.py
index e092196..c9064bb 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_paths.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_paths.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 import sys
 from unittest import TestCase
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_queue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_queue.py b/slider-agent/src/main/python/kazoo/tests/test_queue.py
index 6a9ec68..4c13ca9 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_queue.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_queue.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 import uuid
 
 from nose import SkipTest

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_retry.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_retry.py b/slider-agent/src/main/python/kazoo/tests/test_retry.py
index b85739b..84c8d41 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_retry.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_retry.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 import unittest
 
 from nose.tools import eq_

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_security.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_security.py b/slider-agent/src/main/python/kazoo/tests/test_security.py
index 4a7e670..587c265 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_security.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_security.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 import unittest
 
 from nose.tools import eq_

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_threading_handler.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_threading_handler.py b/slider-agent/src/main/python/kazoo/tests/test_threading_handler.py
index b4b93f1..4de5781 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_threading_handler.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_threading_handler.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 import threading
 import unittest
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_watchers.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_watchers.py b/slider-agent/src/main/python/kazoo/tests/test_watchers.py
index 1c51d60..44795c4 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_watchers.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_watchers.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 import time
 import threading
 import uuid

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/util.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/util.py b/slider-agent/src/main/python/kazoo/tests/util.py
index 298be71..906cbc0 100644
--- a/slider-agent/src/main/python/kazoo/tests/util.py
+++ b/slider-agent/src/main/python/kazoo/tests/util.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
 ##############################################################################
 #
 # Copyright Zope Foundation and Contributors.


[11/50] [abbrv] git commit: SLIDER-310 failure threshold can be set per component, which overrides the global value

Posted by st...@apache.org.
SLIDER-310 failure threshold can be set per component, which overrides the global value


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/564462e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/564462e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/564462e8

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 564462e89fcc80b6214585491ce712d65edd1bed
Parents: 2f8bda2
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 14 12:37:05 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 14 13:19:27 2014 +0100

----------------------------------------------------------------------
 .../slider/server/appmaster/state/AppState.java | 17 +++++++++--
 .../TestRegionServerFailureThreshold.groovy     | 31 ++++++++++++++++----
 2 files changed, 40 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/564462e8/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index d616678..d22c716 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1480,7 +1480,8 @@ public class AppState {
       throw new TriggerClusterTeardownException(
         SliderExitCodes.EXIT_DEPLOYMENT_FAILED,
         ErrorStrings.E_UNSTABLE_CLUSTER +
-        " - failed with role %s failing %d times (%d in startup); threshold is %d - last failure: %s",
+        " - failed with role %s failing %d times (%d in startup);" +
+        " threshold is %d - last failure: %s",
         role.getName(),
         role.getFailed(),
         role.getStartFailed(),
@@ -1489,8 +1490,18 @@ public class AppState {
     }
   }
 
-  private int getFailureThresholdForRole(RoleStatus status) {
-    return failureThreshold;
+  /**
+   * Get the failure threshold for a specific role, falling back to
+   * the global one if not
+   * @param roleStatus
+   * @return the threshold for failures
+   */
+  private int getFailureThresholdForRole(RoleStatus roleStatus) {
+    ConfTreeOperations resources =
+        instanceDefinition.getResourceOperations();
+    return resources.getComponentOptInt(roleStatus.getName(),
+        ResourceKeys.CONTAINER_FAILURE_SHORTLIFE,
+        failureThreshold);
   }
   
   /**

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/564462e8/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy
index b3aaa48..eb44ae0 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy
@@ -34,6 +34,8 @@ import org.apache.slider.client.SliderClient
 import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
 import org.junit.Test
 
+import static org.apache.slider.providers.hbase.HBaseKeys.ROLE_WORKER
+
 /**
  * test that if a container is killed too many times,
  * the AM stays down
@@ -44,10 +46,17 @@ import org.junit.Test
 class TestRegionServerFailureThreshold extends HBaseMiniClusterTestBase {
 
   @Test
-  public void testFailedRegionService() throws Throwable {
+  public void testRegionServerFailureThreshold() throws Throwable {
     failureThresholdTestRun("", true, 2, 5)
   }
 
+  /**
+   * Sets the failure threshold then runs the #of kill attempts
+   * @param testName
+   * @param toKill
+   * @param threshold
+   * @param killAttempts
+   */
   private void failureThresholdTestRun(
       String testName,
       boolean toKill,
@@ -57,16 +66,23 @@ class TestRegionServerFailureThreshold extends HBaseMiniClusterTestBase {
     int regionServerCount = 1
     String clustername = createMiniCluster(testName, configuration, 1, 1, 1, true, true)
     describe(
-        "Create a single region service HBase instance then " + action + " the RS");
+        "Create a single region service HBase instance" +
+        "then $action the RS $killAttempts times with a threshold of $threshold");
 
     //now launch the cluster
+    def globalThreshold = threshold - 1
     ServiceLauncher<SliderClient> launcher = createHBaseCluster(
         clustername,
         regionServerCount,
         [
+            Arguments.ARG_RES_COMP_OPT,
+            ROLE_WORKER,
+            ResourceKeys.CONTAINER_FAILURE_THRESHOLD,
+            Integer.toString(threshold),
+
             Arguments.ARG_RESOURCE_OPT, 
             ResourceKeys.CONTAINER_FAILURE_THRESHOLD,
-            Integer.toString(threshold)
+            Integer.toString(globalThreshold)
         ],
         true,
         true)
@@ -74,9 +90,14 @@ class TestRegionServerFailureThreshold extends HBaseMiniClusterTestBase {
     addToTeardown(client);
     def aggregateConf = client.loadPersistedClusterDescription(clustername)
     log.info aggregateConf.toString()
-    def failureOptValue = aggregateConf.resourceOperations.globalOptions.getMandatoryOptionInt(
+
+    def resourceOperations = aggregateConf.resourceOperations
+    def failureOptValue = resourceOperations.globalOptions.getMandatoryOptionInt(
         ResourceKeys.CONTAINER_FAILURE_THRESHOLD)
-    assert threshold == failureOptValue
+    assert globalThreshold == failureOptValue
+    def workerThreshold = resourceOperations.getComponentOptInt(ROLE_WORKER,
+        ResourceKeys.CONTAINER_FAILURE_THRESHOLD, 0)
+    assert threshold == workerThreshold
     ClusterDescription status = client.getClusterDescription(clustername)
 
     ClusterStatus clustat = basicHBaseClusterStartupSequence(client)


[10/50] [abbrv] git commit: SLIDER-313. Slider AM fails to restart on kill, post newly created application.

Posted by st...@apache.org.
SLIDER-313. Slider AM fails to restart on kill, post newly created application.


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2f8bda2f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2f8bda2f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2f8bda2f

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 2f8bda2f3da4e8ddc37710e66d29d36652754a0e
Parents: 9b20e47
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Aug 13 14:23:49 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Aug 13 14:23:49 2014 -0700

----------------------------------------------------------------------
 .../providers/agent/AgentProviderService.java   | 27 ++++++++++++--------
 1 file changed, 16 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2f8bda2f/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 0c57b79..43ee7a6 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.slider.api.ClusterDescription;
 import org.apache.slider.api.ClusterDescriptionKeys;
 import org.apache.slider.api.ClusterNode;
@@ -63,6 +62,7 @@ import org.apache.slider.providers.agent.application.metadata.Metainfo;
 import org.apache.slider.providers.agent.application.metadata.OSPackage;
 import org.apache.slider.providers.agent.application.metadata.OSSpecific;
 import org.apache.slider.server.appmaster.actions.ProviderReportedContainerLoss;
+import org.apache.slider.server.appmaster.state.ContainerPriority;
 import org.apache.slider.server.appmaster.state.StateAccessForProviders;
 import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType;
 import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
@@ -339,16 +339,21 @@ public class AgentProviderService extends AbstractProviderService implements
       String applicationId, Map<Integer, ProviderRole> providerRoleMap) {
     for (Container container : liveContainers) {
       // get the role name and label
-      Priority priority = container.getPriority();
-      String roleName = providerRoleMap.get(priority.getPriority()).name;
-      String label = getContainerLabel(container, roleName);
-      log.info("Rebuilding in-memory: container {} in role {} in cluster {}",
-          container.getId().toString(), roleName, applicationId);
-      getComponentStatuses()
-          .put(
-              label,
-              new ComponentInstanceState(roleName, container.getId(),
-                  applicationId));
+      ProviderRole role = providerRoleMap.get(ContainerPriority
+          .extractRole(container));
+      if (role != null) {
+        String roleName = role.name;
+        String label = getContainerLabel(container, roleName);
+        log.info("Rebuilding in-memory: container {} in role {} in cluster {}",
+            container.getId(), roleName, applicationId);
+        getComponentStatuses().put(
+            label,
+            new ComponentInstanceState(roleName, container.getId(),
+                applicationId));
+      } else {
+        log.warn("Role not found for container {} in cluster {}",
+            container.getId(), applicationId);
+      }
     }
   }
 


[15/50] [abbrv] git commit: SLIDER-322 move all <> constructors to typed

Posted by st...@apache.org.
SLIDER-322 move all <> constructors to typed


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/daba289e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/daba289e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/daba289e

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: daba289e9327150078f3a38724db2dcbaec2ab12
Parents: 52f1f6f
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 14 16:47:45 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 14 16:47:45 2014 +0100

----------------------------------------------------------------------
 .../src/main/java/org/apache/slider/Slider.java |  2 +-
 .../apache/slider/api/ClusterDescription.java   | 14 ++---
 .../org/apache/slider/client/SliderClient.java  |  6 +--
 .../slider/client/SliderYarnClientImpl.java     |  8 +--
 .../common/params/AbstractActionArgs.java       |  6 +--
 .../params/AppAndResouceOptionArgsDelegate.java |  8 +--
 .../org/apache/slider/common/params/ArgOps.java |  8 +--
 .../apache/slider/common/params/CommonArgs.java |  4 +-
 .../common/params/DontSplitArguments.java       |  2 +-
 .../slider/common/tools/ConfigHelper.java       |  4 +-
 .../slider/common/tools/CoreFileSystem.java     |  2 +-
 .../apache/slider/common/tools/SliderUtils.java | 10 ++--
 .../org/apache/slider/core/conf/ConfTree.java   |  6 +--
 .../slider/core/conf/ConfTreeOperations.java    |  4 +-
 .../apache/slider/core/conf/MapOperations.java  |  2 +-
 .../slider/core/launch/AbstractLauncher.java    | 10 ++--
 .../core/launch/ClasspathConstructor.java       |  2 +-
 .../slider/core/launch/CommandLineBuilder.java  |  2 +-
 .../slider/core/main/ServiceLauncher.java       |  2 +-
 .../registry/docstore/PublishedConfigSet.java   |  2 +-
 .../docstore/PublishedConfiguration.java        |  4 +-
 .../slider/core/registry/docstore/UriMap.java   |  2 +-
 .../slider/core/registry/info/RegistryView.java |  3 +-
 .../core/registry/info/ServiceInstanceData.java |  4 +-
 .../apache/slider/core/zk/ZKIntegration.java    |  2 +-
 .../apache/slider/core/zk/ZookeeperUtils.java   |  6 +--
 .../providers/AbstractProviderService.java      |  4 +-
 .../apache/slider/providers/ProviderUtils.java  |  2 +-
 .../providers/agent/AgentClientProvider.java    |  2 +-
 .../providers/agent/AgentLaunchParameter.java   |  2 +-
 .../providers/agent/AgentProviderService.java   | 47 ++++++++--------
 .../providers/agent/ComponentCommandOrder.java  |  9 ++--
 .../agent/application/metadata/Application.java |  8 +--
 .../agent/application/metadata/Component.java   |  2 +-
 .../slideram/SliderAMClientProvider.java        |  4 +-
 .../slideram/SliderAMProviderService.java       |  2 +-
 .../server/appmaster/SliderAppMaster.java       | 18 +++----
 .../appmaster/actions/ActionKillContainer.java  |  2 +-
 .../server/appmaster/actions/QueueService.java  |  6 +--
 .../appmaster/monkey/ChaosMonkeyService.java    |  2 +-
 .../slider/server/appmaster/state/AppState.java | 56 ++++++++++----------
 .../MostRecentContainerReleaseSelector.java     |  2 +-
 .../server/appmaster/state/NodeInstance.java    |  2 +-
 .../slider/server/appmaster/state/NodeMap.java  |  2 +-
 .../state/OutstandingRequestTracker.java        |  6 +--
 .../appmaster/state/ProviderAppState.java       |  4 +-
 .../server/appmaster/state/RoleHistory.java     | 15 +++---
 .../server/appmaster/state/RoleStatus.java      |  2 +-
 .../server/appmaster/web/SliderAMWebApp.java    |  2 +-
 .../web/SliderAmFilterInitializer.java          |  2 +-
 .../server/appmaster/web/SliderAmIpFilter.java  |  2 +-
 .../server/appmaster/web/WebAppApiImpl.java     |  2 +-
 .../web/rest/publisher/PublisherResource.java   |  4 +-
 .../slider/server/avro/RoleHistoryWriter.java   |  6 +--
 .../server/servicemonitor/MonitorUtils.java     |  2 +-
 .../services/curator/RegistryBinderService.java |  6 +--
 .../registry/RegistryRestResources.java         |  3 +-
 .../registry/SliderRegistryService.java         |  2 +-
 58 files changed, 181 insertions(+), 174 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/Slider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/Slider.java b/slider-core/src/main/java/org/apache/slider/Slider.java
index 0d25f00..5fc8618 100644
--- a/slider-core/src/main/java/org/apache/slider/Slider.java
+++ b/slider-core/src/main/java/org/apache/slider/Slider.java
@@ -42,7 +42,7 @@ public class Slider extends SliderClient {
     //turn the args to a list
     List<String> argsList = Arrays.asList(args);
     //create a new list, as the ArrayList type doesn't push() on an insert
-    List<String> extendedArgs = new ArrayList<>(argsList);
+    List<String> extendedArgs = new ArrayList<String>(argsList);
     //insert the service name
     extendedArgs.add(0, SERVICE_CLASSNAME);
     //now have the service launcher do its work

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java b/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
index d5869a6..d875d66 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
@@ -165,40 +165,40 @@ public class ClusterDescription implements Cloneable {
    * the Slider AM and the application that it deploys
    */
   public Map<String, String> options =
-    new HashMap<>();
+    new HashMap<String, String>();
 
   /**
    * cluster information
    * This is only valid when querying the cluster status.
    */
   public Map<String, String> info =
-    new HashMap<>();
+    new HashMap<String, String>();
 
   /**
    * Statistics. This is only relevant when querying the cluster status
    */
   public Map<String, Map<String, Integer>> statistics =
-    new HashMap<>();
+    new HashMap<String, Map<String, Integer>>();
 
   /**
    * Instances: role->count
    */
   public Map<String, List<String>> instances =
-    new HashMap<>();
+    new HashMap<String, List<String>>();
 
   /**
    * Role options, 
    * role -> option -> value
    */
   public Map<String, Map<String, String>> roles =
-    new HashMap<>();
+    new HashMap<String, Map<String, String>>();
 
 
   /**
    * List of key-value pairs to add to a client config to set up the client
    */
   public Map<String, String> clientProperties =
-    new HashMap<>();
+    new HashMap<String, String>();
 
   /**
    * Status information
@@ -569,7 +569,7 @@ public class ClusterDescription implements Cloneable {
    */
   @JsonIgnore
   public Set<String> getRoleNames() {
-    return new HashSet<>(roles.keySet());
+    return new HashSet<String>(roles.keySet());
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 1337570..356f99f 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1381,7 +1381,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
     verifyBindingsDefined();
     SliderUtils.validateClusterName(name);
     log.debug("actionFlex({})", name);
-    Map<String, Integer> roleInstances = new HashMap<>();
+    Map<String, Integer> roleInstances = new HashMap<String, Integer>();
     Map<String, String> roleMap = args.getComponentMap();
     for (Map.Entry<String, String> roleEntry : roleMap.entrySet()) {
       String key = roleEntry.getKey();
@@ -1934,7 +1934,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
 
     if (uuids.length == 0) {
       // short cut on an empty list
-      return new LinkedList<>();
+      return new LinkedList<ClusterNode>();
     }
     return createClusterOperations().listClusterNodes(uuids);
   }
@@ -2133,7 +2133,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
                                       + serviceType
                                       + " name " + name);
     }
-    List<ServiceInstanceData> sids = new ArrayList<>(size);
+    List<ServiceInstanceData> sids = new ArrayList<ServiceInstanceData>(size);
     for (CuratorServiceInstance<ServiceInstanceData> instance : instances) {
       ServiceInstanceData payload = instance.payload;
       logInstance(payload, registryArgs.verbose);

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index e7b492b..3151a09 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -70,10 +70,10 @@ public class SliderYarnClientImpl extends YarnClientImpl {
    */
   public List<ApplicationReport> listInstances(String user)
     throws YarnException, IOException {
-    Set<String> types = new HashSet<>(1);
+    Set<String> types = new HashSet<String>(1);
     types.add(SliderKeys.APP_TYPE);
     List<ApplicationReport> allApps = getApplications(types);
-    List<ApplicationReport> results = new ArrayList<>();
+    List<ApplicationReport> results = new ArrayList<ApplicationReport>();
     for (ApplicationReport report : allApps) {
       if (StringUtils.isEmpty(user) || user.equals(report.getUser())) {
         results.add(report);
@@ -97,7 +97,7 @@ public class SliderYarnClientImpl extends YarnClientImpl {
                                                                   YarnException {
     List<ApplicationReport> instances = listInstances(user);
     List<ApplicationReport> results =
-      new ArrayList<>(instances.size());
+      new ArrayList<ApplicationReport>(instances.size());
     for (ApplicationReport report : instances) {
       if (report.getName().equals(appname)) {
         results.add(report);
@@ -240,7 +240,7 @@ public class SliderYarnClientImpl extends YarnClientImpl {
                                                                       IOException {
     List<ApplicationReport> instances = listInstances(user);
     List<ApplicationReport> results =
-      new ArrayList<>(instances.size());
+      new ArrayList<ApplicationReport>(instances.size());
     for (ApplicationReport app : instances) {
       if (app.getName().equals(appname)
           && isApplicationLive(app)) {

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
index 3a8cea4..f4a4569 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
@@ -55,7 +55,7 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
    * This is the default parameter
    */
   @Parameter
-  public final List<String> parameters = new ArrayList<>();
+  public final List<String> parameters = new ArrayList<String>();
 
   /**
    * get the name: relies on arg 1 being the cluster name in all operations 
@@ -76,7 +76,7 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
    */
 
   @Parameter(names = ARG_DEFINE, arity = 1, description = "Definitions")
-  public final List<String> definitions = new ArrayList<>();
+  public final List<String> definitions = new ArrayList<String>();
 
   /**
    * System properties
@@ -84,7 +84,7 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
   @Parameter(names = {ARG_SYSPROP}, arity = 1,
              description = "system properties in the form name value" +
                            " These are set after the JVM is started.")
-  public final List<String> sysprops = new ArrayList<>(0);
+  public final List<String> sysprops = new ArrayList<String>(0);
 
 
   @Parameter(names = {ARG_MANAGER_SHORT, ARG_MANAGER},

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
index 248e4c2..1f07de3 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
@@ -37,7 +37,7 @@ public class AppAndResouceOptionArgsDelegate extends AbstractArgsDelegate {
   @Parameter(names = {ARG_OPTION, ARG_OPTION_SHORT}, arity = 2,
              description = ARG_OPTION + "<name> <value>",
              splitter = DontSplitArguments.class)
-  public List<String> optionTuples = new ArrayList<>(0);
+  public List<String> optionTuples = new ArrayList<String>(0);
 
 
   /**
@@ -47,7 +47,7 @@ public class AppAndResouceOptionArgsDelegate extends AbstractArgsDelegate {
              description = "Component option " + ARG_COMP_OPT +
                            " <component> <name> <option>",
              splitter = DontSplitArguments.class)
-  public List<String> compOptTriples = new ArrayList<>(0);
+  public List<String> compOptTriples = new ArrayList<String>(0);
 
   /**
    * Resource Options
@@ -55,7 +55,7 @@ public class AppAndResouceOptionArgsDelegate extends AbstractArgsDelegate {
   @Parameter(names = {ARG_RESOURCE_OPT, ARG_RESOURCE_OPT_SHORT}, arity = 2,
              description = "Resource option "+ ARG_RESOURCE_OPT + "<name> <value>",
              splitter = DontSplitArguments.class)
-  public List<String> resOptionTuples = new ArrayList<>(0);
+  public List<String> resOptionTuples = new ArrayList<String>(0);
 
 
   /**
@@ -65,7 +65,7 @@ public class AppAndResouceOptionArgsDelegate extends AbstractArgsDelegate {
              description = "Component resource option " + ARG_RES_COMP_OPT +
                            " <component> <name> <option>",
              splitter = DontSplitArguments.class)
-  public List<String> resCompOptTriples = new ArrayList<>(0);
+  public List<String> resCompOptTriples = new ArrayList<String>(0);
 
 
   public Map<String, String> getOptionsMap() throws

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java b/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
index b53ebd3..83754b3 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
@@ -44,7 +44,7 @@ public class ArgOps {
    * create a 3-tuple
    */
   public static List<Object> triple(String msg, int min, int max) {
-    List<Object> l = new ArrayList<>(3);
+    List<Object> l = new ArrayList<Object>(3);
     l.add(msg);
     l.add(min);
     l.add(max);
@@ -100,7 +100,7 @@ public class ArgOps {
   public static Map<String, String> convertTupleListToMap(String description,
                                                           List<String> list) throws
                                                                              BadCommandArgumentsException {
-    Map<String, String> results = new HashMap<>();
+    Map<String, String> results = new HashMap<String, String>();
     if (list != null && !list.isEmpty()) {
       int size = list.size();
       if (size % 2 != 0) {
@@ -134,7 +134,7 @@ public class ArgOps {
                                                                          List<String> list) throws
                                                                                             BadCommandArgumentsException {
     Map<String, Map<String, String>> results =
-      new HashMap<>();
+      new HashMap<String, Map<String, String>>();
     if (list != null && !list.isEmpty()) {
       int size = list.size();
       if (size % 3 != 0) {
@@ -149,7 +149,7 @@ public class ArgOps {
         Map<String, String> roleMap = results.get(role);
         if (roleMap == null) {
           //demand create new role map
-          roleMap = new HashMap<>();
+          roleMap = new HashMap<String, String>();
           results.put(role, roleMap);
         }
         if (roleMap.get(key) != null) {

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
index 83e658d..5d94182 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
@@ -62,11 +62,11 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
 
    */
 
-  public Map<String, String> definitionMap = new HashMap<>();
+  public Map<String, String> definitionMap = new HashMap<String, String>();
   /**
    * System properties
    */
-  public Map<String, String> syspropsMap = new HashMap<>();
+  public Map<String, String> syspropsMap = new HashMap<String, String>();
 
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java b/slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
index 0344305..3225133 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
@@ -27,7 +27,7 @@ public class DontSplitArguments implements IParameterSplitter {
 
   @Override
   public List<String> split(String value) {
-    List<String> list = new ArrayList<>(1);
+    List<String> list = new ArrayList<String>(1);
     list.add(value);
     return list;
   }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
index 2f0b9ac..f8fd226 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
@@ -81,7 +81,7 @@ public class ConfigHelper {
 
    */
   public static TreeSet<String> sortedConfigKeys(Iterable<Map.Entry<String, String>> conf) {
-    TreeSet<String> sorted = new TreeSet<>();
+    TreeSet<String> sorted = new TreeSet<String>();
     for (Map.Entry<String, String> entry : conf) {
       sorted.add(entry.getKey());
     }
@@ -510,7 +510,7 @@ public class ConfigHelper {
    * @return hash map
    */
   public static Map<String, String> buildMapFromConfiguration(Configuration conf) {
-    Map<String, String> map = new HashMap<>();
+    Map<String, String> map = new HashMap<String, String>();
     return SliderUtils.mergeEntries(map, conf);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index 9718be9..2381a82 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -442,7 +442,7 @@ public class CoreFileSystem {
     //copied to the destination
     FileStatus[] fileset = fileSystem.listStatus(srcDir);
     Map<String, LocalResource> localResources =
-            new HashMap<>(fileset.length);
+            new HashMap<String, LocalResource>(fileset.length);
     for (FileStatus entry : fileset) {
 
       LocalResource resource = createAmResource(entry.getPath(),

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 7a80f24..188b7d9 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -440,7 +440,7 @@ public final class SliderUtils {
    * @return a stringified list
    */
   public static List<String> collectionToStringList(Collection c) {
-    List<String> l = new ArrayList<>(c.size());
+    List<String> l = new ArrayList<String>(c.size());
     for (Object o : c) {
       l.add(o.toString());
     }
@@ -839,7 +839,7 @@ public final class SliderUtils {
    * @return a possibly empty map of environment variables.
    */
   public static Map<String, String> buildEnvMap(Map<String, String> roleOpts) {
-    Map<String, String> env = new HashMap<>();
+    Map<String, String> env = new HashMap<String, String>();
     if (roleOpts != null) {
       for (Map.Entry<String, String> entry: roleOpts.entrySet()) {
         String key = entry.getKey();
@@ -866,7 +866,7 @@ public final class SliderUtils {
       Map<String, String> optionMap = entry.getValue();
       Map<String, String> existingMap = clusterRoleMap.get(key);
       if (existingMap == null) {
-        existingMap = new HashMap<>();
+        existingMap = new HashMap<String, String>();
       }
       log.debug("Overwriting role options with command line values {}",
                 stringifyMap(optionMap));
@@ -1031,7 +1031,7 @@ public final class SliderUtils {
   }
 
     public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) {
-    Map<String, Map<String, String>> dest = new HashMap<>();
+    Map<String, Map<String, String>> dest = new HashMap<String, Map<String, String>>();
     for (Map.Entry<String, Map<String, String>> entry : src.entrySet()) {
       dest.put(entry.getKey(), stringMapClone(entry.getValue()));
     }
@@ -1039,7 +1039,7 @@ public final class SliderUtils {
   }
 
   public static Map<String, String> stringMapClone(Map<String, String> src) {
-    Map<String, String> dest =  new HashMap<>();
+    Map<String, String> dest =  new HashMap<String, String>();
     return mergeEntries(dest, src.entrySet());
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
index 2cbfd54..5517771 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
@@ -55,14 +55,14 @@ public final class ConfTree {
   /**
    * Metadata
    */
-  public Map<String, Object> metadata = new HashMap<>(INITAL_MAP_CAPACITY);
+  public Map<String, Object> metadata = new HashMap<String, Object >(INITAL_MAP_CAPACITY);
 
 
   /**
    * Global options
    */
   public Map<String, String> global =
-    new HashMap<>(INITAL_MAP_CAPACITY);
+    new HashMap<String, String>(INITAL_MAP_CAPACITY);
 
 
   /**
@@ -70,7 +70,7 @@ public final class ConfTree {
    * role -> option -> value
    */
   public Map<String, Map<String, String>> components =
-    new HashMap<>(INITAL_MAP_CAPACITY);
+    new HashMap<String, Map<String, String>>(INITAL_MAP_CAPACITY);
 
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
index 1cb537a..bb17547 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
@@ -148,7 +148,7 @@ public class ConfTreeOperations {
       return operations;
     }
     //create a new instances
-    Map<String, String> map = new HashMap<>();
+    Map<String, String> map = new HashMap<String, String>();
     confTree.components.put(name, map);
     return new MapOperations(name, map);
   }
@@ -159,7 +159,7 @@ public class ConfTreeOperations {
    */
   @JsonIgnore
   public Set<String> getComponentNames() {
-    return new HashSet<>(confTree.components.keySet());
+    return new HashSet<String>(confTree.components.keySet());
   }
   
   

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
index 98a51bc..4b1b44f 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
@@ -47,7 +47,7 @@ public class MapOperations implements Map<String, String> {
   public final String name;
 
   public MapOperations() {
-    options = new HashMap<>();
+    options = new HashMap<String, String>();
     name = "";
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
index 8652b4c..644f627 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -58,15 +58,15 @@ public abstract class AbstractLauncher extends Configured {
   /**
    * Env vars; set up at final launch stage
    */
-  protected final Map<String, String> envVars = new HashMap<>();
+  protected final Map<String, String> envVars = new HashMap<String, String>();
   protected final MapOperations env = new MapOperations("env", envVars);
   protected final ContainerLaunchContext containerLaunchContext =
     Records.newRecord(ContainerLaunchContext.class);
-  protected final List<String> commands = new ArrayList<>(20);
+  protected final List<String> commands = new ArrayList<String>(20);
   protected final Map<String, LocalResource> localResources =
-    new HashMap<>();
+    new HashMap<String, LocalResource>();
   private final Map<String, ByteBuffer> serviceData =
-    new HashMap<>();
+    new HashMap<String, ByteBuffer>();
   // security
   Credentials credentials = new Credentials();
 
@@ -269,7 +269,7 @@ public abstract class AbstractLauncher extends Configured {
 
   public String[] dumpEnvToString() {
 
-    List<String> nodeEnv = new ArrayList<>();
+    List<String> nodeEnv = new ArrayList<String>();
 
     for (Map.Entry<String, String> entry : env.entrySet()) {
       String envElt = String.format("%s=\"%s\"",

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
index 3527149..8e49435 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
@@ -39,7 +39,7 @@ public class ClasspathConstructor {
 
     public static final String CLASS_PATH_SEPARATOR = ApplicationConstants.CLASS_PATH_SEPARATOR;
 //  public static final String CLASS_PATH_SEPARATOR = File.pathSeparator;
-  private final List<String> pathElements = new ArrayList<>();
+  private final List<String> pathElements = new ArrayList<String>();
 
   public ClasspathConstructor() {
   }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java b/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
index dbaa981..f50bb48 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
@@ -30,7 +30,7 @@ import java.util.List;
  * Special support for JVM command buildup.
  */
 public class CommandLineBuilder {
-  protected final List<String> argumentList = new ArrayList<>(20);
+  protected final List<String> argumentList = new ArrayList<String>(20);
 
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java b/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
index 846aca1..df12849 100644
--- a/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
@@ -419,7 +419,7 @@ public class ServiceLauncher<S extends Service>
     if (argCount <= 1 ) {
       return new String[0];
     }
-    List<String> argsList = new ArrayList<>(argCount);
+    List<String> argsList = new ArrayList<String>(argCount);
     ListIterator<String> arguments = args.listIterator();
     //skip that first entry
     arguments.next();

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
index f498916..eac34c0 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
@@ -41,7 +41,7 @@ public class PublishedConfigSet {
       RestPaths.PUBLISHED_CONFIGURATION_REGEXP);
   
   public Map<String, PublishedConfiguration> configurations =
-      new HashMap<>();
+      new HashMap<String, PublishedConfiguration>();
 
   public PublishedConfigSet() {
   }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
index 93282cc..f76b93b 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
@@ -46,7 +46,7 @@ public class PublishedConfiguration {
   
   public String updatedTime;
 
-  public Map<String, String> entries = new HashMap<>();
+  public Map<String, String> entries = new HashMap<String, String>();
 
   public PublishedConfiguration() {
   }
@@ -112,7 +112,7 @@ public class PublishedConfiguration {
    * @param entries entries to put
    */
   public void putValues(Iterable<Map.Entry<String, String>> entries) {
-    this.entries = new HashMap<>();
+    this.entries = new HashMap<String, String>();
     for (Map.Entry<String, String> entry : entries) {
       this.entries.put(entry.getKey(), entry.getValue());
     }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java
index 120966f..a76e28d 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java
@@ -29,7 +29,7 @@ import java.util.Map;
 @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
 public class UriMap {
 
-  public Map<String, String> uris = new HashMap<>();
+  public Map<String, String> uris = new HashMap<String, String>();
   
   @JsonIgnore
   public void put(String key, String value) {

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java
index 07e4981..bdf70a2 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java
@@ -31,7 +31,8 @@ public class RegistryView {
   /**
    * Endpoints
    */
-  public Map<String, RegisteredEndpoint> endpoints = new HashMap<>(2);
+  public Map<String, RegisteredEndpoint> endpoints =
+      new HashMap<String, RegisteredEndpoint>(2);
 
   public String configurationsURL;
   

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java
index 80f0b34..c3c7e63 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java
@@ -124,13 +124,13 @@ public class ServiceInstanceData implements Serializable {
   public Map<String, RegisteredEndpoint> listEndpoints(boolean external) {
     RegistryView view = getRegistryView(external);
     if (view == null) {
-      return new HashMap<>(0);
+      return new HashMap<String, RegisteredEndpoint>(0);
     }
     Map<String, RegisteredEndpoint> endpoints = view.endpoints;
     if (endpoints != null) {
       return endpoints;
     } else {
-      return new HashMap<>(0);
+      return new HashMap<String, RegisteredEndpoint>(0);
     }
   }
   

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java b/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
index 54aeb4f..0d96559 100644
--- a/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
+++ b/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
@@ -49,7 +49,7 @@ public class ZKIntegration implements Watcher {
   public static String SVC_SLIDER = "/" + ZK_SERVICES + "/" + ZK_SLIDER;
   public static String SVC_SLIDER_USERS = SVC_SLIDER + "/" + ZK_USERS;
 
-  public static final List<String> ZK_USERS_PATH_LIST = new ArrayList<>();
+  public static final List<String> ZK_USERS_PATH_LIST = new ArrayList<String>();
   static {
     ZK_USERS_PATH_LIST.add(ZK_SERVICES);
     ZK_USERS_PATH_LIST.add(ZK_SLIDER);

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java b/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
index 8bf25f9..61b1ff0 100644
--- a/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
@@ -49,7 +49,7 @@ public class ZookeeperUtils {
     if (strings != null) {
       len = strings.length;
     }
-    List<String> tuples = new ArrayList<>(len);
+    List<String> tuples = new ArrayList<String>(len);
     if (strings != null) {
       for (String s : strings) {
         tuples.add(s.trim());
@@ -70,7 +70,7 @@ public class ZookeeperUtils {
     if (strings != null) {
       len = strings.length;
     }
-    List<HostAndPort> list = new ArrayList<>(len);
+    List<HostAndPort> list = new ArrayList<HostAndPort>(len);
     if (strings != null) {
       for (String s : strings) {
         list.add(HostAndPort.fromString(s.trim()));
@@ -113,7 +113,7 @@ public class ZookeeperUtils {
    * @return
    */
   public static String buildQuorum(List<HostAndPort> hostAndPorts, int defaultPort) {
-    List<String> entries = new ArrayList<>(hostAndPorts.size());
+    List<String> entries = new ArrayList<String>(hostAndPorts.size());
     for (HostAndPort hostAndPort : hostAndPorts) {
       entries.add(buildQuorumEntry(hostAndPort, defaultPort));
     }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index a4a71dd..3fbd3cf 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -291,7 +291,7 @@ public abstract class AbstractProviderService
    */
   @Override
   public Map<String, String> buildProviderStatus() {
-    return new HashMap<>();
+    return new HashMap<String, String>();
   }
 
   /*
@@ -300,7 +300,7 @@ public abstract class AbstractProviderService
    */
   @Override
   public Map<String, String> buildMonitorDetails(ClusterDescription clusterDesc) {
-    Map<String, String> details = new LinkedHashMap<>();
+    Map<String, String> details = new LinkedHashMap<String, String>();
 
     // add in all the 
     buildEndpointDetails(details);

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index db9f494..4b8724a 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -419,7 +419,7 @@ public class ProviderUtils implements RoleKeys {
     }
 
     log.debug("Found {} entries in {}", ls.length, base);
-    List<File> directories = new LinkedList<>();
+    List<File> directories = new LinkedList<File>();
     StringBuilder dirs = new StringBuilder();
     for (File file : ls) {
       log.debug("{}", false);

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 72b0a42..8f92ffa 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -216,7 +216,7 @@ public class AgentClientProvider extends AbstractClientProvider
     }
 
     Application application = metainfo.getApplication();
-    tags = new HashSet<>();
+    tags = new HashSet<String>();
     tags.add("Name: " + application.getName());
     tags.add("Version: " + application.getVersion());
     tags.add("Description: " + SliderUtils.truncate(application.getComment(), 80));

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java
index b839e58..c8b0e1d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java
@@ -94,7 +94,7 @@ class AgentLaunchParameter {
             }
 
             if (trackers == null) {
-              trackers = new HashMap<>(10);
+              trackers = new HashMap<String, CommandTracker>(10);
             }
             String componentName = parameters[0];
             CommandTracker tracker = new CommandTracker(Arrays.copyOfRange(parameters, 1, parameters.length));

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 43ee7a6..3225066 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -132,9 +132,12 @@ public class AgentProviderService extends AbstractProviderService implements
   private Boolean canAnyMasterPublish = null;
   private AgentLaunchParameter agentLaunchParameter = null;
 
-  private final Map<String, ComponentInstanceState> componentStatuses = new ConcurrentHashMap<>();
-  private final Map<String, Map<String, String>> componentInstanceData = new ConcurrentHashMap<>();
-  private final Map<String, Map<String, String>> allocatedPorts = new ConcurrentHashMap<>();
+  private final Map<String, ComponentInstanceState> componentStatuses =
+      new ConcurrentHashMap<String, ComponentInstanceState>();
+  private final Map<String, Map<String, String>> componentInstanceData =
+      new ConcurrentHashMap<String, Map<String, String>>();
+  private final Map<String, Map<String, String>> allocatedPorts =
+      new ConcurrentHashMap<String, Map<String, String>>();
   private final Map<String, String> workFolders =
       Collections.synchronizedMap(new LinkedHashMap<String, String>(MAX_LOG_ENTRIES, 0.75f, false) {
         protected boolean removeEldestEntry(Map.Entry eldest) {
@@ -691,7 +694,7 @@ public class AgentProviderService extends AbstractProviderService implements
    * @return the provider status - map of entries to add to the info section
    */
   public Map<String, String> buildProviderStatus() {
-    Map<String, String> stats = new HashMap<>();
+    Map<String, String> stats = new HashMap<String, String>();
     return stats;
   }
 
@@ -707,7 +710,8 @@ public class AgentProviderService extends AbstractProviderService implements
       workFolders.put(String.format("%s-%s-%s", hostFqdn, containerId, key), folders.get(key));
     }
 
-    publishApplicationInstanceData(LOG_FOLDERS_TAG, LOG_FOLDERS_TAG, (new HashMap<>(this.workFolders)).entrySet());
+    publishApplicationInstanceData(LOG_FOLDERS_TAG, LOG_FOLDERS_TAG,
+        (new HashMap<String, String>(this.workFolders)).entrySet());
   }
 
 
@@ -750,7 +754,7 @@ public class AgentProviderService extends AbstractProviderService implements
             String hostKeyFormat = "${%s_HOST}";
 
             // publish export groups if any
-            Map<String, String> replaceTokens = new HashMap<>();
+            Map<String, String> replaceTokens = new HashMap<String, String>();
             for (Map.Entry<String, Map<String, ClusterNode>> entry : getRoleClusterNodeMapping().entrySet()) {
               String hostName = getHostsList(entry.getValue().values(), true).iterator().next();
               replaceTokens.put(String.format(hostKeyFormat, entry.getKey().toUpperCase(Locale.ENGLISH)), hostName);
@@ -768,7 +772,7 @@ public class AgentProviderService extends AbstractProviderService implements
               List<Export> exports = exportGroup.getExports();
               if (exports != null && !exports.isEmpty()) {
                 String exportGroupName = exportGroup.getName();
-                Map<String, String> map = new HashMap<>();
+                Map<String, String> map = new HashMap<String, String>();
                 for (Export export : exports) {
                   String value = export.getValue();
                   // replace host names
@@ -797,7 +801,7 @@ public class AgentProviderService extends AbstractProviderService implements
                                                         String roleName) {
     String portVarFormat = "${site.%s}";
     String hostNamePattern = "${THIS_HOST}";
-    Map<String, String> toPublish = new HashMap<>();
+    Map<String, String> toPublish = new HashMap<String, String>();
 
     Application application = getMetainfo().getApplication();
     for (Component component : application.getComponents()) {
@@ -831,7 +835,7 @@ public class AgentProviderService extends AbstractProviderService implements
     if (toPublish.size() > 0) {
       Map<String, String> perContainerData = null;
       if (!getComponentInstanceData().containsKey(containerId)) {
-        perContainerData = new ConcurrentHashMap<>();
+        perContainerData = new ConcurrentHashMap<String, String>();
       } else {
         perContainerData = getComponentInstanceData().get(containerId);
       }
@@ -842,7 +846,7 @@ public class AgentProviderService extends AbstractProviderService implements
   }
 
   private void publishComponentInstanceData() {
-    Map<String, String> dataToPublish = new HashMap<>();
+    Map<String, String> dataToPublish = new HashMap<String, String>();
     synchronized (this.componentInstanceData) {
       for (String container : getComponentInstanceData().keySet()) {
         for (String prop : getComponentInstanceData().get(container).keySet()) {
@@ -979,7 +983,7 @@ public class AgentProviderService extends AbstractProviderService implements
     cmd.setServiceName(clusterName);
     cmd.setComponentName(roleName);
     cmd.setRole(roleName);
-    Map<String, String> hostLevelParams = new TreeMap<>();
+    Map<String, String> hostLevelParams = new TreeMap<String, String>();
     hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME));
     hostLevelParams.put(PACKAGE_LIST, getPackageList());
     hostLevelParams.put(CONTAINER_ID, containerId);
@@ -1024,7 +1028,7 @@ public class AgentProviderService extends AbstractProviderService implements
   }
 
   private Map<String, String> setCommandParameters(String scriptPath, boolean recordConfig) {
-    Map<String, String> cmdParams = new TreeMap<>();
+    Map<String, String> cmdParams = new TreeMap<String, String>();
     cmdParams.put("service_package_folder",
                   "${AGENT_WORK_ROOT}/work/app/definition/package");
     cmdParams.put("script", scriptPath);
@@ -1057,7 +1061,7 @@ public class AgentProviderService extends AbstractProviderService implements
     cmd.setClusterName(clusterName);
     cmd.setRoleCommand(StatusCommand.STATUS_COMMAND);
 
-    Map<String, String> hostLevelParams = new TreeMap<>();
+    Map<String, String> hostLevelParams = new TreeMap<String, String>();
     hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME));
     hostLevelParams.put(CONTAINER_ID, containerId);
     cmd.setHostLevelParams(hostLevelParams);
@@ -1085,7 +1089,7 @@ public class AgentProviderService extends AbstractProviderService implements
     cmd.setServiceName(clusterName);
     cmd.setClusterName(clusterName);
     cmd.setRoleCommand(StatusCommand.GET_CONFIG_COMMAND);
-    Map<String, String> hostLevelParams = new TreeMap<>();
+    Map<String, String> hostLevelParams = new TreeMap<String, String>();
     hostLevelParams.put(CONTAINER_ID, containerId);
     cmd.setHostLevelParams(hostLevelParams);
 
@@ -1112,7 +1116,7 @@ public class AgentProviderService extends AbstractProviderService implements
     cmd.setServiceName(clusterName);
     cmd.setComponentName(roleName);
     cmd.setRole(roleName);
-    Map<String, String> hostLevelParams = new TreeMap<>();
+    Map<String, String> hostLevelParams = new TreeMap<String, String>();
     hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME));
     hostLevelParams.put(CONTAINER_ID, containerId);
     cmd.setHostLevelParams(hostLevelParams);
@@ -1149,7 +1153,8 @@ public class AgentProviderService extends AbstractProviderService implements
       ConfTreeOperations appConf, String containerId)
       throws SliderException {
 
-    Map<String, Map<String, String>> configurations = new TreeMap<>();
+    Map<String, Map<String, String>> configurations =
+        new TreeMap<String, Map<String, String>>();
     Map<String, String> tokens = getStandardTokenMap(appConf);
 
     List<String> configs = getApplicationConfigurationTypes(appConf);
@@ -1164,7 +1169,7 @@ public class AgentProviderService extends AbstractProviderService implements
   }
 
   private Map<String, String> getStandardTokenMap(ConfTreeOperations appConf) throws SliderException {
-    Map<String, String> tokens = new HashMap<>();
+    Map<String, String> tokens = new HashMap<String, String>();
     String nnuri = appConf.get("site.fs.defaultFS");
     tokens.put("${NN_URI}", nnuri);
     tokens.put("${NN_HOST}", URI.create(nnuri).getHost());
@@ -1180,7 +1185,7 @@ public class AgentProviderService extends AbstractProviderService implements
   private List<String> getApplicationConfigurationTypes(ConfTreeOperations appConf) {
     // for now, reading this from appConf.  In the future, modify this method to
     // process metainfo.xml
-    List<String> configList = new ArrayList<>();
+    List<String> configList = new ArrayList<String>();
     configList.add(GLOBAL_CONFIG_TAG);
 
     String configTypes = appConf.get("config_types");
@@ -1190,13 +1195,13 @@ public class AgentProviderService extends AbstractProviderService implements
     }
 
     // remove duplicates.  mostly worried about 'global' being listed
-    return new ArrayList<>(new HashSet<>(configList));
+    return new ArrayList<String>(new HashSet<String>(configList));
   }
 
   private void addNamedConfiguration(String configName, Map<String, String> sourceConfig,
                                      Map<String, Map<String, String>> configurations,
                                      Map<String, String> tokens, String containerId) {
-    Map<String, String> config = new HashMap<>();
+    Map<String, String> config = new HashMap<String, String>();
     if (configName.equals(GLOBAL_CONFIG_TAG)) {
       addDefaultGlobalConfig(config);
     }
@@ -1235,7 +1240,7 @@ public class AgentProviderService extends AbstractProviderService implements
 
   private Iterable<String> getHostsList(Collection<ClusterNode> values,
                                         boolean hostOnly) {
-    List<String> hosts = new ArrayList<>();
+    List<String> hosts = new ArrayList<String>();
     for (ClusterNode cn : values) {
       hosts.add(hostOnly ? cn.host : cn.host + "/" + cn.name);
     }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
index a23cc38..f4ace5f 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
@@ -37,7 +37,8 @@ public class ComponentCommandOrder {
   public static final Logger log =
       LoggerFactory.getLogger(ComponentCommandOrder.class);
   private static char SPLIT_CHAR = '-';
-  Map<Command, Map<String, List<ComponentState>>> dependencies = new HashMap<>();
+  Map<Command, Map<String, List<ComponentState>>> dependencies =
+      new HashMap<Command, Map<String, List<ComponentState>>>();
 
   public ComponentCommandOrder(List<CommandOrder> commandOrders) {
     if (commandOrders != null && commandOrders.size() > 0) {
@@ -48,13 +49,13 @@ public class ComponentCommandOrder {
         if (requiredStates.size() > 0) {
           Map<String, List<ComponentState>> compDep = dependencies.get(componentCmd.command);
           if (compDep == null) {
-            compDep = new HashMap<>();
+            compDep = new HashMap<String, List<ComponentState>>();
             dependencies.put(componentCmd.command, compDep);
           }
 
           List<ComponentState> requirements = compDep.get(componentCmd.componentName);
           if (requirements == null) {
-            requirements = new ArrayList<>();
+            requirements = new ArrayList<ComponentState>();
             compDep.put(componentCmd.componentName, requirements);
           }
 
@@ -70,7 +71,7 @@ public class ComponentCommandOrder {
     }
 
     String[] componentStates = requires.split(",");
-    List<ComponentState> retList = new ArrayList<>();
+    List<ComponentState> retList = new ArrayList<ComponentState>();
     for (String componentStateStr : componentStates) {
       retList.add(getComponentState(componentStateStr));
     }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
index 016b4d4..d994e33 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
@@ -34,10 +34,10 @@ public class Application {
   ConfigurationDependencies configDependencies;
 
   public Application() {
-    exportGroups = new ArrayList<>();
-    components = new ArrayList<>();
-    osSpecifics = new ArrayList<>();
-    commandOrders = new ArrayList<>();
+    exportGroups = new ArrayList<ExportGroup>();
+    components = new ArrayList<Component>();
+    osSpecifics = new ArrayList<OSSpecific>();
+    commandOrders = new ArrayList<CommandOrder>();
   }
 
   public String getName() {

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
index 9dba87c..f7b77ff 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
@@ -33,7 +33,7 @@ public class Component {
 
   public Component() {
     publishConfig = Boolean.FALSE.toString();
-    exports = new ArrayList<>();
+    exports = new ArrayList<Export>();
   }
 
   public String getName() {

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
index c3870b8..ffff76c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
@@ -83,7 +83,7 @@ public class SliderAMClientProvider extends AbstractClientProvider implements
    * List of roles
    */
   public static final List<ProviderRole> ROLES =
-    new ArrayList<>();
+    new ArrayList<ProviderRole>();
 
   public static final int KEY_AM = ROLE_AM_PRIORITY_INDEX;
 
@@ -147,7 +147,7 @@ public class SliderAMClientProvider extends AbstractClientProvider implements
     throws IOException, SliderException {
 
     Map<String, LocalResource> providerResources =
-        new HashMap<>();
+        new HashMap<String, LocalResource>();
 
 
     ProviderUtils.addProviderJar(providerResources,

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
index 184c25a..071fc19 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
@@ -100,7 +100,7 @@ public class SliderAMProviderService extends AbstractProviderService implements
 
   @Override
   public List<ProviderRole> getRoles() {
-    return new ArrayList<>(0);
+    return new ArrayList<ProviderRole>(0);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 7825e24..4a5c9b9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -401,7 +401,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     //look at settings of Hadoop Auth, to pick up a problem seen once
     checkAndWarnForAuthTokenProblems();
 
-    executorService = new WorkflowExecutorService<>("AmExecutor",
+    executorService = new WorkflowExecutorService<ExecutorService>("AmExecutor",
         Executors.newCachedThreadPool(
         new ServiceThreadFactory("AmExecutor", true)));
     addService(executorService);
@@ -642,7 +642,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
 
       //build the role map
       List<ProviderRole> providerRoles =
-        new ArrayList<>(providerService.getRoles());
+        new ArrayList<ProviderRole>(providerService.getRoles());
       providerRoles.addAll(SliderAMClientProvider.ROLES);
 
       // Start up the WebApp and track the URL for it
@@ -664,7 +664,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
                       .start(webApp);
       appMasterTrackingUrl = "http://" + appMasterHostname + ":" + webApp.port();
       WebAppService<SliderAMWebApp> webAppService =
-        new WebAppService<>("slider", webApp);
+        new WebAppService<SliderAMWebApp>("slider", webApp);
 
       webAppService.init(serviceConf);
       webAppService.start();
@@ -744,7 +744,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
 
       // build up environment variables that the AM wants set in every container
       // irrespective of provider and role.
-      envVars = new HashMap<>();
+      envVars = new HashMap<String, String>();
       if (hadoop_user_name != null) {
         envVars.put(HADOOP_USER_NAME, hadoop_user_name);
       }
@@ -1106,8 +1106,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
   @Override //AMRMClientAsync
   public void onContainersAllocated(List<Container> allocatedContainers) {
     LOG_YARN.info("onContainersAllocated({})", allocatedContainers.size());
-    List<ContainerAssignment> assignments = new ArrayList<>();
-    List<AbstractRMOperation> operations = new ArrayList<>();
+    List<ContainerAssignment> assignments = new ArrayList<ContainerAssignment>();
+    List<AbstractRMOperation> operations = new ArrayList<AbstractRMOperation>();
     
     //app state makes all the decisions
     appState.onContainersAllocated(allocatedContainers, assignments, operations);
@@ -1203,7 +1203,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
       log.info(
           "Scheduling the failure window reset interval to every {} seconds",
           seconds);
-      RenewingAction<ResetFailureWindow> renew = new RenewingAction<>(
+      RenewingAction<ResetFailureWindow> renew = new RenewingAction<ResetFailureWindow>(
           reset, seconds, seconds, TimeUnit.SECONDS, 0);
       actionQueues.renewing("failures", renew);
     } else {
@@ -1504,7 +1504,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
   protected synchronized void launchProviderService(AggregateConf instanceDefinition,
                                                     File confDir)
     throws IOException, SliderException {
-    Map<String, String> env = new HashMap<>();
+    Map<String, String> env = new HashMap<String, String>();
     boolean execStarted = providerService.exec(instanceDefinition, confDir, env, this);
     if (execStarted) {
       providerService.registerServiceListener(this);
@@ -1793,7 +1793,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     //turn the args to a list
     List<String> argsList = Arrays.asList(args);
     //create a new list, as the ArrayList type doesn't push() on an insert
-    List<String> extendedArgs = new ArrayList<>(argsList);
+    List<String> extendedArgs = new ArrayList<String>(argsList);
     //insert the service name
     extendedArgs.add(0, SERVICE_CLASSNAME);
     //now have the service launcher do its work

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
index 95edca4..c1e7e6e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
@@ -58,7 +58,7 @@ public class ActionKillContainer extends AsyncAction {
   public void execute(SliderAppMaster appMaster,
       QueueAccess queueService,
       AppState appState) throws Exception {
-    List<AbstractRMOperation> opsList = new LinkedList<>();
+      List<AbstractRMOperation> opsList = new LinkedList<AbstractRMOperation>();
     ContainerReleaseOperation release = new ContainerReleaseOperation(containerId);
     opsList.add(release);
     //now apply the operations

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
index ac59096..6ad579d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
@@ -57,19 +57,19 @@ implements Runnable, QueueAccess {
    * Immediate actions.
    */
   public final BlockingDeque<AsyncAction> actionQueue =
-      new LinkedBlockingDeque<>();
+      new LinkedBlockingDeque<AsyncAction>();
 
   /**
    * Actions to be scheduled in the future
    */
-  public final DelayQueue<AsyncAction> scheduledActions = new DelayQueue<>();
+  public final DelayQueue<AsyncAction> scheduledActions = new DelayQueue<AsyncAction>();
 
   /**
    * Map of renewing actions by name ... this is to allow them to 
    * be cancelled by name
    */
   private final Map<String, RenewingAction<? extends AsyncAction>> renewingActions
-      = new ConcurrentHashMap<>();
+      = new ConcurrentHashMap<String, RenewingAction<? extends AsyncAction>>();
   
   /**
    * Create a queue instance with a single thread executor

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
index cd4b46b..592889c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
@@ -74,7 +74,7 @@ public class ChaosMonkeyService extends AbstractService {
   }
 
   public RenewingAction<MonkeyPlayAction> getChaosAction(long time, TimeUnit timeUnit) {
-    RenewingAction<MonkeyPlayAction> action = new RenewingAction<>(
+    RenewingAction<MonkeyPlayAction> action = new RenewingAction<MonkeyPlayAction>(
         new MonkeyPlayAction(this, 0, TimeUnit.MILLISECONDS),
         time,
         time,

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index d22c716..0d2afcc 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -135,7 +135,7 @@ public class AppState {
    * Client properties created via the provider -static for the life
    * of the application
    */
-  private Map<String, String> clientProperties = new HashMap<>();
+  private Map<String, String> clientProperties = new HashMap<String, String>();
 
   /**
    The cluster description published to callers
@@ -146,13 +146,13 @@ public class AppState {
   private ClusterDescription clusterSpec = new ClusterDescription();
 
   private final Map<Integer, RoleStatus> roleStatusMap =
-    new ConcurrentHashMap<>();
+    new ConcurrentHashMap<Integer, RoleStatus>();
 
   private final Map<String, ProviderRole> roles =
-    new ConcurrentHashMap<>();
+    new ConcurrentHashMap<String, ProviderRole>();
 
   private final Map<Integer, ProviderRole> rolePriorityMap = 
-    new ConcurrentHashMap<>();
+    new ConcurrentHashMap<Integer, ProviderRole>();
 
   /**
    * The master node.
@@ -164,7 +164,7 @@ public class AppState {
    * been allocated but are not live; it is a superset of the live list
    */
   private final ConcurrentMap<ContainerId, RoleInstance> activeContainers =
-    new ConcurrentHashMap<>();
+    new ConcurrentHashMap<ContainerId, RoleInstance>();
 
   /**
    * Hash map of the containers we have released, but we
@@ -172,7 +172,7 @@ public class AppState {
    * containers is treated as a successful outcome
    */
   private final ConcurrentMap<ContainerId, Container> containersBeingReleased =
-    new ConcurrentHashMap<>();
+    new ConcurrentHashMap<ContainerId, Container>();
   
   /**
    * Counter for completed containers ( complete denotes successful or failed )
@@ -207,34 +207,34 @@ public class AppState {
    * the node is promoted from here to the containerMap
    */
   private final Map<ContainerId, RoleInstance> startingNodes =
-    new ConcurrentHashMap<>();
+    new ConcurrentHashMap<ContainerId, RoleInstance>();
 
   /**
    * List of completed nodes. This isn't kept in the CD as it gets too
    * big for the RPC responses. Indeed, we should think about how deep to get this
    */
   private final Map<ContainerId, RoleInstance> completedNodes
-    = new ConcurrentHashMap<>();
+    = new ConcurrentHashMap<ContainerId, RoleInstance>();
 
   /**
    * Nodes that failed to start.
    * Again, kept out of the CD
    */
   private final Map<ContainerId, RoleInstance> failedNodes =
-    new ConcurrentHashMap<>();
+    new ConcurrentHashMap<ContainerId, RoleInstance>();
 
   /**
    * Nodes that came assigned to a role above that
    * which were asked for -this appears to happen
    */
-  private final Set<ContainerId> surplusNodes = new HashSet<>();
+  private final Set<ContainerId> surplusNodes = new HashSet<ContainerId>();
 
   /**
    * Map of containerID -> cluster nodes, for status reports.
    * Access to this should be synchronized on the clusterDescription
    */
   private final Map<ContainerId, RoleInstance> liveNodes =
-    new ConcurrentHashMap<>();
+    new ConcurrentHashMap<ContainerId, RoleInstance>();
   private final AtomicInteger completionOfNodeNotInLiveListEvent =
     new AtomicInteger();
   private final AtomicInteger completionOfUnknownContainerEvent =
@@ -472,7 +472,7 @@ public class AppState {
     this.applicationInfo = applicationInfo != null ? applicationInfo
                                                    : new HashMap<String, String>();
 
-    clientProperties = new HashMap<>();
+    clientProperties = new HashMap<String, String>();
     containerReleaseSelector = releaseSelector;
 
 
@@ -794,7 +794,7 @@ public class AppState {
 
   public synchronized List<RoleInstance> cloneActiveContainerList() {
     Collection<RoleInstance> values = activeContainers.values();
-    return new ArrayList<>(values);
+    return new ArrayList<RoleInstance>(values);
   }
   
 
@@ -818,7 +818,7 @@ public class AppState {
   public synchronized List<RoleInstance> cloneLiveContainerInfoList() {
     List<RoleInstance> allRoleInstances;
     Collection<RoleInstance> values = getLiveNodes().values();
-    allRoleInstances = new ArrayList<>(values);
+    allRoleInstances = new ArrayList<RoleInstance>(values);
     return allRoleInstances;
   }
 
@@ -845,7 +845,7 @@ public class AppState {
     Collection<String> containerIDs) {
     //first, a hashmap of those containerIDs is built up
     Set<String> uuidSet = new HashSet<String>(containerIDs);
-    List<RoleInstance> nodes = new ArrayList<>(uuidSet.size());
+    List<RoleInstance> nodes = new ArrayList<RoleInstance>(uuidSet.size());
     Collection<RoleInstance> clusterNodes = getLiveNodes().values();
 
     for (RoleInstance node : clusterNodes) {
@@ -863,7 +863,7 @@ public class AppState {
    * @return a list of nodes, may be empty
    */
   public synchronized List<RoleInstance> enumLiveNodesInRole(String role) {
-    List<RoleInstance> nodes = new ArrayList<>();
+    List<RoleInstance> nodes = new ArrayList<RoleInstance>();
     Collection<RoleInstance> allRoleInstances = getLiveNodes().values();
     for (RoleInstance node : allRoleInstances) {
       if (role.isEmpty() || role.equals(node.role)) {
@@ -883,7 +883,7 @@ public class AppState {
    */
   public synchronized List<RoleInstance> enumNodesWithRoleId(int roleId,
       boolean active) {
-    List<RoleInstance> nodes = new ArrayList<>();
+    List<RoleInstance> nodes = new ArrayList<RoleInstance>();
     Collection<RoleInstance> allRoleInstances;
     allRoleInstances = active? activeContainers.values() : liveNodes.values();
     for (RoleInstance node : allRoleInstances) {
@@ -900,11 +900,11 @@ public class AppState {
    * @return the map of Role name to list of role instances
    */
   private synchronized Map<String, List<String>> createRoleToInstanceMap() {
-    Map<String, List<String>> map = new HashMap<>();
+    Map<String, List<String>> map = new HashMap<String, List<String>>();
     for (RoleInstance node : getLiveNodes().values()) {
       List<String> containers = map.get(node.role);
       if (containers == null) {
-        containers = new ArrayList<>();
+        containers = new ArrayList<String>();
         map.put(node.role, containers);
       }
       containers.add(node.id);
@@ -916,12 +916,12 @@ public class AppState {
    * @return the map of Role name to list of Cluster Nodes, ready
    */
   private synchronized Map<String, Map<String, ClusterNode>> createRoleToClusterNodeMap() {
-    Map<String, Map<String, ClusterNode>> map = new HashMap<>();
+    Map<String, Map<String, ClusterNode>> map = new HashMap<String, Map<String, ClusterNode>>();
     for (RoleInstance node : getLiveNodes().values()) {
       
       Map<String, ClusterNode> containers = map.get(node.role);
       if (containers == null) {
-        containers = new HashMap<>();
+        containers = new HashMap<String, ClusterNode>();
         map.put(node.role, containers);
       }
       Messages.RoleInstanceState pbuf = node.toProtobuf();
@@ -1401,7 +1401,7 @@ public class AppState {
     MapOperations infoOps = new MapOperations("info", cd.info);
     infoOps.mergeWithoutOverwrite(applicationInfo);
     SliderUtils.addBuildInfo(infoOps, "status");
-    cd.statistics = new HashMap<>();
+    cd.statistics = new HashMap<String, Map<String, Integer>>();
 
     // build the map of node -> container IDs
     Map<String, List<String>> instanceMap = createRoleToInstanceMap();
@@ -1410,7 +1410,7 @@ public class AppState {
     //build the map of node -> containers
     Map<String, Map<String, ClusterNode>> clusterNodes =
       createRoleToClusterNodeMap();
-    cd.status = new HashMap<>();
+    cd.status = new HashMap<String, Object>();
     cd.status.put(ClusterDescriptionKeys.KEY_CLUSTER_LIVE, clusterNodes);
 
 
@@ -1429,7 +1429,7 @@ public class AppState {
       cd.statistics.put(rolename, stats);
     }
 
-    Map<String, Integer> sliderstats = new HashMap<>();
+    Map<String, Integer> sliderstats = new HashMap<String, Integer>();
     sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_COMPLETED,
         completedContainerCount.get());
     sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED,
@@ -1453,7 +1453,7 @@ public class AppState {
   public synchronized List<AbstractRMOperation> reviewRequestAndReleaseNodes()
       throws SliderInternalStateException, TriggerClusterTeardownException {
     log.debug("in reviewRequestAndReleaseNodes()");
-    List<AbstractRMOperation> allOperations = new ArrayList<>();
+    List<AbstractRMOperation> allOperations = new ArrayList<AbstractRMOperation>();
     for (RoleStatus roleStatus : getRoleStatusMap().values()) {
       if (!roleStatus.getExcludeFromFlexing()) {
         List<AbstractRMOperation> operations = reviewOneRole(roleStatus);
@@ -1527,7 +1527,7 @@ public class AppState {
   @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
   private List<AbstractRMOperation> reviewOneRole(RoleStatus role)
       throws SliderInternalStateException, TriggerClusterTeardownException {
-    List<AbstractRMOperation> operations = new ArrayList<>();
+    List<AbstractRMOperation> operations = new ArrayList<AbstractRMOperation>();
     int delta;
     String details;
     int expected;
@@ -1622,7 +1622,7 @@ public class AppState {
    */
   public List<AbstractRMOperation> releaseContainer(ContainerId containerId)
       throws SliderInternalStateException {
-    List<AbstractRMOperation> operations = new ArrayList<>();
+    List<AbstractRMOperation> operations = new ArrayList<AbstractRMOperation>();
     List<RoleInstance> activeRoleInstances = cloneActiveContainerList();
     for (RoleInstance role : activeRoleInstances) {
       if (role.container.getId().equals(containerId)) {
@@ -1666,7 +1666,7 @@ public class AppState {
     Collection<RoleInstance> targets = cloneActiveContainerList();
     log.info("Releasing {} containers", targets.size());
     List<AbstractRMOperation> operations =
-      new ArrayList<>(targets.size());
+      new ArrayList<AbstractRMOperation>(targets.size());
     for (RoleInstance instance : targets) {
       Container possible = instance.container;
       ContainerId id = possible.getId();

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
index 9d936a1..841dda3 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
@@ -40,7 +40,7 @@ public class MostRecentContainerReleaseSelector implements ContainerReleaseSelec
 
   private static class newerThan implements Comparator<RoleInstance>, Serializable {
     private final Comparator<Long> innerComparator =
-        new Comparators.ComparatorReverser<>(new Comparators.LongComparator());
+        new Comparators.ComparatorReverser<Long>(new Comparators.LongComparator());
     public int compare(RoleInstance o1, RoleInstance o2) {
       return innerComparator.compare(o1.createTime, o2.createTime);
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
index fad60f2..1ba2282 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
@@ -41,7 +41,7 @@ public class NodeInstance {
    */
   public NodeInstance(String hostname, int roles) {
     this.hostname = hostname;
-    nodeEntries = new ArrayList<>(roles);
+    nodeEntries = new ArrayList<NodeEntry>(roles);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
index 372a0a5..570c194 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
@@ -73,7 +73,7 @@ public class NodeMap extends HashMap<String, NodeInstance> {
    * in that role
    */
   public List<NodeInstance> listActiveNodes(int role) {
-    List<NodeInstance> nodes = new ArrayList<>();
+    List<NodeInstance> nodes = new ArrayList<NodeInstance>();
     for (NodeInstance instance : values()) {
       if (instance.getActiveRoleInstances(role) > 0) {
         nodes.add(instance);

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
index fa2c754..d847962 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
@@ -43,7 +43,7 @@ public class OutstandingRequestTracker {
     LoggerFactory.getLogger(OutstandingRequestTracker.class);
 
   private Map<OutstandingRequest, OutstandingRequest> requests =
-    new HashMap<>();
+    new HashMap<OutstandingRequest, OutstandingRequest>();
 
   /**
    * Create a new request for the specific role. If a
@@ -161,7 +161,7 @@ public class OutstandingRequestTracker {
    * @return possibly empty list of hostnames
    */
   public synchronized List<NodeInstance> cancelOutstandingRequests(int role) {
-    List<NodeInstance> hosts = new ArrayList<>();
+    List<NodeInstance> hosts = new ArrayList<NodeInstance>();
     Iterator<Map.Entry<OutstandingRequest,OutstandingRequest>> iterator =
       requests.entrySet().iterator();
     while (iterator.hasNext()) {
@@ -178,6 +178,6 @@ public class OutstandingRequestTracker {
   }
   
   public synchronized List<OutstandingRequest> listOutstandingRequests() {
-    return new ArrayList<>(requests.values());
+    return new ArrayList<OutstandingRequest>(requests.values());
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
index 6caf1a9..a5b36d4 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
@@ -39,7 +39,7 @@ public class ProviderAppState implements StateAccessForProviders {
 
 
   private final Map<String, PublishedConfigSet> publishedConfigSets =
-      new ConcurrentHashMap<>(5);
+      new ConcurrentHashMap<String, PublishedConfigSet>(5);
   private static final PatternValidator validator = new PatternValidator(
       RestPaths.PUBLISHED_CONFIGURATION_SET_REGEXP);
   private String applicationName;
@@ -92,7 +92,7 @@ public class ProviderAppState implements StateAccessForProviders {
   public List<String> listConfigSets() {
 
     synchronized (publishedConfigSets) {
-      List<String> sets = new ArrayList<>(publishedConfigSets.keySet());
+      List<String> sets = new ArrayList<String>(publishedConfigSets.keySet());
       return sets;
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 2338103..edcf7ea 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -60,7 +60,7 @@ public class RoleHistory {
     LoggerFactory.getLogger(RoleHistory.class);
   private final List<ProviderRole> providerRoles;
   private final Map<String, ProviderRole> providerRoleMap =
-    new HashMap<>();
+    new HashMap<String, ProviderRole>();
   private long startTime;
   /**
    * Time when saved
@@ -107,9 +107,8 @@ public class RoleHistory {
     nodemap = new NodeMap(roleSize);
     resetAvailableNodeLists();
 
-    resetAvailableNodeLists();
     outstandingRequests = new OutstandingRequestTracker();
-    Map<Integer, RoleStatus> roleStats = new HashMap<>();
+    Map<Integer, RoleStatus> roleStats = new HashMap<Integer, RoleStatus>();
 
 
     for (ProviderRole providerRole : providerRoles) {
@@ -141,7 +140,7 @@ public class RoleHistory {
    */
   public void addNewProviderRole(ProviderRole providerRole)
     throws BadConfigException {
-    Map<Integer, RoleStatus> roleStats = new HashMap<>();
+    Map<Integer, RoleStatus> roleStats = new HashMap<Integer, RoleStatus>();
 
 
     for (ProviderRole role : providerRoles) {
@@ -155,7 +154,7 @@ public class RoleHistory {
    * Clear the lists of available nodes
    */
   private synchronized void resetAvailableNodeLists() {
-    availableNodes = new HashMap<>(roleSize);
+    availableNodes = new HashMap<Integer, LinkedList<NodeInstance>>(roleSize);
   }
 
   /**
@@ -433,7 +432,7 @@ public class RoleHistory {
   private LinkedList<NodeInstance> getOrCreateNodesForRoleId(int id) {
     LinkedList<NodeInstance> instances = availableNodes.get(id);
     if (instances == null) {
-      instances = new LinkedList<>();
+      instances = new LinkedList<NodeInstance>();
       availableNodes.put(id, instances);
     }
     return instances;
@@ -569,7 +568,7 @@ public class RoleHistory {
     List<Container> requested =
       new ArrayList<Container>(allocatedContainers.size());
     List<Container> unrequested =
-      new ArrayList<>(allocatedContainers.size());
+      new ArrayList<Container>(allocatedContainers.size());
     outstandingRequests.partitionRequests(this, allocatedContainers, requested, unrequested);
     
     //give the unrequested ones lower priority
@@ -748,7 +747,7 @@ public class RoleHistory {
    */
   @VisibleForTesting
   public List<NodeInstance> cloneAvailableList(int role) {
-    return new LinkedList<>(getOrCreateNodesForRoleId(role));
+    return new LinkedList<NodeInstance>(getOrCreateNodesForRoleId(role));
   }
 
   /**


[25/50] [abbrv] git commit: SLIDER-262. Slider agent should provide process supervision such as auto-restart. Minor bug fix.

Posted by st...@apache.org.
SLIDER-262. Slider agent should provide process supervision such as auto-restart. Minor bug fix.


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/18b05450
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/18b05450
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/18b05450

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 18b05450e886db00fdf38acb83d2b419d149493d
Parents: 806ebb2
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Aug 13 12:48:01 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Aug 14 17:57:43 2014 -0700

----------------------------------------------------------------------
 .../providers/agent/AgentProviderService.java    |  2 +-
 .../application/metadata/MetainfoParser.java     |  2 +-
 .../agent/TestAgentProviderService.java          | 19 ++++++++++---------
 3 files changed, 12 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/18b05450/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index a40926a..b22ef39 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -1131,7 +1131,7 @@ public class AgentProviderService extends AbstractProviderService implements
 
   @VisibleForTesting
   protected void addStartCommand(String roleName, String containerId, HeartBeatResponse response,
-                                 String scriptPath, Boolean isMarkedAutoRestart)
+                                 String scriptPath, boolean isMarkedAutoRestart)
       throws
       SliderException {
     assert getAmState().isApplicationLive();

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/18b05450/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
index 23134b6..ada5b83 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
@@ -39,7 +39,6 @@ public class MetainfoParser {
     digester.addBeanPropertySetter("*/application/comment");
     digester.addBeanPropertySetter("*/application/version");
     digester.addBeanPropertySetter("*/application/exportedConfigs");
-    digester.addBeanPropertySetter("*/application/autoStartOnFailure");
 
     digester.addObjectCreate("*/commandOrder", CommandOrder.class);
     digester.addBeanPropertySetter("*/commandOrder/command");
@@ -60,6 +59,7 @@ public class MetainfoParser {
     digester.addBeanPropertySetter("*/component/publishConfig");
     digester.addBeanPropertySetter("*/component/minInstanceCount");
     digester.addBeanPropertySetter("*/component/maxInstanceCount");
+    digester.addBeanPropertySetter("*/component/autoStartOnFailure");
     digester.addObjectCreate("*/component/export", Export.class);
     digester.addBeanPropertySetter("*/component/export/name");
     digester.addBeanPropertySetter("*/component/export/value");

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/18b05450/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 463799c..1aeedbc 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -63,6 +63,7 @@ import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus;
 import org.junit.Assert;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
+import org.mockito.Matchers;
 import org.mockito.Mockito;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -566,11 +567,11 @@ public class TestAgentProviderService {
     String role_hm = "HBASE_MASTER";
     String role_hrs = "HBASE_REGIONSERVER";
 
-    AgentProviderService aps = new AgentProviderService();
-    AgentProviderService mockAps = Mockito.spy(aps);
+    AgentProviderService aps1 = new AgentProviderService();
+    AgentProviderService mockAps = Mockito.spy(aps1);
     doReturn(metainfo).when(mockAps).getMetainfo();
 
-    AgentProviderService mockAps2 = Mockito.spy(aps);
+    AgentProviderService mockAps2 = Mockito.spy(aps1);
     doReturn(metainfo2).when(mockAps2).getMetainfo();
 
     Assert.assertTrue(mockAps.isMaster(role_hm));
@@ -644,7 +645,7 @@ public class TestAgentProviderService {
           anyString(),
           any(HeartBeatResponse.class),
           anyString(),
-          anyBoolean());
+          Matchers.anyBoolean());
       doNothing().when(mockAps).addGetConfigCommand(
           anyString(),
           anyString(),
@@ -747,7 +748,7 @@ public class TestAgentProviderService {
                                                                 anyString(),
                                                                 any(HeartBeatResponse.class),
                                                                 anyString(),
-                                                                anyBoolean());
+                                                                Matchers.anyBoolean());
       // RS still does not start
       hb = new HeartBeat();
       hb.setResponseId(3);
@@ -758,7 +759,7 @@ public class TestAgentProviderService {
                                                                 anyString(),
                                                                 any(HeartBeatResponse.class),
                                                                 anyString(),
-                                                                anyBoolean());
+                                                                Matchers.anyBoolean());
 
       // MASTER succeeds install and issues start
       hb = new HeartBeat();
@@ -778,7 +779,7 @@ public class TestAgentProviderService {
                                                                 anyString(),
                                                                 any(HeartBeatResponse.class),
                                                                 anyString(),
-                                                                anyBoolean());
+                                                                Matchers.anyBoolean());
       Map<String, String> allocatedPorts = mockAps.getAllocatedPorts();
       Assert.assertTrue(allocatedPorts != null);
       Assert.assertTrue(allocatedPorts.size() == 1);
@@ -794,7 +795,7 @@ public class TestAgentProviderService {
                                                                 anyString(),
                                                                 any(HeartBeatResponse.class),
                                                                 anyString(),
-                                                                anyBoolean());
+                                                                Matchers.anyBoolean());
       // MASTER succeeds start
       hb = new HeartBeat();
       hb.setResponseId(3);
@@ -819,7 +820,7 @@ public class TestAgentProviderService {
                                                                 anyString(),
                                                                 any(HeartBeatResponse.class),
                                                                 anyString(),
-                                                                anyBoolean());
+                                                                Matchers.anyBoolean());
     // JDK7 
     } catch (SliderException he) {
       log.warn(he.getMessage());


[23/50] [abbrv] git commit: SLIDER-262. Slider agent should provide process supervision such as auto-restart

Posted by st...@apache.org.
SLIDER-262. Slider agent should provide process supervision such as auto-restart


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/506eb79e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/506eb79e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/506eb79e

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 506eb79e35ff6187648f4d874c22bb3fe3b18e58
Parents: 004fc8d
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Aug 12 18:38:15 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Aug 14 17:56:52 2014 -0700

----------------------------------------------------------------------
 .../src/main/python/agent/ActionQueue.py        | 22 +++++++---
 slider-agent/src/main/python/agent/Constants.py |  1 +
 .../src/main/python/agent/Controller.py         | 25 ++++++++++-
 .../python/agent/CustomServiceOrchestrator.py   | 44 +++++++++++---------
 slider-agent/src/main/python/agent/Heartbeat.py |  6 ++-
 .../src/test/python/agent/TestActionQueue.py    |  5 ++-
 .../agent/TestCustomServiceOrchestrator.py      |  8 ++--
 .../src/test/python/agent/TestHeartbeat.py      | 25 +++++++++--
 .../slider/common/tools/CoreFileSystem.java     |  3 ++
 .../providers/agent/AgentProviderService.java   | 35 +++++++++++++++-
 .../web/rest/agent/HeartBeatResponse.java       | 11 +++++
 .../agent/TestAgentProviderService.java         | 21 ++++++----
 12 files changed, 162 insertions(+), 44 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-agent/src/main/python/agent/ActionQueue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/ActionQueue.py b/slider-agent/src/main/python/agent/ActionQueue.py
index d4d8bc2..774d1e6 100644
--- a/slider-agent/src/main/python/agent/ActionQueue.py
+++ b/slider-agent/src/main/python/agent/ActionQueue.py
@@ -49,6 +49,7 @@ class ActionQueue(threading.Thread):
   FAILED_STATUS = 'FAILED'
 
   STORE_APPLIED_CONFIG = 'record_config'
+  AUTO_RESTART = 'auto_restart'
 
   def __init__(self, config, controller):
     super(ActionQueue, self).__init__()
@@ -120,6 +121,9 @@ class ActionQueue(threading.Thread):
     logger.debug(pprint.pformat(command))
 
     taskId = command['taskId']
+
+    reportResult = not command[Constants.AUTO_GENERATED]
+
     # Preparing 'IN_PROGRESS' report
     in_progress_status = self.commandStatuses.generate_report_template(command)
     in_progress_status.update({
@@ -127,12 +131,17 @@ class ActionQueue(threading.Thread):
       'tmperr': self.tmpdir + os.sep + 'errors-' + str(taskId) + '.txt',
       'structuredOut': self.tmpdir + os.sep + 'structured-out-' + str(
         taskId) + '.json',
-      'status': self.IN_PROGRESS_STATUS
+      'status': self.IN_PROGRESS_STATUS,
+      'reportResult': reportResult
     })
-    self.commandStatuses.put_command_status(command, in_progress_status)
+    self.commandStatuses.put_command_status(command, in_progress_status, reportResult)
     store_config = False
     if ActionQueue.STORE_APPLIED_CONFIG in command['commandParams']:
       store_config = 'true' == command['commandParams'][ActionQueue.STORE_APPLIED_CONFIG]
+    store_command = False
+    if ActionQueue.AUTO_RESTART in command['roleParams']:
+      store_command = 'true' == command['roleParams'][ActionQueue.AUTO_RESTART]
+
 
     # running command
     commandresult = self.customServiceOrchestrator.runCommand(command,
@@ -141,7 +150,7 @@ class ActionQueue(threading.Thread):
                                                               in_progress_status[
                                                                 'tmperr'],
                                                               True,
-                                                              store_config)
+                                                              store_config or store_command)
     # dumping results
     status = self.COMPLETED_STATUS
     if commandresult[Constants.EXIT_CODE] != 0:
@@ -152,6 +161,7 @@ class ActionQueue(threading.Thread):
       'stderr': commandresult['stderr'],
       Constants.EXIT_CODE: commandresult[Constants.EXIT_CODE],
       'status': status,
+      'reportResult': reportResult
     })
     if roleResult['stdout'] == '':
       roleResult['stdout'] = 'None'
@@ -170,7 +180,7 @@ class ActionQueue(threading.Thread):
         roleResult['allocatedPorts'] = commandresult[Constants.ALLOCATED_PORTS]
       if Constants.FOLDERS in commandresult:
         roleResult['folders'] = commandresult[Constants.FOLDERS]
-    self.commandStatuses.put_command_status(command, roleResult)
+    self.commandStatuses.put_command_status(command, roleResult, reportResult)
 
   # Store action result to agent response queue
   def result(self):
@@ -185,8 +195,8 @@ class ActionQueue(threading.Thread):
       service = command['serviceName']
       component = command['componentName']
       reportResult = True
-      if 'auto_generated' in command:
-        reportResult = not command['auto_generated']
+      if Constants.AUTO_GENERATED in command:
+        reportResult = not command[Constants.AUTO_GENERATED]
 
       component_status = self.customServiceOrchestrator.requestComponentStatus(command)
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-agent/src/main/python/agent/Constants.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Constants.py b/slider-agent/src/main/python/agent/Constants.py
index a120999..2975266 100644
--- a/slider-agent/src/main/python/agent/Constants.py
+++ b/slider-agent/src/main/python/agent/Constants.py
@@ -32,3 +32,4 @@ DO_NOT_HEARTBEAT = "DO_NOT_HEARTBEAT"
 DO_NOT_HEARTBEAT_AFTER_ = "DO_NOT_HEARTBEAT_AFTER_"
 ZK_QUORUM="zk_quorum"
 ZK_REG_PATH="zk_reg_path"
+AUTO_GENERATED="auto_generated"

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-agent/src/main/python/agent/Controller.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Controller.py b/slider-agent/src/main/python/agent/Controller.py
index 1a01bf8..77dcafb 100644
--- a/slider-agent/src/main/python/agent/Controller.py
+++ b/slider-agent/src/main/python/agent/Controller.py
@@ -86,6 +86,7 @@ class Controller(threading.Thread):
     self.statusCommand = None
     self.failureCount = 0
     self.heartBeatRetryCount = 0
+    self.autoRestart = False
 
 
   def __del__(self):
@@ -224,6 +225,9 @@ class Controller(threading.Thread):
 
         serverId = int(response['responseId'])
 
+        if 'restartEnabled' in response:
+          restartEnabled = 'true' == response['restartEnabled']
+
         if 'hasMappedComponents' in response.keys():
           self.hasMappedComponents = response['hasMappedComponents'] != False
 
@@ -257,6 +261,17 @@ class Controller(threading.Thread):
           logger.info("No commands sent from the Server.")
           pass
 
+        # Add a start command
+        if self.componentActualState == State.INSTALLED and \
+                self.componentExpectedState == State.STARTED and restartEnabled:
+          stored_command = self.actionQueue.customServiceOrchestrator.stored_command
+          if len(stored_command) > 0:
+             auto_start_command = self.create_start_command(stored_command)
+             if auto_start_command:
+               self.updateStateBasedOnCommand([auto_start_command])
+               self.addToQueue([auto_start_command])
+          pass
+
         # Add a status command
         if (self.componentActualState != State.STARTING and \
                 self.componentExpectedState == State.STARTED) and \
@@ -328,6 +343,14 @@ class Controller(threading.Thread):
     pass
     logger.info("Controller stopped heart-beating.")
 
+  def create_start_command(self, stored_command):
+    taskId = int(stored_command['taskId'])
+    taskId = taskId + 1
+    stored_command['taskId'] = taskId
+    stored_command['commandId'] = "{0}-1".format(taskId)
+    stored_command[Constants.AUTO_GENERATED] = True
+    pass
+
   def updateStateBasedOnCommand(self, commands):
     for command in commands:
       if command["roleCommand"] == "START":
@@ -388,7 +411,7 @@ class Controller(threading.Thread):
     statusCommand["hostLevelParams"] = command["hostLevelParams"]
     statusCommand["serviceName"] = command["serviceName"]
     statusCommand["taskId"] = "status"
-    statusCommand['auto_generated'] = True
+    statusCommand[Constants.AUTO_GENERATED] = True
     logger.info("Status command: " + pprint.pformat(statusCommand))
     return statusCommand
     pass

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 0078c9c..6b2ace5 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -56,7 +56,7 @@ class CustomServiceOrchestrator():
     self.status_commands_stderr = os.path.realpath(posixpath.join(self.tmp_dir,
                                                                   'status_command_stderr.txt'))
     self.public_fqdn = hostname.public_hostname()
-    self.applied_configs = {}
+    self.stored_command = {}
     # Clean up old status command files if any
     try:
       os.unlink(self.status_commands_stdout)
@@ -68,7 +68,7 @@ class CustomServiceOrchestrator():
 
 
   def runCommand(self, command, tmpoutfile, tmperrfile,
-                 override_output_files=True, store_config=False):
+                 override_output_files=True, store_command=False):
     allocated_port = {}
     try:
       script_type = command['commandParams']['script_type']
@@ -86,7 +86,7 @@ class CustomServiceOrchestrator():
       # We don't support anything else yet
         message = "Unknown script type {0}".format(script_type)
         raise AgentException(message)
-      json_path = self.dump_command_to_json(command, allocated_port, store_config)
+      json_path = self.dump_command_to_json(command, allocated_port, store_command)
       py_file_list = [script_tuple]
       # filter None values
       filtered_py_file_list = [i for i in py_file_list if i]
@@ -154,22 +154,28 @@ class CustomServiceOrchestrator():
     return path
 
   def getConfig(self, command):
-    if 'commandParams' in command and 'config_type' in command['commandParams']:
-      config_type = command['commandParams']['config_type']
-      logger.info("Requesting applied config for type {0}".format(config_type))
-      if config_type in self.applied_configs:
-        return {
-          'configurations': {config_type: self.applied_configs[config_type]}
-        }
+    if 'configurations' in self.stored_command:
+      if 'commandParams' in command and 'config_type' in command['commandParams']:
+        config_type = command['commandParams']['config_type']
+        logger.info("Requesting applied config for type {0}".format(config_type))
+        if config_type in self.stored_command['configurations']:
+          return {
+            'configurations': {config_type: self.stored_command['configurations'][config_type]}
+          }
+        else:
+          return {
+            'configurations': {}
+          }
+        pass
       else:
+        logger.info("Requesting all applied config.")
         return {
-          'configurations': {}
+          'configurations': self.stored_command['configurations']
         }
       pass
     else:
-      logger.info("Requesting all applied config.")
       return {
-        'configurations': self.applied_configs
+        'configurations': {}
       }
     pass
 
@@ -184,7 +190,7 @@ class CustomServiceOrchestrator():
       override_output_files = False
 
     if command['roleCommand'] == "GET_CONFIG":
-      return self.getConfig(command)
+     return self.getConfig(command)
 
     else:
       res = self.runCommand(command, self.status_commands_stdout,
@@ -198,7 +204,7 @@ class CustomServiceOrchestrator():
       return res
     pass
 
-  def dump_command_to_json(self, command, allocated_ports, store_config=False):
+  def dump_command_to_json(self, command, allocated_ports, store_command=False):
     """
     Converts command to json file and returns file path
     """
@@ -221,7 +227,7 @@ class CustomServiceOrchestrator():
     if os.path.isfile(file_path):
       os.unlink(file_path)
 
-    self.finalize_command(command, store_config, allocated_ports)
+    self.finalize_command(command, store_command, allocated_ports)
 
     with os.fdopen(os.open(file_path, os.O_WRONLY | os.O_CREAT,
                            0600), 'w') as f:
@@ -238,7 +244,7 @@ class CustomServiceOrchestrator():
   Either a port gets allocated or if not then just set the value to "0"
   """
 
-  def finalize_command(self, command, store_config, allocated_ports):
+  def finalize_command(self, command, store_command, allocated_ports):
     component = command['componentName']
     allocated_for_this_component_format = "${{{0}.ALLOCATED_PORT}}"
     allocated_for_any = ".ALLOCATED_PORT}"
@@ -267,9 +273,9 @@ class CustomServiceOrchestrator():
         pass
       pass
 
-    if store_config:
+    if store_command:
       logger.info("Storing applied config: " + pprint.pformat(command['configurations']))
-      self.applied_configs = command['configurations']
+      self.stored_command = command
 
   pass
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-agent/src/main/python/agent/Heartbeat.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Heartbeat.py b/slider-agent/src/main/python/agent/Heartbeat.py
index 4f2207c..aa403d4 100644
--- a/slider-agent/src/main/python/agent/Heartbeat.py
+++ b/slider-agent/src/main/python/agent/Heartbeat.py
@@ -57,7 +57,11 @@ class Heartbeat:
     if not self.actionQueue.commandQueue.empty():
       commandsInProgress = True
     if len(queueResult) != 0:
-      heartbeat['reports'] = queueResult['reports']
+      heartbeat['reports'] = []
+      for report in queueResult['reports']:
+        if report['reportResult']:
+          del report['reportResult']
+          heartbeat['reports'].append(report)
       if len(heartbeat['reports']) > 0:
         # There may be IN_PROGRESS tasks
         commandsInProgress = True

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-agent/src/test/python/agent/TestActionQueue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestActionQueue.py b/slider-agent/src/test/python/agent/TestActionQueue.py
index 21cc5dd..f0935a9 100644
--- a/slider-agent/src/test/python/agent/TestActionQueue.py
+++ b/slider-agent/src/test/python/agent/TestActionQueue.py
@@ -61,6 +61,8 @@ class TestActionQueue(TestCase):
     'serviceName': u'HBASE',
     'configurations': {'global': {}},
     'configurationTags': {'global': {'tag': 'v1'}},
+    'auto_generated': False,
+    'roleParams': {'auto_restart':'false'},
     'commandParams': {'script_type': 'PYTHON',
                       'script': 'scripts/abc.py',
                       'command_timeout': '600'}
@@ -365,7 +367,8 @@ class TestActionQueue(TestCase):
                 'structuredOut': '',
                 'exitcode': 0,
                 'allocatedPorts': {},
-                'folders': {'AGENT_LOG_ROOT': tempdir, 'AGENT_WORK_ROOT': tempdir}}
+                'folders': {'AGENT_LOG_ROOT': tempdir, 'AGENT_WORK_ROOT': tempdir},
+                'reportResult': True}
     self.assertEqual(len(report['reports']), 1)
     self.assertEqual(report['reports'][0], expected)
     self.assertTrue(os.path.isfile(configname))

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index 48e4df4..6408809 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -38,6 +38,7 @@ from socket import socket
 
 
 class TestCustomServiceOrchestrator(TestCase):
+
   def setUp(self):
     # disable stdout
     out = StringIO.StringIO()
@@ -415,9 +416,10 @@ class TestCustomServiceOrchestrator(TestCase):
     }
 
     ret = orchestrator.runCommand(command, "out.txt", "err.txt", True, True)
+    self.assertEqual.__self__.maxDiff = None
     self.assertEqual(ret['exitcode'], 0)
     self.assertTrue(run_file_mock.called)
-    self.assertEqual(orchestrator.applied_configs, expected)
+    self.assertEqual(orchestrator.stored_command, command)
 
     ret = orchestrator.requestComponentStatus(command_get)
     self.assertEqual(ret['configurations'], expected)
@@ -498,7 +500,7 @@ class TestCustomServiceOrchestrator(TestCase):
     self.assertEqual(command['configurations']['oozie-site']['ignore_port2'], "[0,0]")
     self.assertEqual(command['configurations']['oozie-site']['ignore_port3'], "[0,0,0]")
     self.assertEqual(command['configurations']['oozie-site']['ignore_port4'], "${HBASE_RS}{a}{b}{c}")
-    self.assertEqual(orchestrator.applied_configs, {})
+    self.assertEqual(orchestrator.stored_command, {})
     self.assertEqual(len(allocated_ports), 1)
     self.assertTrue('oozie-site.a_port' in allocated_ports)
     self.assertEqual(allocated_ports['oozie-site.a_port'], '10023')
@@ -513,7 +515,7 @@ class TestCustomServiceOrchestrator(TestCase):
     self.assertEqual(command['configurations']['hbase-site']['log_root'], tempdir + "/log")
     self.assertEqual(command['configurations']['hbase-site']['blog_root'], "/b/" + tempdir + "/log")
     self.assertEqual(command['configurations']['oozie-site']['b_port'], "0")
-    self.assertEqual(orchestrator.applied_configs, command['configurations'])
+    self.assertEqual(orchestrator.stored_command, command)
 
 
   def test_port_allocation(self):

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-agent/src/test/python/agent/TestHeartbeat.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestHeartbeat.py b/slider-agent/src/test/python/agent/TestHeartbeat.py
index a9da65d..70870ec 100644
--- a/slider-agent/src/test/python/agent/TestHeartbeat.py
+++ b/slider-agent/src/test/python/agent/TestHeartbeat.py
@@ -86,7 +86,8 @@ class TestHeartbeat(TestCase):
                    'role': u'DATANODE',
                    'actionId': '1-1',
                    'taskId': 3,
-                   'exitcode': 777},
+                   'exitcode': 777,
+                   'reportResult' : True},
 
                   {'status': 'COMPLETED',
                    'stderr': 'stderr',
@@ -97,7 +98,8 @@ class TestHeartbeat(TestCase):
                    'role': 'role',
                    'actionId': 17,
                    'taskId': 'taskId',
-                   'exitcode': 0},
+                   'exitcode': 0,
+                   'reportResult' : True},
 
                   {'status': 'FAILED',
                    'stderr': 'stderr',
@@ -108,7 +110,8 @@ class TestHeartbeat(TestCase):
                    'role': u'DATANODE',
                    'actionId': '1-1',
                    'taskId': 3,
-                   'exitcode': 13},
+                   'exitcode': 13,
+                   'reportResult' : True},
 
                   {'status': 'COMPLETED',
                    'stderr': 'stderr',
@@ -120,8 +123,21 @@ class TestHeartbeat(TestCase):
                    'role': u'DATANODE',
                    'actionId': '1-1',
                    'taskId': 3,
-                   'exitcode': 0}
+                   'exitcode': 0,
+                   'reportResult' : True},
 
+                  {'status': 'COMPLETED',
+                   'stderr': 'stderr',
+                   'stdout': 'out',
+                   'clusterName': u'cc',
+                   'configurationTags': {'global': {'tag': 'v1'}},
+                   'roleCommand': u'INSTALL',
+                   'serviceName': u'HDFS',
+                   'role': u'DATANODE',
+                   'actionId': '1-1',
+                   'taskId': 3,
+                   'exitcode': 0,
+                   'reportResult' : False}
       ],
       'componentStatus': [
         {'status': 'HEALTHY', 'componentName': 'DATANODE', 'reportResult' : True},
@@ -159,6 +175,7 @@ class TestHeartbeat(TestCase):
        'stderr': 'stderr'}],  'componentStatus': [
       {'status': 'HEALTHY', 'componentName': 'DATANODE'},
       {'status': 'UNHEALTHY', 'componentName': 'NAMENODE'}]}
+    self.assertEqual.__self__.maxDiff = None
     self.assertEquals(hb, expected)
 
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index a145b8f..def252a 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -50,6 +50,9 @@ import java.io.IOException;
 import java.nio.charset.Charset;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Enumeration;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
 
 import static org.apache.slider.common.SliderXmlConfKeys.CLUSTER_DIRECTORY_PERMISSIONS;
 import static org.apache.slider.common.SliderXmlConfKeys.DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS;

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 559c4e7..a40926a 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -495,7 +495,7 @@ public class AgentProviderService extends AbstractProviderService implements
           boolean canExecute = commandOrder.canExecute(roleName, command, getComponentStatuses().values());
           if (canExecute) {
             log.info("Starting {} on {}.", roleName, containerId);
-            addStartCommand(roleName, containerId, response, scriptPath);
+            addStartCommand(roleName, containerId, response, scriptPath, isMarkedAutoRestart(roleName));
             componentStatus.commandIssued(command);
           } else {
             log.info("Start of {} on {} delayed as dependencies have not started.", roleName, containerId);
@@ -509,6 +509,13 @@ public class AgentProviderService extends AbstractProviderService implements
           addGetConfigCommand(roleName, containerId, response);
         }
       }
+
+      // if restart is required then signal
+      response.setRestartEnabled(false);
+      if (componentStatus.getState() == State.STARTED
+          && command == Command.NOP && isMarkedAutoRestart(roleName)) {
+        response.setRestartEnabled(true);
+      }
     } catch (SliderException e) {
       componentStatus.applyCommandResult(CommandResult.FAILED, command);
       log.warn("Component instance failed operation.", e);
@@ -936,6 +943,25 @@ public class AgentProviderService extends AbstractProviderService implements
   }
 
   /**
+   * Checks if the role is marked auto-restart
+   * @param roleName
+   * @return
+   */
+  protected boolean isMarkedAutoRestart(String roleName) {
+    Application application = getMetainfo().getApplication();
+    if (application == null) {
+      log.error("Malformed app definition: Expect application as the top level element for metainfo.xml");
+    } else {
+      for (Component component : application.getComponents()) {
+        if (component.getName().equals(roleName)) {
+          return component.getRequiresAutoRestart();
+        }
+      }
+    }
+    return false;
+  }
+
+  /**
    * Can any master publish config explicitly, if not a random master is used
    * @return
    */
@@ -1104,7 +1130,8 @@ public class AgentProviderService extends AbstractProviderService implements
   }
 
   @VisibleForTesting
-  protected void addStartCommand(String roleName, String containerId, HeartBeatResponse response, String scriptPath)
+  protected void addStartCommand(String roleName, String containerId, HeartBeatResponse response,
+                                 String scriptPath, Boolean isMarkedAutoRestart)
       throws
       SliderException {
     assert getAmState().isApplicationLive();
@@ -1126,6 +1153,10 @@ public class AgentProviderService extends AbstractProviderService implements
     hostLevelParams.put(CONTAINER_ID, containerId);
     cmd.setHostLevelParams(hostLevelParams);
 
+    Map<String, String> roleParams = new TreeMap<>();
+    cmd.setRoleParams(roleParams);
+    cmd.getRoleParams().put("auto_restart", Boolean.toString(isMarkedAutoRestart));
+
     cmd.setCommandParams(setCommandParameters(scriptPath, true));
 
     Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId);

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java
index ca2db32..0545499 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java
@@ -40,6 +40,7 @@ public class HeartBeatResponse {
   RegistrationCommand registrationCommand;
 
   boolean restartAgent = false;
+  boolean restartEnabled = true;
   boolean hasMappedComponents = false;
 
   @JsonProperty("responseId")
@@ -92,6 +93,16 @@ public class HeartBeatResponse {
     this.restartAgent = restartAgent;
   }
 
+  @JsonProperty("restartEnabled")
+  public boolean getRstartEnabled() {
+    return restartEnabled;
+  }
+
+  @JsonProperty("restartEnabled")
+  public void setRestartEnabled(boolean restartEnabled) {
+    this.restartEnabled = restartEnabled;
+  }
+
   @JsonProperty("hasMappedComponents")
   public boolean hasMappedComponents() {
     return hasMappedComponents;

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index a68e770..463799c 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -78,6 +78,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import static org.easymock.EasyMock.anyBoolean;
 import static org.easymock.EasyMock.anyObject;
 import static org.easymock.EasyMock.createNiceMock;
 import static org.easymock.EasyMock.expect;
@@ -642,7 +643,8 @@ public class TestAgentProviderService {
           anyString(),
           anyString(),
           any(HeartBeatResponse.class),
-          anyString());
+          anyString(),
+          anyBoolean());
       doNothing().when(mockAps).addGetConfigCommand(
           anyString(),
           anyString(),
@@ -744,7 +746,8 @@ public class TestAgentProviderService {
       Mockito.verify(mockAps, Mockito.times(0)).addStartCommand(anyString(),
                                                                 anyString(),
                                                                 any(HeartBeatResponse.class),
-                                                                anyString());
+                                                                anyString(),
+                                                                anyBoolean());
       // RS still does not start
       hb = new HeartBeat();
       hb.setResponseId(3);
@@ -754,7 +757,8 @@ public class TestAgentProviderService {
       Mockito.verify(mockAps, Mockito.times(0)).addStartCommand(anyString(),
                                                                 anyString(),
                                                                 any(HeartBeatResponse.class),
-                                                                anyString());
+                                                                anyString(),
+                                                                anyBoolean());
 
       // MASTER succeeds install and issues start
       hb = new HeartBeat();
@@ -773,7 +777,8 @@ public class TestAgentProviderService {
       Mockito.verify(mockAps, Mockito.times(1)).addStartCommand(anyString(),
                                                                 anyString(),
                                                                 any(HeartBeatResponse.class),
-                                                                anyString());
+                                                                anyString(),
+                                                                anyBoolean());
       Map<String, String> allocatedPorts = mockAps.getAllocatedPorts();
       Assert.assertTrue(allocatedPorts != null);
       Assert.assertTrue(allocatedPorts.size() == 1);
@@ -788,7 +793,8 @@ public class TestAgentProviderService {
       Mockito.verify(mockAps, Mockito.times(1)).addStartCommand(anyString(),
                                                                 anyString(),
                                                                 any(HeartBeatResponse.class),
-                                                                anyString());
+                                                                anyString(),
+                                                                anyBoolean());
       // MASTER succeeds start
       hb = new HeartBeat();
       hb.setResponseId(3);
@@ -812,7 +818,8 @@ public class TestAgentProviderService {
       Mockito.verify(mockAps, Mockito.times(2)).addStartCommand(anyString(),
                                                                 anyString(),
                                                                 any(HeartBeatResponse.class),
-                                                                anyString());
+                                                                anyString(),
+                                                                anyBoolean());
     // JDK7 
     } catch (SliderException he) {
       log.warn(he.getMessage());
@@ -959,7 +966,7 @@ public class TestAgentProviderService {
 
     replay(access);
 
-    mockAps.addStartCommand("HBASE_MASTER", "cid1", hbr, "");
+    mockAps.addStartCommand("HBASE_MASTER", "cid1", hbr, "", Boolean.FALSE);
     Assert.assertTrue(hbr.getExecutionCommands().get(0).getConfigurations().containsKey("hbase-site"));
     Map<String, String> hbaseSiteConf = hbr.getExecutionCommands().get(0).getConfigurations().get("hbase-site");
     Assert.assertTrue(hbaseSiteConf.containsKey("a.port"));


[17/50] [abbrv] git commit: SLIDER-322 revert code constructs to java6, retaining java7 code commented out as JDK7 for fast switch-back

Posted by st...@apache.org.
SLIDER-322 revert code constructs to java6, retaining java7 code commented out as JDK7 for fast switch-back


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/397585a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/397585a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/397585a0

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 397585a052ca9dacc358003f50fa53b934832b50
Parents: daba289
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 14 17:16:18 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 14 17:16:18 2014 +0100

----------------------------------------------------------------------
 .../org/apache/slider/client/SliderClient.java  | 94 +++++++++++++++++++-
 .../slider/common/tools/ConfigHelper.java       | 56 ++++++++----
 .../slider/core/persist/JsonSerDeser.java       | 27 +++++-
 .../PublishedConfigurationOutputter.java        | 12 +++
 .../server/appmaster/SliderAppMaster.java       | 19 +++-
 .../server/appmaster/web/SliderAmIpFilter.java  |  6 +-
 .../agent/TestAgentProviderService.java         | 45 ++++++----
 .../agent/TestComponentInstanceState.java       | 11 +--
 .../providers/agent/TestHeartbeatMonitor.java   |  4 +-
 .../management/TestAMManagementWebServices.java |  9 +-
 .../publisher/TestAgentProviderService.java     |  2 +-
 .../workflow/ProcessCommandFactory.java         |  6 +-
 .../slider/test/MiniZooKeeperCluster.java       |  6 +-
 13 files changed, 242 insertions(+), 55 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 356f99f..93f6207 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -206,6 +206,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
    * @return the exit code
    * @throws Throwable anything that went wrong
    */
+/* JDK7
+
   @Override
   public int runService() throws Throwable {
 
@@ -278,6 +280,66 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
     return exitCode;
   }
 
+*/
+  @Override
+  public int runService() throws Throwable {
+
+    // choose the action
+    String action = serviceArgs.getAction();
+    int exitCode = EXIT_SUCCESS;
+    String clusterName = serviceArgs.getClusterName();
+    // actions
+    if (ACTION_BUILD.equals(action)) {
+      exitCode = actionBuild(clusterName, serviceArgs.getActionBuildArgs());
+    } else if (ACTION_CREATE.equals(action)) {
+      exitCode = actionCreate(clusterName, serviceArgs.getActionCreateArgs());
+    } else if (ACTION_FREEZE.equals(action)) {
+      exitCode = actionFreeze(clusterName,
+          serviceArgs.getActionFreezeArgs());
+    } else if (ACTION_THAW.equals(action)) {
+      exitCode = actionThaw(clusterName, serviceArgs.getActionThawArgs());
+    } else if (ACTION_DESTROY.equals(action)) {
+      exitCode = actionDestroy(clusterName);
+    } else if (ACTION_EXISTS.equals(action)) {
+      exitCode = actionExists(clusterName,
+          serviceArgs.getActionExistsArgs().live);
+    } else if (ACTION_FLEX.equals(action)) {
+      exitCode = actionFlex(clusterName, serviceArgs.getActionFlexArgs());
+    } else if (ACTION_GETCONF.equals(action)) {
+      exitCode = actionGetConf(clusterName, serviceArgs.getActionGetConfArgs());
+    } else if (ACTION_HELP.equals(action) ||
+               ACTION_USAGE.equals(action)) {
+      log.info(serviceArgs.usage());
+
+    } else if (ACTION_KILL_CONTAINER.equals(action)) {
+      exitCode = actionKillContainer(clusterName,
+          serviceArgs.getActionKillContainerArgs());
+
+    } else if (ACTION_AM_SUICIDE.equals(action)) {
+      exitCode = actionAmSuicide(clusterName,
+          serviceArgs.getActionAMSuicideArgs());
+
+    } else if (ACTION_LIST.equals(action)) {
+      exitCode = actionList(clusterName);
+    } else if (ACTION_REGISTRY.equals(action)) {
+      exitCode = actionRegistry(
+          serviceArgs.getActionRegistryArgs());
+    } else if (ACTION_STATUS.equals(action)) {
+      exitCode = actionStatus(clusterName,
+          serviceArgs.getActionStatusArgs());
+    } else if (ACTION_UPDATE.equals(action)) {
+      exitCode = actionUpdate(clusterName, serviceArgs.getActionUpdateArgs());
+
+    } else if (ACTION_VERSION.equals(action)) {
+
+      exitCode = actionVersion();
+    } else {
+      throw new SliderException(EXIT_UNIMPLEMENTED,
+          "Unimplemented: " + action);
+    }
+
+    return exitCode;
+  }
   /**
    * Delete the zookeeper node associated with the calling user and the cluster
    **/
@@ -331,7 +393,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
           client.createPath(zkPath, "", ZooDefs.Ids.OPEN_ACL_UNSAFE,
                             CreateMode.PERSISTENT);
           return zkPath;
-        } catch (InterruptedException | KeeperException e) {
+          
+          //JDK7
+//        } catch (InterruptedException | KeeperException e) {
+        } catch (InterruptedException e) {
+          log.warn("Unable to create zk node {}", zkPath, e);
+        } catch ( KeeperException e) {
           log.warn("Unable to create zk node {}", zkPath, e);
         }
       }
@@ -1676,7 +1743,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
           return EXIT_FALSE;
         }
       }
-    } catch (YarnException | IOException e) {
+
+// JDK7    } catch (YarnException | IOException e) {
+    } catch (YarnException e) {
+      log.warn("Exception while waiting for the cluster {} to shut down: {}",
+               clustername, e);
+    } catch ( IOException e) {
       log.warn("Exception while waiting for the cluster {} to shut down: {}",
                clustername, e);
     }
@@ -1740,6 +1812,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
     }
     try {
       String description = "Slider Application Instance " + clustername;
+// JDK7      
+/*
       switch (format) {
         case Arguments.FORMAT_XML:
           Configuration siteConf = getSiteConf(status, clustername);
@@ -1753,6 +1827,17 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
         default:
           throw new BadCommandArgumentsException("Unknown format: " + format);
       }
+*/
+      if (Arguments.FORMAT_XML.equals(format)) {
+        Configuration siteConf = getSiteConf(status, clustername);
+        siteConf.writeXml(writer);
+      } else if (Arguments.FORMAT_PROPERTIES.equals(format)) {
+        Properties props = new Properties();
+        props.putAll(status.clientProperties);
+        props.store(writer, description);
+      } else {
+          throw new BadCommandArgumentsException("Unknown format: " + format);
+      }
     } finally {
       // data is written.
       // close the file
@@ -2318,7 +2403,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
     try {
       maybeStartRegistry();
       return registry.instanceIDs(SliderKeys.APP_TYPE);
-    } catch (YarnException | IOException e) {
+/// JDK7    } catch (YarnException | IOException e) {
+    } catch (IOException e) {
+      throw e;
+    } catch (YarnException e) {
       throw e;
     } catch (Exception e) {
       throw new IOException(e);

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
index f8fd226..b7e1323 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
@@ -182,24 +182,53 @@ public class ConfigHelper {
   public Document parseConfiguration(FileSystem fs,
                                      Path path) throws
                                                 IOException {
-    int len = (int) fs.getLength(path);
-    byte[] data = new byte[len];
-    try(FSDataInputStream in = fs.open(path)) {
-      in.readFully(0, data);
-    }
 
+
+    byte[] data = loadBytes(fs, path);
     //this is here to track down a parse issue
     //related to configurations
-    String s = new String(data, 0, len);
+    String s = new String(data, 0, data.length);
     log.debug("XML resource {} is \"{}\"", path, s);
+/* JDK7
     try (ByteArrayInputStream in = new ByteArrayInputStream(data)) {
       Document document = parseConfigXML(in);
       return document;
     } catch (ParserConfigurationException | SAXException e) {
       throw new IOException(e);
     }
+*/
+    ByteArrayInputStream in= null;
+    try {
+      in = new ByteArrayInputStream(data);
+      Document document = parseConfigXML(in);
+      return document;
+    } catch (ParserConfigurationException e) {
+      throw new IOException(e);
+    } catch (SAXException e) {
+      throw new IOException(e);
+    } finally {
+      IOUtils.closeStream(in);
+    }
   }
-  
+
+  public static byte[] loadBytes(FileSystem fs, Path path) throws IOException {
+    int len = (int) fs.getLength(path);
+    byte[] data = new byte[len];
+    /* JDK7
+    try(FSDataInputStream in = fs.open(path)) {
+      in.readFully(0, data);
+    }
+*/
+    FSDataInputStream in = null;
+    in = fs.open(path);
+    try {
+      in.readFully(0, data);
+    } finally {
+      IOUtils.closeStream(in);
+    }
+    return data;
+  }
+
   /**
    * Load a configuration from ANY FS path. The normal Configuration
    * loader only works with file:// URIs
@@ -209,13 +238,9 @@ public class ConfigHelper {
    * @throws IOException
    */
   public static Configuration loadConfiguration(FileSystem fs,
-                                                Path path) throws
-                                                                   IOException {
-    int len = (int) fs.getLength(path);
-    byte[] data = new byte[len];
-    try (FSDataInputStream in = fs.open(path)) {
-      in.readFully(0, data);
-    }
+                                                Path path) throws IOException {
+    byte[] data = loadBytes(fs, path);
+
     ByteArrayInputStream in2;
 
     in2 = new ByteArrayInputStream(data);
@@ -523,7 +548,8 @@ public class ConfigHelper {
    * @param valuesource the source of values
    * @return a new configuration where <code>foreach key in keysource, get(key)==valuesource.get(key)</code>
    */
-  public static Configuration resolveConfiguration(Iterable<Map.Entry<String, String>> keysource,
+  public static Configuration resolveConfiguration(
+      Iterable<Map.Entry<String, String>> keysource,
       Configuration valuesource) {
     Configuration result = new Configuration(false);
     for (Map.Entry<String, String> entry : keysource) {

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
index 3512168..ab71683 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
@@ -105,7 +105,8 @@ public class JsonSerDeser<T> {
    * @throws IOException IO problems
    * @throws JsonMappingException failure to map from the JSON to this class
    */
-  public T fromResource(String resource)
+/* JDK7
+ public T fromResource(String resource)
     throws IOException, JsonParseException, JsonMappingException {
     try(InputStream resStream = this.getClass().getResourceAsStream(resource)) {
       if (resStream == null) {
@@ -116,6 +117,30 @@ public class JsonSerDeser<T> {
       log.error("Exception while parsing json resource {}: {}", resource, e);
       throw e;
     }
+  }*/
+
+  /**
+   * Convert from a JSON file
+   * @param resource input file
+   * @return the parsed JSON
+   * @throws IOException IO problems
+   * @throws JsonMappingException failure to map from the JSON to this class
+   */
+  public synchronized T fromResource(String resource)
+      throws IOException, JsonParseException, JsonMappingException {
+    InputStream resStream = null;
+    try {
+      resStream = this.getClass().getResourceAsStream(resource);
+      if (resStream == null) {
+        throw new FileNotFoundException(resource);
+      }
+      return (T) (mapper.readValue(resStream, classType));
+    } catch (IOException e) {
+      log.error("Exception while parsing json resource {}: {}", resource, e);
+      throw e;
+    } finally {
+      IOUtils.closeStream(resStream);
+    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
index 929b8ef..bf812dd 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
@@ -48,12 +48,24 @@ public abstract class PublishedConfigurationOutputter {
    * @param dest destination file
    * @throws IOException
    */
+/* JDK7
   public void save(File dest) throws IOException {
     try(FileOutputStream out = new FileOutputStream(dest)) {
       save(out);
       out.close();
     }
   }
+*/
+  public void save(File dest) throws IOException {
+    FileOutputStream out = null;
+    try {
+      out = new FileOutputStream(dest);
+      save(out);
+      out.close();
+    } finally {
+      org.apache.hadoop.io.IOUtils.closeStream(out);
+    }
+  }
 
   /**
    * Save the content. The default saves the asString() value

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 4a5c9b9..acd6870 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -459,7 +459,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     String action = serviceArgs.getAction();
     List<String> actionArgs = serviceArgs.getActionArgs();
     int exitCode;
-    switch (action) {
+/*  JDK7
+  switch (action) {
       case SliderActions.ACTION_HELP:
         log.info(getName() + serviceArgs.usage());
         exitCode = LauncherExitCodes.EXIT_USAGE;
@@ -470,6 +471,15 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
       default:
         throw new SliderException("Unimplemented: " + action);
     }
+    */
+    if (action.equals(SliderActions.ACTION_HELP)) {
+      log.info(getName() + serviceArgs.usage());
+      exitCode = SliderExitCodes.EXIT_USAGE;
+    } else if (action.equals(SliderActions.ACTION_CREATE)) {
+      exitCode = createAndRunCluster(actionArgs.get(0));
+    } else {
+      throw new SliderException("Unimplemented: " + action);
+    }
     log.info("Exiting AM; final exit code = {}", exitCode);
     return exitCode;
   }
@@ -1048,9 +1058,16 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     try {
       log.info("Unregistering AM status={} message={}", appStatus, appMessage);
       asyncRMClient.unregisterApplicationMaster(appStatus, appMessage, null);
+/* JDK7
     } catch (YarnException | IOException e) {
       log.info("Failed to unregister application: " + e, e);
     }
+*/
+    } catch (IOException e) {
+      log.info("Failed to unregister application: " + e, e);
+    } catch (YarnException e) {
+      log.info("Failed to unregister application: " + e, e);
+    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
index aad443e..4c66876 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
@@ -133,7 +133,11 @@ public class SliderAmIpFilter implements Filter {
             principal);
         chain.doFilter(requestWrapper, resp);
       }
-    } catch (IOException | ServletException e) {
+// JKD7    } catch (IOException | ServletException e) {
+    } catch (IOException e) {
+      log.warn("When fetching {}: {}", requestURI, e);
+      throw e;
+    } catch (ServletException e) {
       log.warn("When fetching {}: {}", requestURI, e);
       throw e;
     }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 22de0d2..ee61129 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -291,7 +291,10 @@ public class TestAgentProviderService {
                                           resourceComponent,
                                           appComponent,
                                           containerTmpDirPath);
-    } catch (SliderException | IOException he) {
+      // JDK7
+    } catch (IOException he) {
+      log.warn("{}", he, he);
+    } catch (SliderException he) {
       log.warn("{}", he, he);
     }
 
@@ -317,14 +320,14 @@ public class TestAgentProviderService {
       public ClusterDescription getClusterStatus() {
         ClusterDescription cd = new ClusterDescription();
         cd.status = new HashMap<String, Object>();
-        Map<String, Map<String, ClusterNode>> roleMap = new HashMap<>();
+        Map<String, Map<String, ClusterNode>> roleMap = new HashMap<String, Map<String, ClusterNode>>();
         ClusterNode cn1 = new ClusterNode(new MockContainerId(1));
         cn1.host = "FIRST_HOST";
-        Map<String, ClusterNode> map1 = new HashMap<>();
+        Map<String, ClusterNode> map1 = new HashMap<String, ClusterNode>();
         map1.put("FIRST_CONTAINER", cn1);
         ClusterNode cn2 = new ClusterNode(new MockContainerId(2));
         cn2.host = "SECOND_HOST";
-        Map<String, ClusterNode> map2 = new HashMap<>();
+        Map<String, ClusterNode> map2 = new HashMap<String, ClusterNode>();
         map2.put("SECOND_CONTAINER", cn2);
         ClusterNode cn3 = new ClusterNode(new MockContainerId(3));
         cn3.host = "THIRD_HOST";
@@ -350,7 +353,7 @@ public class TestAgentProviderService {
     };
 
     aps.setAmState(appState);
-    Map<String, String> tokens = new HashMap<>();
+    Map<String, String> tokens = new HashMap<String, String>();
     aps.addRoleRelatedTokens(tokens);
     Assert.assertEquals(2, tokens.size());
     Assert.assertEquals("FIRST_HOST", tokens.get("${FIRST_ROLE_HOST}"));
@@ -367,7 +370,7 @@ public class TestAgentProviderService {
     doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
     doReturn(metainfo).when(mockAps).getMetainfo();
 
-    Map<String, String> ports = new HashMap<>();
+    Map<String, String> ports = new HashMap<String, String>();
     ports.put("global.listen_port", "10010");
     mockAps.processAndPublishComponentSpecificData(ports,
                                                    "cid1",
@@ -408,17 +411,18 @@ public class TestAgentProviderService {
     status.setClusterName("test");
     status.setComponentName("HBASE_MASTER");
     status.setRoleCommand("GET_CONFIG");
-    Map<String, String> hbaseSite = new HashMap<>();
+    Map<String, String> hbaseSite = new HashMap<String, String>();
     hbaseSite.put("hbase.master.info.port", "60012");
     hbaseSite.put("c", "d");
-    Map<String, Map<String, String>> configs = new HashMap<>();
+    Map<String, Map<String, String>> configs = 
+        new HashMap<String, Map<String, String>>();
     configs.put("hbase-site", hbaseSite);
     configs.put("global", hbaseSite);
     status.setConfigs(configs);
-    hb.setComponentStatus(new ArrayList<>(Arrays.asList(status)));
+    hb.setComponentStatus(new ArrayList<ComponentStatus>(Arrays.asList(status)));
 
-    Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<>();
-    Map<String, ClusterNode> container = new HashMap<>();
+    Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<String, Map<String, ClusterNode>>();
+    Map<String, ClusterNode> container = new HashMap<String, ClusterNode>();
     ClusterNode cn1 = new ClusterNode(new MockContainerId(1));
     cn1.host = "HOST1";
     container.put("cid1", cn1);
@@ -754,7 +758,7 @@ public class TestAgentProviderService {
       cr.setRole("HBASE_MASTER");
       cr.setRoleCommand("INSTALL");
       cr.setStatus("COMPLETED");
-      Map<String, String> ap = new HashMap<>();
+      Map<String, String> ap = new HashMap<String, String>();
       ap.put("a.port", "10233");
       cr.setAllocatedPorts(ap);
       hb.setReports(Arrays.asList(cr));
@@ -803,7 +807,10 @@ public class TestAgentProviderService {
                                                                 anyString(),
                                                                 any(HeartBeatResponse.class),
                                                                 anyString());
-    } catch (SliderException | IOException he) {
+    // JDK7 
+    } catch (SliderException he) {
+      log.warn(he.getMessage());
+    } catch (IOException he) {
       log.warn(he.getMessage());
     }
 
@@ -882,8 +889,8 @@ public class TestAgentProviderService {
     doReturn("HOST1").when(mockAps).getClusterInfoPropertyValue(anyString());
     doReturn(metainfo).when(mockAps).getMetainfo();
 
-    Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<>();
-    Map<String, ClusterNode> container = new HashMap<>();
+    Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<String, Map<String, ClusterNode>>();
+    Map<String, ClusterNode> container = new HashMap<String, ClusterNode>();
     ClusterNode cn1 = new ClusterNode(new MockContainerId(1));
     cn1.host = "HOST1";
     container.put("cid1", cn1);
@@ -929,18 +936,18 @@ public class TestAgentProviderService {
 
     doReturn("HOST1").when(mockAps).getClusterInfoPropertyValue(anyString());
 
-    Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<>();
-    Map<String, ClusterNode> container = new HashMap<>();
+    Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<String, Map<String, ClusterNode>>();
+    Map<String, ClusterNode> container = new HashMap<String, ClusterNode>();
     ClusterNode cn1 = new ClusterNode(new MockContainerId(1));
     cn1.host = "HOST1";
     container.put("cid1", cn1);
     roleClusterNodeMap.put("HBASE_MASTER", container);
     doReturn(roleClusterNodeMap).when(mockAps).getRoleClusterNodeMapping();
-    Map<String, String> allocatedPorts = new HashMap<>();
+    Map<String, String> allocatedPorts = new HashMap<String, String>();
     allocatedPorts.put("hbase-site.a.port", "10023");
     allocatedPorts.put("hbase-site.b.port", "10024");
     doReturn(allocatedPorts).when(mockAps).getAllocatedPorts();
-    Map<String, String> allocatedPorts2 = new HashMap<>();
+    Map<String, String> allocatedPorts2 = new HashMap<String, String>();
     allocatedPorts2.put("hbase-site.random.port", "10025");
     doReturn(allocatedPorts2).when(mockAps).getAllocatedPorts(anyString());
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentInstanceState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentInstanceState.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentInstanceState.java
index e1f3d02..a723394 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentInstanceState.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentInstanceState.java
@@ -44,7 +44,7 @@ public class TestComponentInstanceState {
   }
 
   @Test
-  public void testGetNextStateBasedOnResult() {
+  public void testGetNextStateBasedOnResult() throws Exception {
     TestCase.assertEquals(State.INSTALLING, State.INSTALLING.getNextState(CommandResult.IN_PROGRESS));
     TestCase.assertEquals(State.STARTING, State.STARTING.getNextState(CommandResult.IN_PROGRESS));
     expectExceptionOnGetNextForResult(IllegalArgumentException.class, State.INIT, CommandResult.IN_PROGRESS);
@@ -68,7 +68,7 @@ public class TestComponentInstanceState {
   }
 
   @Test
-  public void testGetNextStateBasedOnCommand() {
+  public void testGetNextStateBasedOnCommand() throws Exception {
     for (State state : states) {
       TestCase.assertEquals(state, state.getNextState(Command.NOP));
     }
@@ -87,7 +87,8 @@ public class TestComponentInstanceState {
     expectIllegalArgumentException(State.STARTED, Command.START);
   }
 
-  protected void expectIllegalArgumentException(State state, Command command) {
+  protected void expectIllegalArgumentException(State state, Command command) throws
+      Exception {
     expectExceptionOnGetNextForCommand(IllegalArgumentException.class,
         state, command);
   }
@@ -274,7 +275,7 @@ public class TestComponentInstanceState {
   }
 
   private <T extends Throwable> void expectExceptionOnGetNextForResult(
-      Class<T> expected, State state, CommandResult result) {
+      Class<T> expected, State state, CommandResult result) throws Exception {
     try {
       state.getNextState(result);
       TestCase.fail("Must fail");
@@ -286,7 +287,7 @@ public class TestComponentInstanceState {
   }
 
   private <T extends Throwable> void expectExceptionOnGetNextForCommand(
-      Class<T> expected, State state, Command command) {
+      Class<T> expected, State state, Command command) throws Exception {
     try {
       state.getNextState(command);
       TestCase.fail("Must fail");

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
index 1ace44e..7314b72 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
@@ -56,7 +56,7 @@ public class TestHeartbeatMonitor {
     AgentProviderService provider = createNiceMock(AgentProviderService.class);
     HeartbeatMonitor hbm = new HeartbeatMonitor(provider, 500);
     Assert.assertFalse(hbm.isAlive());
-    Map<String, ComponentInstanceState> statuses = new HashMap<>();
+    Map<String, ComponentInstanceState> statuses = new HashMap<String, ComponentInstanceState>();
     ContainerId container1 = new MockContainerId(1);
     ComponentInstanceState state = new ComponentInstanceState("HBASE_MASTER",
         container1, "Aid");
@@ -79,7 +79,7 @@ public class TestHeartbeatMonitor {
     long now = 100000;
     int wakeupInterval = 2 * 1000;
 
-    Map<String, ComponentInstanceState> statuses = new HashMap<>();
+    Map<String, ComponentInstanceState> statuses = new HashMap<String, ComponentInstanceState>();
     ContainerId masterContainer = new MockContainerId(1); 
     ContainerId slaveContainer = new MockContainerId(2); 
     ComponentInstanceState masterState = new ComponentInstanceState("HBASE_MASTER",

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
index fd66bee..91aa2b0 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
@@ -177,7 +177,14 @@ public class TestAMManagementWebServices extends JerseyTest {
               fs,
               historyPath,
               null, null, new SimpleReleaseSelector());
-        } catch (IOException | BadClusterStateException | URISyntaxException | BadConfigException e) {
+// JDK7        } catch (IOException | BadClusterStateException | URISyntaxException | BadConfigException e) {
+        } catch (IOException e) {
+          log.error("{}", e, e);
+        } catch (BadClusterStateException e) {
+          log.error("{}", e, e);
+        } catch (URISyntaxException e) {
+          log.error("{}", e, e);
+        } catch (BadConfigException e) {
           log.error("{}", e, e);
         }
         ProviderAppState providerAppState = new ProviderAppState("undefined",

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
index a97e351..eb368e3 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
@@ -46,7 +46,7 @@ public class TestAgentProviderService extends AgentProviderService {
       QueueAccess queueAccess,
       List<Container> liveContainers) {
     super.bind(stateAccessor, reg, queueAccess, liveContainers);
-    Map<String,String> dummyProps = new HashMap<>();
+    Map<String,String> dummyProps = new HashMap<String, String>();
     dummyProps.put("prop1", "val1");
     dummyProps.put("prop2", "val2");
     log.info("publishing dummy-site.xml with values {}", dummyProps);

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java b/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
index e77eeb3..45fdc86 100644
--- a/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
+++ b/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
@@ -37,7 +37,7 @@ public class ProcessCommandFactory {
    * @return commands
    */
   public List<String> ls(File dir) {
-    List<String> commands = new ArrayList<>(5);
+    List<String> commands = new ArrayList<String>(5);
     commands.add("ls");
     commands.add("-1");
     commands.add(dir.getAbsolutePath());
@@ -50,7 +50,7 @@ public class ProcessCommandFactory {
    * @return commands
    */
   public List<String> echo(String text) {
-    List<String> commands = new ArrayList<>(5);
+    List<String> commands = new ArrayList<String>(5);
     commands.add("echo");
     commands.add(text);
     return commands;
@@ -72,7 +72,7 @@ public class ProcessCommandFactory {
    * @return commands
    */
   public List<String> exitFalse() {
-    List<String> commands = new ArrayList<>(2);
+    List<String> commands = new ArrayList<String>(2);
     commands.add("false");
     return commands;
   }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/test/java/org/apache/slider/test/MiniZooKeeperCluster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/test/MiniZooKeeperCluster.java b/slider-core/src/test/java/org/apache/slider/test/MiniZooKeeperCluster.java
index cc2cc9b..d739324 100644
--- a/slider-core/src/test/java/org/apache/slider/test/MiniZooKeeperCluster.java
+++ b/slider-core/src/test/java/org/apache/slider/test/MiniZooKeeperCluster.java
@@ -75,9 +75,9 @@ public class MiniZooKeeperCluster {
     this.started = false;
     this.configuration = configuration;
     activeZKServerIndex = -1;
-    zooKeeperServers = new ArrayList<>();
-    clientPortList = new ArrayList<>();
-    standaloneServerFactoryList = new ArrayList<>();
+    zooKeeperServers = new ArrayList<ZooKeeperServer>();
+    clientPortList = new ArrayList<Integer>();
+    standaloneServerFactoryList = new ArrayList<NIOServerCnxnFactory>();
   }
 
   public void setDefaultClientPort(int clientPort) {


[08/50] [abbrv] git commit: SLIDER-202: Chaos Monkey adds a (slow) minicluster test.

Posted by st...@apache.org.
SLIDER-202: Chaos Monkey adds a (slow) minicluster test.


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/53028eaa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/53028eaa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/53028eaa

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 53028eaae347899cef9e695e9ee8b41594c6377b
Parents: 52bd11b
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 13 17:30:49 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 13 17:30:49 2014 +0100

----------------------------------------------------------------------
 .../org/apache/slider/api/InternalKeys.java     | 49 +++++++---
 .../apache/slider/common/tools/SliderUtils.java |  2 +-
 .../server/appmaster/SliderAppMaster.java       | 39 ++++----
 .../TestStandaloneAMMonkeyRestart.groovy        | 94 ++++++++++++++++++++
 .../TestRegionServerFailureThreshold.groovy     |  5 +-
 5 files changed, 154 insertions(+), 35 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/53028eaa/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
index 4045c0e..ad384e2 100644
--- a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
@@ -84,31 +84,54 @@ public interface InternalKeys {
    * Flag to indicate whether or not the chaos monkey is enabled:
    * {@value}
    */
-  String INTERNAL_CHAOS_MONKEY_ENABLED = "internal.chaos.monkey.enabled";
-  boolean DEFAULT_INTERNAL_CHAOS_MONKEY_ENABLED = false;
+  String CHAOS_MONKEY_ENABLED = "internal.chaos.monkey.enabled";
+  boolean DEFAULT_CHAOS_MONKEY_ENABLED = false;
 
 
   /**
    * Rate
    */
 
-  String INTERNAL_CHAOS_MONKEY_RATE = "internal.chaos.monkey.rate";
-
-  int DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_DAYS = 0;
-  int DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_HOURS = 1;
-  int DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_MINUTES = 0;
+  String CHAOS_MONKEY_INTERVAL = "internal.chaos.monkey.interval";
+  String CHAOS_MONKEY_INTERVAL_DAYS = CHAOS_MONKEY_INTERVAL + ".days";
+  String CHAOS_MONKEY_INTERVAL_HOURS = CHAOS_MONKEY_INTERVAL + ".hours";
+  String CHAOS_MONKEY_INTERVAL_MINUTES = CHAOS_MONKEY_INTERVAL + ".minutes";
+  String CHAOS_MONKEY_INTERVAL_SECONDS = CHAOS_MONKEY_INTERVAL + ".seconds";
   
-  String INTERNAL_CHAOS_MONKEY_PROBABILITY =
+  int DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS = 0;
+  int DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS = 0;
+  int DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES = 0;
+
+  /**
+   * Prefix for all chaos monkey probabilities
+   */
+  String CHAOS_MONKEY_PROBABILITY =
       "internal.chaos.monkey.probability";
   /**
    * Probabilies are out of 10000 ; 100==1%
    */
 
-  String INTERNAL_CHAOS_MONKEY_PROBABILITY_AM_FAILURE = INTERNAL_CHAOS_MONKEY_PROBABILITY +".amfailure";
-  int DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE = 10;
-  String INTERNAL_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE =
-      INTERNAL_CHAOS_MONKEY_PROBABILITY + ".containerfailure";
-  int DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE = 100;
+  /**
+   * Probability of a monkey check killing the AM:  {@value}
+   */
+  String CHAOS_MONKEY_PROBABILITY_AM_FAILURE = CHAOS_MONKEY_PROBABILITY +".amfailure";
+
+  /**
+   * Default probability of a monkey check killing the AM:  {@value}
+   */
+  int DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE = 0;
+
+  /**
+   * Probability of a monkey check killing a container:  {@value}
+   */
+
+  String CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE =
+      CHAOS_MONKEY_PROBABILITY + ".containerfailure";
+
+  /**
+   * Default probability of a monkey check killing the a container:  {@value}
+   */
+  int DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE = 0;
 
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/53028eaa/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index d7f159c..7a80f24 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -1476,7 +1476,7 @@ public final class SliderUtils {
           }
           is = new ByteArrayInputStream(content);
         } else {
-          log.info("Size unknown. Reading {}", zipEntry.getName());
+          log.debug("Size unknown. Reading {}", zipEntry.getName());
           ByteArrayOutputStream baos = new ByteArrayOutputStream();
           while (true) {
             int byteRead = zis.read();

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/53028eaa/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 1642cc5..7825e24 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -779,17 +779,16 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     // now do the registration
     registerServiceInstance(clustername, appid);
 
-    sliderAMProvider.start();
+    // chaos monkey
+    maybeStartMonkey();
 
+    // Start the Slider AM provider
+    sliderAMProvider.start();
 
-    
-    // launch the provider; this is expected to trigger a callback that
+    // launch the real provider; this is expected to trigger a callback that
     // starts the node review process
     launchProviderService(instanceDefinition, confDir);
 
-    // chaos monkey
-    maybeStartMonkey();
-
     try {
       //now block waiting to be told to exit the process
       waitForAMCompletionSignal();
@@ -1459,7 +1458,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     rmOperationHandler.execute(operations);
   }
 
-
   /**
    * Get the RM operations handler for direct scheduling of work.
    */
@@ -1739,42 +1737,45 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     }
     signalAMComplete(exitCode, exception.toString());
   }
-  
-  public boolean maybeStartMonkey() {
+
+  /**
+   * Start the chaos monkey
+   * @return true if it started
+   */
+  private boolean maybeStartMonkey() {
     MapOperations internals = getGlobalInternalOptions();
 
     Boolean enabled =
-        internals.getOptionBool(InternalKeys.INTERNAL_CHAOS_MONKEY_ENABLED,
-            InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_ENABLED);
+        internals.getOptionBool(InternalKeys.CHAOS_MONKEY_ENABLED,
+            InternalKeys.DEFAULT_CHAOS_MONKEY_ENABLED);
     if (!enabled) {
       log.info("Chaos monkey disabled");
     }
     
     long monkeyInterval = internals.getTimeRange(
-        InternalKeys.INTERNAL_CHAOS_MONKEY_RATE,
-        InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_DAYS,
-        InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_HOURS,
-        InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_MINUTES,
+        InternalKeys.CHAOS_MONKEY_INTERVAL,
+        InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS,
+        InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS,
+        InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES,
         0);
     log.info("Adding Chaos Monkey scheduled every {} seconds ({} hours)",
         monkeyInterval, monkeyInterval/(60*60));
     monkey = new ChaosMonkeyService(metrics, actionQueues);
     int amKillProbability = internals.getOptionInt(
-        InternalKeys.INTERNAL_CHAOS_MONKEY_PROBABILITY_AM_FAILURE,
+        InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE,
         InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE);
     if (amKillProbability > 0) {
-      log.info("Adding AM killer with probability %f", amKillProbability/100.0);
       monkey.addTarget("AM killer",
           new ChaosKillAM(actionQueues, -1), amKillProbability
       );
     }
     int containerKillProbability = internals.getOptionInt(
-        InternalKeys.INTERNAL_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE,
+        InternalKeys.CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE,
         InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE);
     if (containerKillProbability > 0) {
       monkey.addTarget("Container killer",
           new ChaosKillContainer(appState, actionQueues, rmOperationHandler),
-          amKillProbability
+          containerKillProbability
       );
     }
     initAndAddService(monkey);

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/53028eaa/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
new file mode 100644
index 0000000..162bab0
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.SleepJob
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.api.InternalKeys
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.ActionAMSuicideArgs
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.exceptions.ErrorStrings
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * kill a masterless AM and verify it shuts down. This test
+ * also sets the retry count to 1 to stop recreation attempts
+ */
+@CompileStatic
+@Slf4j
+
+class TestStandaloneAMMonkeyRestart extends AgentMiniClusterTestBase {
+
+
+  @Test
+  public void testStandaloneAMMonkeyRestart() throws Throwable {
+    describe "Run a Standalone AM with the Chaos monkey set to kill it"
+    // patch the configuration for AM restart
+    int threshold = 2;
+    YarnConfiguration conf = getRestartableConfiguration(threshold)
+
+    String clustername = createMiniCluster("", conf, 1, true)
+    ServiceLauncher<SliderClient> launcher =
+        createStandaloneAMWithArgs(clustername,
+            [
+                Arguments.ARG_OPTION, InternalKeys.CHAOS_MONKEY_ENABLED, "true",
+                Arguments.ARG_OPTION, InternalKeys.CHAOS_MONKEY_INTERVAL_SECONDS, "8",
+                Arguments.ARG_OPTION, InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE, "75000",
+            ],
+            true,
+            false)
+    SliderClient sliderClient = launcher.service
+    addToTeardown(sliderClient);
+
+    ApplicationReport report
+    report = waitForClusterLive(sliderClient, 30000)
+    describe "Waiting for the cluster to fail"
+    sleep(40000)
+    // end of process
+    report = sliderClient.applicationReport
+    log.info(report.diagnostics)
+    assert report.currentApplicationAttemptId.attemptId == threshold
+    assert YarnApplicationState.FAILED == report.yarnApplicationState  
+    assert FinalApplicationStatus.FAILED == report.finalApplicationStatus
+  }
+
+  /**
+   * Get a restartable configuration
+   * @param restarts
+   * @return
+   */
+  public YarnConfiguration getRestartableConfiguration(int restarts) {
+    def conf = new YarnConfiguration(configuration)
+    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, restarts)
+    conf.setInt(SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, restarts)
+    conf
+  }
+
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/53028eaa/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy
index 1322fd3..b3aaa48 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy
@@ -149,8 +149,9 @@ class TestRegionServerFailureThreshold extends HBaseMiniClusterTestBase {
         }
       }
     } catch (BadClusterStateException e) {
-      assert e.toString().contains(ErrorStrings.E_APPLICATION_NOT_RUNNING)
-      assert e.exitCode == SliderExitCodes.EXIT_BAD_STATE
+      assertExceptionDetails(e,
+          SliderExitCodes.EXIT_BAD_STATE, 
+          ErrorStrings.E_APPLICATION_NOT_RUNNING)
     }
     ApplicationReport report = client.applicationReport
     log.info(report.diagnostics)


[35/50] [abbrv] git commit: SLIDER-320 add unregister operation, rename PublishRegistryDetails. AM queues an unregister on container completion

Posted by st...@apache.org.
SLIDER-320 add unregister operation, rename PublishRegistryDetails. AM queues an unregister on container completion


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/01c7390e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/01c7390e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/01c7390e

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 01c7390e52fedc9ac32123a9ddd1fc0f2d0f12b7
Parents: 31bfae2
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 15 15:33:26 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 15 15:33:26 2014 +0100

----------------------------------------------------------------------
 .../providers/agent/AgentProviderService.java   |  4 +-
 .../server/appmaster/SliderAppMaster.java       | 24 ++++++----
 .../actions/PublishRegistryDetails.java         | 44 ------------------
 .../actions/RegisterComponentInstance.java      | 48 ++++++++++++++++++++
 .../actions/UnregisterComponentInstance.java    | 46 +++++++++++++++++++
 5 files changed, 111 insertions(+), 55 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/01c7390e/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 680074d..60c99ac 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -63,7 +63,7 @@ import org.apache.slider.providers.agent.application.metadata.Metainfo;
 import org.apache.slider.providers.agent.application.metadata.OSPackage;
 import org.apache.slider.providers.agent.application.metadata.OSSpecific;
 import org.apache.slider.server.appmaster.actions.ProviderReportedContainerLoss;
-import org.apache.slider.server.appmaster.actions.PublishRegistryDetails;
+import org.apache.slider.server.appmaster.actions.RegisterComponentInstance;
 import org.apache.slider.server.appmaster.state.ContainerPriority;
 import org.apache.slider.server.appmaster.state.RoleInstance;
 import org.apache.slider.server.appmaster.state.StateAccessForProviders;
@@ -572,7 +572,7 @@ public class AgentProviderService extends AbstractProviderService implements
     
     // and update registration entries
     if (instance != null) {
-      queueAccess.put(new PublishRegistryDetails(instance.getId(), 0,
+      queueAccess.put(new RegisterComponentInstance(instance.getId(), 0,
           TimeUnit.MILLISECONDS));
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/01c7390e/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index c54a268..93adfb2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -98,7 +98,7 @@ import org.apache.slider.providers.SliderProviderFactory;
 import org.apache.slider.providers.slideram.SliderAMClientProvider;
 import org.apache.slider.providers.slideram.SliderAMProviderService;
 import org.apache.slider.server.appmaster.actions.ActionKillContainer;
-import org.apache.slider.server.appmaster.actions.PublishRegistryDetails;
+import org.apache.slider.server.appmaster.actions.RegisterComponentInstance;
 import org.apache.slider.server.appmaster.actions.QueueExecutor;
 import org.apache.slider.server.appmaster.actions.ActionHalt;
 import org.apache.slider.server.appmaster.actions.QueueService;
@@ -106,6 +106,7 @@ import org.apache.slider.server.appmaster.actions.ActionStopSlider;
 import org.apache.slider.server.appmaster.actions.AsyncAction;
 import org.apache.slider.server.appmaster.actions.RenewingAction;
 import org.apache.slider.server.appmaster.actions.ResetFailureWindow;
+import org.apache.slider.server.appmaster.actions.UnregisterComponentInstance;
 import org.apache.slider.server.appmaster.monkey.ChaosKillAM;
 import org.apache.slider.server.appmaster.monkey.ChaosKillContainer;
 import org.apache.slider.server.appmaster.monkey.ChaosMonkeyService;
@@ -910,11 +911,22 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
       return false;
     }
     // this is where component registrations will go
+    log.info("Registering component {}", id);
 
     return true;
   }
   
   /**
+   * unregister a component. At the time this message is received,
+   * the component may already been deleted from/never added to
+   * the app state
+   * @param id the component
+   */
+  public void unregisterComponent(ContainerId id) {
+    log.info("Unregistering component {}", id);
+  }
+  
+  /**
    * looks for a specific case where a token file is provided as an environment
    * variable, yet the file is not there.
    * 
@@ -1178,15 +1190,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
       }
 
       getProviderService().notifyContainerCompleted(containerId);
+      queue(new UnregisterComponentInstance(containerId, 0, TimeUnit.MILLISECONDS));
     }
 
-    // ask for more containers if any failed
-    // In the case of Slider, we don't expect containers to complete since
-    // Slider is a long running application. Keep track of how many containers
-    // are completing. If too many complete, abort the application
-    // TODO: this needs to be better thought about (and maybe something to
-    // better handle in Yarn for long running apps)
-
     try {
       reviewRequestAndReleaseNodes();
     } catch (SliderInternalStateException e) {
@@ -1681,7 +1687,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
       nmClientAsync.getContainerStatusAsync(containerId,
                                             cinfo.container.getNodeId());
       // push out a registration
-      queue(new PublishRegistryDetails(containerId, 0, TimeUnit.MILLISECONDS));
+      queue(new RegisterComponentInstance(containerId, 0, TimeUnit.MILLISECONDS));
       
     } else {
       //this is a hypothetical path not seen. We react by warning

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/01c7390e/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java
deleted file mode 100644
index cb6d140..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.state.AppState;
-
-import java.util.concurrent.TimeUnit;
-
-public class PublishRegistryDetails extends AsyncAction {
-  
-
-  public final ContainerId containerId;
-
-  public PublishRegistryDetails(ContainerId containerId, long delay,
-      TimeUnit timeUnit) {
-    super("PublishRegistryDetails :" + containerId.toString(),
-        delay, timeUnit);
-    this.containerId = containerId;
-  }
-
-  @Override
-  public void execute(SliderAppMaster appMaster,
-      QueueAccess queueService,
-      AppState appState) throws Exception {
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/01c7390e/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
new file mode 100644
index 0000000..a8a6fe2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.state.AppState;
+
+import java.util.concurrent.TimeUnit;
+
+public class RegisterComponentInstance extends AsyncAction {
+  
+
+  public final ContainerId containerId;
+
+  public RegisterComponentInstance(ContainerId containerId, long delay,
+      TimeUnit timeUnit) {
+    super("RegisterComponentInstance :" + containerId,
+        delay, timeUnit);
+    Preconditions.checkArgument(containerId != null);
+    this.containerId = containerId;
+  }
+
+  @Override
+  public void execute(SliderAppMaster appMaster,
+      QueueAccess queueService,
+      AppState appState) throws Exception {
+
+    appMaster.registerComponent(containerId);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/01c7390e/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java
new file mode 100644
index 0000000..78d9c1c
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.state.AppState;
+
+import java.util.concurrent.TimeUnit;
+
+public class UnregisterComponentInstance extends AsyncAction {
+  
+
+  public final ContainerId containerId;
+
+  public UnregisterComponentInstance(ContainerId containerId, long delay,
+      TimeUnit timeUnit) {
+    super("UnregisterComponentInstance :" + containerId.toString(),
+        delay, timeUnit);
+    this.containerId = containerId;
+  }
+
+  @Override
+  public void execute(SliderAppMaster appMaster,
+      QueueAccess queueService,
+      AppState appState) throws Exception {
+    appMaster.unregisterComponent(containerId);
+
+  }
+}


[40/50] [abbrv] git commit: SLIDER-324. Exports should be attributed to individual master components rather than only one master component (app changes)

Posted by st...@apache.org.
SLIDER-324. Exports should be attributed to individual master components rather than only one master component (app changes)


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/14b54a75
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/14b54a75
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/14b54a75

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 14b54a75bed9f6e74e27b54c096cd1469b023b7c
Parents: 2a25ab9
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sun Aug 17 14:15:16 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sun Aug 17 14:15:16 2014 -0700

----------------------------------------------------------------------
 app-packages/accumulo/metainfo.xml |  3 ++-
 app-packages/hbase/appConfig.json  |  1 +
 app-packages/hbase/metainfo.xml    | 30 ++++++++++++++++++++++--------
 app-packages/hbase/pom.xml         |  2 +-
 4 files changed, 26 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/14b54a75/app-packages/accumulo/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/metainfo.xml b/app-packages/accumulo/metainfo.xml
index 4cf6c79..b1aa9de 100644
--- a/app-packages/accumulo/metainfo.xml
+++ b/app-packages/accumulo/metainfo.xml
@@ -40,7 +40,7 @@
             </value>
           </export>
           <export>
-            <name>org.apache.slider.jmx</name>
+            <name>app.jmx</name>
             <value>
               ${site.global.monitor_protocol}://${ACCUMULO_MONITOR_HOST}:${site.accumulo-site.monitor.port.client}/xml
             </value>
@@ -85,6 +85,7 @@
         <name>ACCUMULO_MONITOR</name>
         <category>MASTER</category>
         <publishConfig>true</publishConfig>
+        <appExports>QuickLinks-app.jmx,QuickLinks-org.apache.slider.monitor</appExports>
         <commandScript>
           <script>scripts/accumulo_monitor.py</script>
           <scriptType>PYTHON</scriptType>

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/14b54a75/app-packages/hbase/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json
index 24d9a84..d00ae6d 100644
--- a/app-packages/hbase/appConfig.json
+++ b/app-packages/hbase/appConfig.json
@@ -25,6 +25,7 @@
     "site.global.ganglia_server_id": "Application1",
     "site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
     "site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}",
+    "site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
     "site.hbase-site.hbase.hstore.flush.retries.number": "120",
     "site.hbase-site.hbase.client.keyvalue.maxsize": "10485760",
     "site.hbase-site.hbase.hstore.compactionThreshold": "3",

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/14b54a75/app-packages/hbase/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase/metainfo.xml b/app-packages/hbase/metainfo.xml
index a5dab0c..aae048d 100644
--- a/app-packages/hbase/metainfo.xml
+++ b/app-packages/hbase/metainfo.xml
@@ -53,11 +53,11 @@
             <value>http://${HBASE_THRIFT_HOST}:${site.global.hbase_thrift_port}</value>
           </export>
           <export>
-            <name>org.apache.slider.metrics</name>
+            <name>app.metrics</name>
             <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value>
           </export>
           <export>
-            <name>org.apache.slider.ganglia</name>
+            <name>app.ganglia</name>
             <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value>
           </export>
         </exports>
@@ -81,6 +81,17 @@
         <category>MASTER</category>
         <minInstanceCount>1</minInstanceCount>
         <maxInstanceCount>2</maxInstanceCount>
+        <appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor,QuickLinks-app.metrics,QuickLinks-app.ganglia</appExports>
+        <componentExports>
+          <componentExport>
+            <name>app.jmx</name>
+            <value>${THIS_HOST}:${site.hbase-site.hbase.master.info.port}/jmx</value>
+          </componentExport>
+          <componentExport>
+            <name>app.monitor</name>
+            <value>${THIS_HOST}:${site.hbase-site.hbase.master.info.port}/master-status</value>
+          </componentExport>
+        </componentExports>
         <commandScript>
           <script>scripts/hbase_master.py</script>
           <scriptType>PYTHON</scriptType>
@@ -100,8 +111,9 @@
 
       <component>
         <name>HBASE_REST</name>
-        <category>SLAVE</category>
-        <minInstanceCount>1</minInstanceCount>
+        <category>MASTER</category>
+        <minInstanceCount>0</minInstanceCount>
+        <appExports>QuickLinks-org.apache.slider.hbase.rest</appExports>
         <commandScript>
           <script>scripts/hbase_rest.py</script>
           <scriptType>PYTHON</scriptType>
@@ -110,8 +122,9 @@
 
       <component>
         <name>HBASE_THRIFT</name>
-        <category>SLAVE</category>
-        <minInstanceCount>1</minInstanceCount>
+        <category>MASTER</category>
+        <minInstanceCount>0</minInstanceCount>
+        <appExports>QuickLinks-org.apache.slider.hbase.thrift</appExports>
         <commandScript>
           <script>scripts/hbase_thrift.py</script>
           <scriptType>PYTHON</scriptType>
@@ -120,8 +133,9 @@
 
       <component>
         <name>HBASE_THRIFT2</name>
-        <category>SLAVE</category>
-        <minInstanceCount>1</minInstanceCount>
+        <category>MASTER</category>
+        <minInstanceCount>0</minInstanceCount>
+        <appExports>QuickLinks-org.apache.slider.hbase.thrift2</appExports>
         <commandScript>
           <script>scripts/hbase_thrift2.py</script>
           <scriptType>PYTHON</scriptType>

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/14b54a75/app-packages/hbase/pom.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase/pom.xml b/app-packages/hbase/pom.xml
index 4450f8b..7dede6c 100644
--- a/app-packages/hbase/pom.xml
+++ b/app-packages/hbase/pom.xml
@@ -19,7 +19,7 @@
   <parent>
     <groupId>org.apache.slider</groupId>
     <artifactId>slider</artifactId>
-    <version>0.31.0-incubating-SNAPSHOT</version>
+    <version>0.41.0-incubating-SNAPSHOT</version>
     <relativePath>../../pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>


[27/50] [abbrv] git commit: SLIDER-325 codahale metrics not being uploaded to AM classpath

Posted by st...@apache.org.
SLIDER-325 codahale metrics not being uploaded to AM classpath


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/388805a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/388805a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/388805a1

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 388805a1db2342973f3dcc6355a4205f01bd7a5a
Parents: 599e972
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 15 13:05:53 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 15 13:05:53 2014 +0100

----------------------------------------------------------------------
 .../slideram/SliderAMClientProvider.java        | 28 ++++++++++----------
 1 file changed, 14 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/388805a1/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
index ffff76c..dc84f02 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
@@ -19,6 +19,7 @@
 package org.apache.slider.providers.slideram;
 
 import com.beust.jcommander.JCommander;
+import com.codahale.metrics.MetricRegistry;
 import com.google.gson.GsonBuilder;
 import org.apache.curator.CuratorZookeeperClient;
 import org.apache.curator.framework.CuratorFramework;
@@ -59,8 +60,8 @@ import java.util.Map;
  * This keeps aspects of role, cluster validation and Clusterspec setup
  * out of the core slider client
  */
-public class SliderAMClientProvider extends AbstractClientProvider implements
-    SliderKeys {
+public class SliderAMClientProvider extends AbstractClientProvider
+    implements SliderKeys {
 
 
   protected static final Logger log =
@@ -113,9 +114,8 @@ public class SliderAMClientProvider extends AbstractClientProvider implements
                                                     AggregateConf instanceDefinition,
                                                     Path clusterDirPath,
                                                     Path generatedConfDirPath,
-                                                    boolean secure) throws
-      SliderException,
-                                                                    IOException {
+                                                    boolean secure)
+      throws SliderException, IOException {
 
     super.preflightValidateClusterConfiguration(sliderFileSystem, clustername, configuration, instanceDefinition, clusterDirPath, generatedConfDirPath, secure);
     //add a check for the directory being writeable by the current user
@@ -167,7 +167,7 @@ public class SliderAMClientProvider extends AbstractClientProvider implements
       CuratorZookeeperClient.class,
       ServiceInstance.class,
       ServiceNames.class,
-
+      MetricRegistry.class
     };
     String[] jars =
       {
@@ -179,6 +179,7 @@ public class SliderAMClientProvider extends AbstractClientProvider implements
         "curator-client.jar",
         "curator-x-discovery.jar",
         "curator-x-discovery-service.jar",
+        "metrics-core.jar"
       };
     ProviderUtils.addDependencyJars(providerResources, fileSystem, tempPath,
                                     libdir, jars,
@@ -209,11 +210,11 @@ public class SliderAMClientProvider extends AbstractClientProvider implements
    * add them to the command line
    */
   public void addJVMOptions(AggregateConf aggregateConf,
-                            JavaCommandLineBuilder cmdLine) throws
-                                                        BadConfigException {
-    
+                            JavaCommandLineBuilder cmdLine)
+      throws BadConfigException {
+
     MapOperations sliderAM =
-      aggregateConf.getAppConfOperations().getMandatoryComponent(
+        aggregateConf.getAppConfOperations().getMandatoryComponent(
         SliderKeys.COMPONENT_AM);
     cmdLine.forceIPv4().headless();
     String heap = sliderAM.getOption(RoleKeys.JVM_HEAP,
@@ -227,11 +228,10 @@ public class SliderAMClientProvider extends AbstractClientProvider implements
 
 
   @Override
-  public void prepareInstanceConfiguration(AggregateConf aggregateConf) throws
-      SliderException,
-                                                                        IOException {
+  public void prepareInstanceConfiguration(AggregateConf aggregateConf)
+      throws SliderException, IOException {
     mergeTemplates(aggregateConf,
-                   INTERNAL_JSON, RESOURCES_JSON, APPCONF_JSON
+        INTERNAL_JSON, RESOURCES_JSON, APPCONF_JSON
                   );
   }
 }


[41/50] [abbrv] git commit: Merge branch 'feature/SLIDER-324_Refactor_Exports' into develop

Posted by st...@apache.org.
Merge branch 'feature/SLIDER-324_Refactor_Exports' into develop


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/72daf371
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/72daf371
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/72daf371

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 72daf3713141c6d17dd1e3aad5bb85c2a76c4e96
Parents: 8050466 14b54a7
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sun Aug 17 14:16:04 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sun Aug 17 14:16:04 2014 -0700

----------------------------------------------------------------------
 app-packages/accumulo/metainfo.xml              |   3 +-
 app-packages/hbase/appConfig.json               |   1 +
 app-packages/hbase/metainfo.xml                 |  30 ++-
 app-packages/hbase/pom.xml                      |   2 +-
 app-packages/storm/metainfo.xml                 |  28 +--
 .../providers/agent/AgentProviderService.java   | 216 +++++++++++--------
 .../agent/application/metadata/Component.java   |  21 +-
 .../application/metadata/ComponentExport.java   |  54 +++++
 .../application/metadata/MetainfoParser.java    |   9 +-
 .../agent/TestAgentProviderService.java         |  55 +++--
 .../metadata/MetainfoParserTest.java            |  59 -----
 .../metadata/TestMetainfoParser.java            |  63 ++++++
 .../agent/application/metadata/metainfo.xml     |  74 ++++++-
 13 files changed, 411 insertions(+), 204 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/72daf371/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------


[26/50] [abbrv] git commit: Merging feature/SLIDER-181_Storm_Package to develop (jdk6 change)

Posted by st...@apache.org.
Merging feature/SLIDER-181_Storm_Package to develop (jdk6 change)


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/599e9721
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/599e9721
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/599e9721

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 599e9721e161feed526a9fe712791af10ce5ca03
Parents: 47822df
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Aug 14 18:07:20 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Aug 14 18:07:20 2014 -0700

----------------------------------------------------------------------
 .../org/apache/slider/providers/agent/AgentProviderService.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/599e9721/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 8ce1f18..91bc0a6 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -1180,7 +1180,7 @@ public class AgentProviderService extends AbstractProviderService implements
     hostLevelParams.put(CONTAINER_ID, containerId);
     cmd.setHostLevelParams(hostLevelParams);
 
-    Map<String, String> roleParams = new TreeMap<>();
+    Map<String, String> roleParams = new TreeMap<String, String>();
     cmd.setRoleParams(roleParams);
     cmd.getRoleParams().put("auto_restart", Boolean.toString(isMarkedAutoRestart));
 


[02/50] [abbrv] git commit: SLIDER-202: add codahale metrics

Posted by st...@apache.org.
SLIDER-202: add codahale metrics


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8d4217e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8d4217e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8d4217e7

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 8d4217e7988e561d6e9432f86a76c86adcfaee87
Parents: d38b31a
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 12 20:43:00 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 12 20:43:00 2014 +0100

----------------------------------------------------------------------
 pom.xml             |  8 ++++++++
 slider-core/pom.xml | 46 +++++-----------------------------------------
 2 files changed, 13 insertions(+), 41 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8d4217e7/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 253d40f..ed823c4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -143,11 +143,14 @@
 
     <jackson.version>1.9.13</jackson.version>
     <jcommander.version>1.30</jcommander.version>
+
     <jersey.version>1.9</jersey.version>
     <servlet-api.version>2.5</servlet-api.version>
     <jsr311-api.version>1.1.1</jsr311-api.version>
+
     <junit.version>4.11</junit.version>
     <log4j.version>1.2.17</log4j.version>
+    <metrics.version>3.0.1</metrics.version>
     <mockito.version>1.8.5</mockito.version>
 
     <!-- ProtocolBuffer version, used to verify the protoc version and -->
@@ -1174,6 +1177,11 @@
         <version>${jersey.version}</version>
       </dependency>
 
+      <dependency>
+        <groupId>com.codahale.metrics</groupId>
+        <artifactId>metrics-core</artifactId>
+        <version>${metrics.version}</version>
+      </dependency>
 
       <!-- ======================================================== -->
       <!-- Mocking -->

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8d4217e7/slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index a44e3bc..d403034 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -309,48 +309,7 @@
       <artifactId>hadoop-minicluster</artifactId>
       <scope>test</scope>
     </dependency>
-<!--
-
-    <dependency>
-      <groupId>org.apache.hbase</groupId>
-      <artifactId>hbase-client</artifactId>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.hbase</groupId>
-      <artifactId>hbase-server</artifactId>
-      <scope>test</scope>
-    </dependency>
-
--->
-
-<!--
-
-    <dependency>
-      <groupId>org.apache.accumulo</groupId>
-      <artifactId>accumulo-core</artifactId>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.accumulo</groupId>
-      <artifactId>accumulo-minicluster</artifactId>
-      <scope>test</scope>
-    </dependency>
-
-    <dependency>
-      <groupId>org.apache.accumulo</groupId>
-      <artifactId>accumulo-start</artifactId>
-      <scope>test</scope>
-    </dependency>
 
-    <dependency>
-      <groupId>org.apache.accumulo</groupId>
-      <artifactId>accumulo-trace</artifactId>
-      <scope>test</scope>
-    </dependency>
--->
 
     <dependency>
       <groupId>junit</groupId>
@@ -398,6 +357,11 @@
       <artifactId>commons-logging</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>com.codahale.metrics</groupId>
+      <artifactId>metrics-core</artifactId>
+    </dependency>
+    
     <!-- ======================================================== -->
     <!-- service registry -->
     <!-- ======================================================== -->


[32/50] [abbrv] git commit: SLIDER-320 rename registration action

Posted by st...@apache.org.
SLIDER-320 rename registration action


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/41e0c5be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/41e0c5be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/41e0c5be

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 41e0c5bec76a922479147642f2132cb17d66615d
Parents: 95dcbc9
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 15 14:28:06 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 15 14:28:06 2014 +0100

----------------------------------------------------------------------
 .../actions/PublishEndpointDetails.java         | 45 --------------------
 .../actions/PublishRegistryDetails.java         | 45 ++++++++++++++++++++
 2 files changed, 45 insertions(+), 45 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41e0c5be/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishEndpointDetails.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishEndpointDetails.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishEndpointDetails.java
deleted file mode 100644
index 67198a2..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishEndpointDetails.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.state.AppState;
-
-import java.util.concurrent.TimeUnit;
-
-public class PublishEndpointDetails extends AsyncAction {
-  
-
-  public final ContainerId containerId;
-
-  public PublishEndpointDetails(ContainerId containerId, long delay,
-      TimeUnit timeUnit) {
-    super("PublishEndpointDetails :" + containerId.toString(),
-        delay, timeUnit);
-    this.containerId = containerId;
-  }
-
-  @Override
-  public void execute(SliderAppMaster appMaster,
-      QueueAccess queueService,
-      AppState appState) throws Exception {
-    
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41e0c5be/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java
new file mode 100644
index 0000000..f96183f
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.state.AppState;
+
+import java.util.concurrent.TimeUnit;
+
+public class PublishRegistryDetails extends AsyncAction {
+  
+
+  public final ContainerId containerId;
+
+  public PublishRegistryDetails(ContainerId containerId, long delay,
+      TimeUnit timeUnit) {
+    super("PublishEndpointDetails :" + containerId.toString(),
+        delay, timeUnit);
+    this.containerId = containerId;
+  }
+
+  @Override
+  public void execute(SliderAppMaster appMaster,
+      QueueAccess queueService,
+      AppState appState) throws Exception {
+    
+  }
+}


[09/50] [abbrv] git commit: Merge branch 'feature/SLIDER-202_chaos_monkey' into develop

Posted by st...@apache.org.
Merge branch 'feature/SLIDER-202_chaos_monkey' into develop


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9b20e47b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9b20e47b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9b20e47b

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 9b20e47b50d6e3f51467c0517f3b0d21242f1f17
Parents: d38b31a 53028ea
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 13 18:10:32 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 13 18:10:32 2014 +0100

----------------------------------------------------------------------
 pom.xml                                         |   8 +
 slider-core/pom.xml                             |  46 +-----
 .../api/ClusterDescriptionOperations.java       |  12 +-
 .../org/apache/slider/api/InternalKeys.java     | 137 ++++++++++++++++
 .../java/org/apache/slider/api/OptionKeys.java  |  69 +-------
 .../org/apache/slider/api/ResourceKeys.java     |  26 +--
 .../org/apache/slider/client/SliderClient.java  |   7 +-
 .../apache/slider/common/tools/SliderUtils.java |   8 +-
 .../slider/core/build/InstanceBuilder.java      |   3 +-
 .../apache/slider/core/conf/MapOperations.java  |  54 ++++++-
 .../apache/slider/providers/ProviderUtils.java  |   9 +-
 .../providers/agent/AgentClientProvider.java    |   8 +-
 .../providers/agent/AgentProviderService.java   |   7 +-
 .../slideram/SliderAMClientProvider.java        |   4 +-
 .../server/appmaster/SliderAppMaster.java       | 157 ++++++++++++------
 .../server/appmaster/actions/ActionHalt.java    |   4 +-
 .../appmaster/actions/ActionKillContainer.java  |  67 ++++++++
 .../server/appmaster/actions/QueueService.java  |   8 +-
 .../server/appmaster/monkey/ChaosEntry.java     |  86 ++++++++++
 .../server/appmaster/monkey/ChaosKillAM.java    |  48 ++++++
 .../appmaster/monkey/ChaosKillContainer.java    |  75 +++++++++
 .../appmaster/monkey/ChaosMonkeyService.java    |  86 ++++++++++
 .../server/appmaster/monkey/ChaosTarget.java    |  24 +++
 .../appmaster/monkey/MonkeyPlayAction.java      |  48 ++++++
 .../slider/server/appmaster/state/AppState.java |  14 +-
 .../TestStandaloneAMMonkeyRestart.groovy        |  94 +++++++++++
 .../standalone/TestStandaloneAgentAM.groovy     |   2 +-
 .../TestMockAppStateContainerFailure.groovy     |   2 +-
 .../TestMockAppStateDynamicRoles.groovy         |   2 +-
 .../TestMockAppStateFlexDynamicRoles.groovy     |   2 +-
 .../appstate/TestMockAppStateFlexing.groovy     |   2 +-
 .../TestMockAppStateRMOperations.groovy         |   2 +-
 .../TestMockAppStateRebuildOnAMRestart.groovy   |   2 +-
 .../TestMockAppStateRolePlacement.groovy        |   2 +-
 .../appstate/TestMockAppStateRoleRelease.groovy |   2 +-
 .../TestMockContainerResourceAllocations.groovy |   2 +-
 .../model/mock/BaseMockAppStateTest.groovy      |   8 +-
 .../model/mock/MockRMOperationHandler.groovy    |   7 +-
 .../model/monkey/TestMockMonkey.groovy          | 160 +++++++++++++++++++
 .../apache/slider/test/SliderTestBase.groovy    |   7 +-
 .../agent/TestAgentProviderService.java         |   9 +-
 .../accumulo/AccumuloClientProvider.java        |   3 +-
 .../accumulo/AccumuloProviderService.java       |   9 +-
 .../providers/hbase/HBaseClientProvider.java    |   3 +-
 .../providers/hbase/HBaseProviderService.java   |   4 +-
 .../TestRegionServerFailureThreshold.groovy     |   5 +-
 ...reezeReconfigureThawLiveRegionService.groovy |   4 +-
 47 files changed, 1097 insertions(+), 251 deletions(-)
----------------------------------------------------------------------



[46/50] [abbrv] git commit: SLIDER-306 in sync w/ trunk and JDK6

Posted by st...@apache.org.
SLIDER-306 in sync w/ trunk and JDK6


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f06b37cf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f06b37cf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f06b37cf

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: f06b37cf415ed0a602f38454096b19d74492a3d4
Parents: 5d59454
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 19 11:07:03 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 19 11:07:03 2014 +0100

----------------------------------------------------------------------
 .../src/main/java/org/apache/slider/client/SliderClient.java     | 2 +-
 .../main/java/org/apache/slider/common/tools/SliderUtils.java    | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f06b37cf/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index fb06454..d608278 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -321,7 +321,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
           serviceArgs.getActionAMSuicideArgs());
 
     } else if (ACTION_LIST.equals(action)) {
-      exitCode = actionList(clusterName);
+      exitCode = actionList(clusterName, serviceArgs.getActionListArgs());
     } else if (ACTION_REGISTRY.equals(action)) {
       exitCode = actionRegistry(
           serviceArgs.getActionRegistryArgs());

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f06b37cf/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 6aff38a..3f83855 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -579,8 +579,8 @@ public final class SliderUtils {
     if (instances.size() <= 1) {
       return;
     }
-    List<ApplicationReport> nonLiveInstance = new ArrayList<>(instances.size());
-    List<ApplicationReport> liveInstance = new ArrayList<>(instances.size());
+    List<ApplicationReport> nonLiveInstance = new ArrayList<ApplicationReport>(instances.size());
+    List<ApplicationReport> liveInstance = new ArrayList<ApplicationReport>(instances.size());
 
     for (ApplicationReport report : instances) {
       if (report.getYarnApplicationState() == YarnApplicationState.RUNNING


[07/50] [abbrv] git commit: SLIDER-202 monkey has mock tests and is set to run in AM if enabled

Posted by st...@apache.org.
SLIDER-202 monkey has mock tests and is set to run in AM if enabled


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/52bd11b5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/52bd11b5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/52bd11b5

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 52bd11b5d4d3a17f231e5343c0bfa6c39492fbb3
Parents: 8ed52a9
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 13 15:06:11 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 13 15:06:11 2014 +0100

----------------------------------------------------------------------
 .../org/apache/slider/api/ResourceKeys.java     |  26 +--
 .../apache/slider/common/tools/SliderUtils.java |   6 +-
 .../apache/slider/core/conf/MapOperations.java  |  54 ++++++-
 .../providers/agent/AgentClientProvider.java    |   8 +-
 .../providers/agent/AgentProviderService.java   |   7 +-
 .../server/appmaster/SliderAppMaster.java       |  88 ++++++----
 .../appmaster/actions/ActionKillContainer.java  |  24 ++-
 .../server/appmaster/actions/QueueService.java  |   8 +-
 .../server/appmaster/monkey/ChaosEntry.java     |  12 +-
 .../server/appmaster/monkey/ChaosKillAM.java    |  12 +-
 .../appmaster/monkey/ChaosKillContainer.java    |  25 +--
 .../appmaster/monkey/ChaosMonkeyService.java    |  23 ++-
 .../model/mock/MockRMOperationHandler.groovy    |   7 +-
 .../model/monkey/TestMockMonkey.groovy          | 160 +++++++++++++++++++
 .../apache/slider/test/SliderTestBase.groovy    |   7 +-
 15 files changed, 360 insertions(+), 107 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
index c2b78cc..3d54140 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
@@ -112,34 +112,14 @@ public interface ResourceKeys {
       "yarn.container.failure.threshold";
 
   /**
-   * Minutes range of the container failure reset window.
-   * This is combined with the hour and day values to
-   * produce the full window.
+   * prefix for the time of the container failure reset window.
    * {@value}
    */
-  String CONTAINER_FAILURE_WINDOW_MINUTES =
-      "yarn.container.failure.window.minutes";
 
-  
-  
-  /**
-   * Hours range of the container failure reset window
-   * This is combined with the minute and day values to
-   * produce the full window.
-   * {@value}
-   */
-  String CONTAINER_FAILURE_WINDOW_HOURS =
-      "yarn.container.failure.window.hours";
+  String CONTAINER_FAILURE_WINDOW =
+      "yarn.container.failure.window";
 
 
-  /**
-   * Hours range of the container failure reset window
-   * This is combined with the hour and minute values to
-   * produce the full window.
-   * {@value}
-   */
-  String CONTAINER_FAILURE_WINDOW_DAYS =
-      "yarn.container.failure.window.days";
 
   int DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS = 0;
   int DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS = 6;

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index edbf2b2..d7f159c 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.InternalKeys;
 import org.apache.slider.api.RoleKeys;
 import org.apache.slider.common.SliderKeys;
 import org.apache.slider.common.SliderXmlConfKeys;
@@ -1303,8 +1303,8 @@ public final class SliderUtils {
       SliderException, IOException {
     Path imagePath;
     String imagePathOption =
-        internalOptions.get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
-    String appHomeOption = internalOptions.get(OptionKeys.INTERNAL_APPLICATION_HOME);
+        internalOptions.get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+    String appHomeOption = internalOptions.get(InternalKeys.INTERNAL_APPLICATION_HOME);
     if (!isUnset(imagePathOption)) {
       imagePath = fs.createPathThatMustExist(imagePathOption);
     } else {

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
index bb57b94..98a51bc 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
@@ -18,6 +18,7 @@
 
 package org.apache.slider.core.conf;
 
+import com.google.common.base.Preconditions;
 import org.apache.slider.common.tools.SliderUtils;
 import org.apache.slider.core.exceptions.BadConfigException;
 import org.slf4j.Logger;
@@ -61,12 +62,11 @@ public class MapOperations implements Map<String, String> {
     this.name = name;
   }
 
-
   /**
-   * Get a cluster option or value
+   * Get an option value
    *
-   * @param key
-   * @param defVal
+   * @param key key
+   * @param defVal default value
    * @return option in map or the default
    */
   public String getOption(String key, String defVal) {
@@ -74,14 +74,27 @@ public class MapOperations implements Map<String, String> {
     return val != null ? val : defVal;
   }
 
+  /**
+   * Get a boolean option
+   *
+   * @param key option key
+   * @param defVal default value
+   * @return option true if the option equals "true", or the default value
+   * if the option was not defined at all.
+   */
+  public Boolean getOptionBool(String key, boolean defVal) {
+    String val = getOption(key, Boolean.toString(defVal));
+    return Boolean.valueOf(val);
+  }
 
   /**
    * Get a cluster option or value
    *
-   * @param key
+   * @param key option key
    * @return the value
    * @throws BadConfigException if the option is missing
    */
+
   public String getMandatoryOption(String key) throws BadConfigException {
     String val = options.get(key);
     if (val == null) {
@@ -247,4 +260,35 @@ public class MapOperations implements Map<String, String> {
     }
     return builder.toString();
   }
+
+  /**
+   * Get the time range of a set of keys
+   * @param basekey
+   * @param defDays
+   * @param defHours
+   * @param defMins
+   * @param defSecs
+   * @return
+   */
+  public long getTimeRange(String basekey,
+      int defDays,
+      int defHours,
+      int defMins,
+      int defSecs) {
+    Preconditions.checkArgument(basekey != null);
+    int days = getOptionInt(basekey + ".days", defDays);
+    int hours = getOptionInt(basekey + ".hours", defHours);
+
+    int minutes = getOptionInt(basekey + ".minutes", defMins);
+    int seconds = getOptionInt(basekey + ".seconds", defSecs);
+    // range check
+    Preconditions.checkState(days >= 0 && hours >= 0 && minutes >= 0
+                             && seconds >= 0,
+        "Time range for %s has negative time component %s:%s:%s:%s",
+        basekey, days, hours, minutes, seconds);
+
+    // calculate total time, schedule the reset if expected
+    long totalMinutes = days * 24 * 60 + hours * 24 + minutes;
+    return totalMinutes * 60 + seconds;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 2ce16c7..72b0a42 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -21,7 +21,7 @@ package org.apache.slider.providers.agent;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
-import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.InternalKeys;
 import org.apache.slider.api.ResourceKeys;
 import org.apache.slider.common.SliderKeys;
 import org.apache.slider.common.tools.SliderFileSystem;
@@ -109,7 +109,7 @@ public class AgentClientProvider extends AbstractClientProvider
         getGlobalOptions().get(AgentKeys.PACKAGE_PATH);
     if (SliderUtils.isUnset(appHome)) {
       String agentImage = instanceDefinition.getInternalOperations().
-          get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+          get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
       sliderFileSystem.verifyFileExists(new Path(agentImage));
     }
   }
@@ -174,12 +174,12 @@ public class AgentClientProvider extends AbstractClientProvider
     String appHome = instanceDefinition.getAppConfOperations().
         getGlobalOptions().get(AgentKeys.PACKAGE_PATH);
     String agentImage = instanceDefinition.getInternalOperations().
-        get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+        get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
 
     if (SliderUtils.isUnset(appHome) && SliderUtils.isUnset(agentImage)) {
       throw new BadConfigException("Either agent package path " +
                                    AgentKeys.PACKAGE_PATH + " or image root " +
-                                   OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH
+                                   InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH
                                    + " must be provided.");
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 59a6e29..0c57b79 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.slider.api.ClusterDescription;
 import org.apache.slider.api.ClusterDescriptionKeys;
 import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.InternalKeys;
 import org.apache.slider.api.OptionKeys;
 import org.apache.slider.api.StatusKeys;
 import org.apache.slider.common.SliderKeys;
@@ -62,8 +63,6 @@ import org.apache.slider.providers.agent.application.metadata.Metainfo;
 import org.apache.slider.providers.agent.application.metadata.OSPackage;
 import org.apache.slider.providers.agent.application.metadata.OSSpecific;
 import org.apache.slider.server.appmaster.actions.ProviderReportedContainerLoss;
-import org.apache.slider.server.appmaster.state.ContainerPriority;
-import org.apache.slider.server.appmaster.state.RoleStatus;
 import org.apache.slider.server.appmaster.state.StateAccessForProviders;
 import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType;
 import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
@@ -265,7 +264,7 @@ public class AgentProviderService extends AbstractProviderService implements
     log.info("PYTHONPATH set to {}", pythonPath);
 
     String agentImage = instanceDefinition.getInternalOperations().
-        get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+        get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
     if (agentImage != null) {
       LocalResource agentImageRes = fileSystem.createAmResource(new Path(agentImage), LocalResourceType.ARCHIVE);
       launcher.addLocalResource(AgentKeys.AGENT_INSTALL_DIR, agentImageRes);
@@ -1169,7 +1168,7 @@ public class AgentProviderService extends AbstractProviderService implements
     tokens.put("${DEFAULT_DATA_DIR}", getAmState()
         .getInternalsSnapshot()
         .getGlobalOptions()
-        .getMandatoryOption(OptionKeys.INTERNAL_DATA_DIR_PATH));
+        .getMandatoryOption(InternalKeys.INTERNAL_DATA_DIR_PATH));
     return tokens;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index e6c7aba..1642cc5 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -19,6 +19,7 @@
 package org.apache.slider.server.appmaster;
 
 import com.codahale.metrics.MetricRegistry;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.protobuf.BlockingService;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -1195,31 +1196,19 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     ResetFailureWindow reset = new ResetFailureWindow();
     ConfTreeOperations ops = new ConfTreeOperations(resources);
     MapOperations globals = ops.getGlobalOptions();
-    int days = globals.getOptionInt(ResourceKeys.CONTAINER_FAILURE_WINDOW_DAYS,
-        ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS);
-    int hours = globals.getOptionInt(
-        ResourceKeys.CONTAINER_FAILURE_WINDOW_HOURS,
-        ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS);
-    int minutes = globals.getOptionInt(
-        ResourceKeys.CONTAINER_FAILURE_WINDOW_MINUTES,
-        ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES);
-    // range check
-    if (days < 0 || hours < 0 || minutes < 0) {
-      throw new BadConfigException("Failure window contains negative values:"
-         + "days=%d hours=%d, minutes=%d",
-          days, hours, minutes);
-    }
-    // calculate total time, schedule the reset if expected
-    int totalMinutes = days * 24 * 60 + hours * 24 + minutes;
-    if (totalMinutes > 0) {
+    long seconds = globals.getTimeRange(ResourceKeys.CONTAINER_FAILURE_WINDOW,
+        ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS,
+        ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS,
+        ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES, 0);
+    if (seconds > 0) {
       log.info(
-          "Scheduling the failure window reset interval  to every {} minutes",
-          totalMinutes);
+          "Scheduling the failure window reset interval to every {} seconds",
+          seconds);
       RenewingAction<ResetFailureWindow> renew = new RenewingAction<>(
-          reset, totalMinutes, totalMinutes, TimeUnit.MINUTES, 0);
+          reset, seconds, seconds, TimeUnit.SECONDS, 0);
       actionQueues.renewing("failures", renew);
     } else {
-      log.warn("Failure window reset interval is not set");
+      log.info("Failure window reset interval is not set");
     }
   }
   
@@ -1458,7 +1447,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     //throws NoSuchNodeException if it is missing
     RoleInstance instance =
       appState.getLiveInstanceByContainerID(containerID);
-    queue(new ActionKillContainer(instance.getId(), 0, TimeUnit.MILLISECONDS));
+    queue(new ActionKillContainer(instance.getId(), 0, TimeUnit.MILLISECONDS,
+        rmOperationHandler));
     Messages.KillContainerResponseProto.Builder builder =
       Messages.KillContainerResponseProto.newBuilder();
     builder.setSuccess(true);
@@ -1469,6 +1459,15 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     rmOperationHandler.execute(operations);
   }
 
+
+  /**
+   * Get the RM operations handler for direct scheduling of work.
+   */
+  @VisibleForTesting
+  public RMOperationHandler getRmOperationHandler() {
+    return rmOperationHandler;
+  }
+
   @Override
   public Messages.AMSuicideResponseProto amSuicide(
       Messages.AMSuicideRequestProto request)
@@ -1742,18 +1741,45 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
   }
   
   public boolean maybeStartMonkey() {
+    MapOperations internals = getGlobalInternalOptions();
+
+    Boolean enabled =
+        internals.getOptionBool(InternalKeys.INTERNAL_CHAOS_MONKEY_ENABLED,
+            InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_ENABLED);
+    if (!enabled) {
+      log.info("Chaos monkey disabled");
+    }
+    
+    long monkeyInterval = internals.getTimeRange(
+        InternalKeys.INTERNAL_CHAOS_MONKEY_RATE,
+        InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_DAYS,
+        InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_HOURS,
+        InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_MINUTES,
+        0);
+    log.info("Adding Chaos Monkey scheduled every {} seconds ({} hours)",
+        monkeyInterval, monkeyInterval/(60*60));
     monkey = new ChaosMonkeyService(metrics, actionQueues);
-    int amKillProbability = 100;
-    int containerKillProbability = 200;
-    monkey.addTarget("AM killer", 
-        new ChaosKillAM(this, actionQueues, -1),
-        amKillProbability);
-    monkey.addTarget("Container killer", 
-        new ChaosKillContainer(this, actionQueues),
-        amKillProbability);
+    int amKillProbability = internals.getOptionInt(
+        InternalKeys.INTERNAL_CHAOS_MONKEY_PROBABILITY_AM_FAILURE,
+        InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE);
+    if (amKillProbability > 0) {
+      log.info("Adding AM killer with probability %f", amKillProbability/100.0);
+      monkey.addTarget("AM killer",
+          new ChaosKillAM(actionQueues, -1), amKillProbability
+      );
+    }
+    int containerKillProbability = internals.getOptionInt(
+        InternalKeys.INTERNAL_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE,
+        InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE);
+    if (containerKillProbability > 0) {
+      monkey.addTarget("Container killer",
+          new ChaosKillContainer(appState, actionQueues, rmOperationHandler),
+          amKillProbability
+      );
+    }
     initAndAddService(monkey);
     // and schedule it
-    schedule(monkey.getChaosAction(60, TimeUnit.SECONDS));
+    schedule(monkey.getChaosAction(monkeyInterval, TimeUnit.SECONDS));
     return true;
   }
   

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
index 7668bd5..95edca4 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
@@ -18,10 +18,12 @@
 
 package org.apache.slider.server.appmaster.actions;
 
+import com.google.common.base.Preconditions;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.slider.server.appmaster.SliderAppMaster;
 import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
 import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
+import org.apache.slider.server.appmaster.operations.RMOperationHandler;
 import org.apache.slider.server.appmaster.state.AppState;
 
 import java.util.LinkedList;
@@ -31,13 +33,27 @@ import java.util.concurrent.TimeUnit;
 public class ActionKillContainer extends AsyncAction {
 
   private final ContainerId containerId;
-  
-  public ActionKillContainer(ContainerId containerId, long delay,
-      TimeUnit timeUnit) {
+  private final RMOperationHandler operationHandler;
+  public ActionKillContainer(
+      ContainerId containerId,
+      long delay,
+      TimeUnit timeUnit,
+      RMOperationHandler operationHandler) {
     super("kill container", delay, timeUnit);
+    this.operationHandler = operationHandler;
+    Preconditions.checkArgument(containerId != null);
+    
     this.containerId = containerId;
   }
 
+  /**
+   * Get the container ID to kill
+   * @return
+   */
+  public ContainerId getContainerId() {
+    return containerId;
+  }
+
   @Override
   public void execute(SliderAppMaster appMaster,
       QueueAccess queueService,
@@ -46,6 +62,6 @@ public class ActionKillContainer extends AsyncAction {
     ContainerReleaseOperation release = new ContainerReleaseOperation(containerId);
     opsList.add(release);
     //now apply the operations
-    appMaster.executeRMOperations(opsList);
+    operationHandler.execute(opsList);
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
index 1154a44..ac59096 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
@@ -55,18 +55,14 @@ implements Runnable, QueueAccess {
 
   /**
    * Immediate actions.
-   * Package scoped for testing.
    */
-  @VisibleForTesting
-  final BlockingDeque<AsyncAction> actionQueue =
+  public final BlockingDeque<AsyncAction> actionQueue =
       new LinkedBlockingDeque<>();
 
   /**
    * Actions to be scheduled in the future
-   * Package scoped for testing.
    */
-  @VisibleForTesting
-  final DelayQueue<AsyncAction> scheduledActions = new DelayQueue<>();
+  public final DelayQueue<AsyncAction> scheduledActions = new DelayQueue<>();
 
   /**
    * Map of renewing actions by name ... this is to allow them to 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
index 13bda57..5905d2f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
@@ -41,7 +41,7 @@ public class ChaosEntry {
 
 
   /**
-   * Constructor -includes validaton of all arguments
+   * Constructor -includes validation of all arguments
    * @param name
    * @param target
    * @param probability
@@ -51,7 +51,8 @@ public class ChaosEntry {
     Preconditions.checkArgument(!StringUtils.isEmpty(name), "missing name");
     Preconditions.checkArgument(target != null, "null target");
     Preconditions.checkArgument(probability > 0, "negative probability");
-    Preconditions.checkArgument(probability < 10000, "probability over 100%");
+    Preconditions.checkArgument(probability <= ChaosMonkeyService.PERCENT_100,
+        "probability over 100%");
     this.name = name;
     this.target = target;
     this.probability = probability;
@@ -71,11 +72,12 @@ public class ChaosEntry {
 
   /**
    * Invoke Chaos if the trigger value is in range of the probability
-   * @param trigger trigger value, 0-10K
+   * @param value trigger value, 0-10K
    * @return true if the chaos method was invoked
    */
-  public boolean maybeInvokeChaos(long trigger) {
-    if (probability < 0) {
+  public boolean maybeInvokeChaos(long value) {
+    log.debug("Probability {} trigger={}", probability, value);
+    if (value < probability) {
       invokeChaos();
       return true;
     }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java
index c464ac3..3c1a914 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java
@@ -18,21 +18,21 @@
 
 package org.apache.slider.server.appmaster.monkey;
 
-import org.apache.slider.server.appmaster.SliderAppMaster;
 import org.apache.slider.server.appmaster.actions.ActionHalt;
 import org.apache.slider.server.appmaster.actions.QueueAccess;
 
 import java.util.concurrent.TimeUnit;
 
+/**
+ * Kill the AM
+ */
 public class ChaosKillAM implements ChaosTarget {
 
-  private final SliderAppMaster appMaster;
+  public static final int DELAY = 1000;
   private final QueueAccess queues;
   private final int exitCode;
 
-  public ChaosKillAM(SliderAppMaster appMaster,
-      QueueAccess queues, int exitCode) {
-    this.appMaster = appMaster;
+  public ChaosKillAM(QueueAccess queues, int exitCode) {
     this.queues = queues;
     this.exitCode = exitCode;
   }
@@ -42,7 +42,7 @@ public class ChaosKillAM implements ChaosTarget {
    */
   @Override
   public void chaosAction() {
-    queues.schedule(new ActionHalt(exitCode, "Chaos invoked halt", 1000,
+    queues.schedule(new ActionHalt(exitCode, "Chaos invoked halt", DELAY,
         TimeUnit.MILLISECONDS));
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
index 0ffcc8a..daf2590 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
@@ -18,9 +18,11 @@
 
 package org.apache.slider.server.appmaster.monkey;
 
-import org.apache.slider.server.appmaster.SliderAppMaster;
+import com.google.common.base.Preconditions;
 import org.apache.slider.server.appmaster.actions.ActionKillContainer;
 import org.apache.slider.server.appmaster.actions.QueueAccess;
+import org.apache.slider.server.appmaster.operations.RMOperationHandler;
+import org.apache.slider.server.appmaster.state.AppState;
 import org.apache.slider.server.appmaster.state.RoleInstance;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -35,24 +37,29 @@ import java.util.concurrent.TimeUnit;
 public class ChaosKillContainer implements ChaosTarget {
   protected static final Logger log =
       LoggerFactory.getLogger(ChaosKillContainer.class);
-  private final SliderAppMaster appMaster;
+  public static final int DELAY = 100;
+  private final AppState appState;
   private final QueueAccess queues;
   private final Random random = new Random();
+  private final RMOperationHandler operationHandler;
 
-  public ChaosKillContainer(SliderAppMaster appMaster,
-      QueueAccess queues) {
-    this.appMaster = appMaster;
+  public ChaosKillContainer(AppState appState,
+      QueueAccess queues,
+      RMOperationHandler operationHandler) {
+    Preconditions.checkNotNull(appState);
+    Preconditions.checkNotNull(queues);
+    this.appState = appState;
     this.queues = queues;
+    this.operationHandler = operationHandler;
   }
 
   /**
-   * Trigger a container kill halt
+   * Trigger a container kill 
    */
-  
   @Override
   public void chaosAction() {
     List<RoleInstance> liveContainers =
-        appMaster.getAppState().cloneLiveContainerInfoList();
+        appState.cloneLiveContainerInfoList();
     int size = liveContainers.size();
     if (size == 0) {
       log.info("No containers to kill");
@@ -63,6 +70,6 @@ public class ChaosKillContainer implements ChaosTarget {
     log.info("Killing {}", roleInstance);
 
     queues.schedule(new ActionKillContainer(roleInstance.getId(),
-        100, TimeUnit.MILLISECONDS));
+        DELAY, TimeUnit.MILLISECONDS, operationHandler));
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
index c525d02..cd4b46b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
@@ -20,7 +20,6 @@ package org.apache.slider.server.appmaster.monkey;
 
 import com.codahale.metrics.MetricRegistry;
 import org.apache.hadoop.service.AbstractService;
-import org.apache.slider.server.appmaster.actions.AsyncAction;
 import org.apache.slider.server.appmaster.actions.QueueAccess;
 import org.apache.slider.server.appmaster.actions.RenewingAction;
 import org.slf4j.Logger;
@@ -28,6 +27,7 @@ import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Random;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -36,8 +36,16 @@ import java.util.concurrent.TimeUnit;
 public class ChaosMonkeyService extends AbstractService {
   protected static final Logger log =
       LoggerFactory.getLogger(ChaosMonkeyService.class);
+  public static final int PERCENT_1 = 100;
+  public static final double PERCENT_1D = 100.0;
+  
+  /**
+   * the percentage value as multiplied up
+   */
+  public static final int PERCENT_100 = 100 * PERCENT_1;
   private final MetricRegistry metrics;
   private final QueueAccess queues;
+  private final Random random = new Random();
 
   private static final List<ChaosEntry> chaosEntries =
       new ArrayList<ChaosEntry>();
@@ -50,14 +58,19 @@ public class ChaosMonkeyService extends AbstractService {
 
 
   public synchronized void addTarget(String name,
-      ChaosTarget target,
-      long probability) {
-
+      ChaosTarget target, long probability) {
+    log.info("Adding {} with probability {}", name, probability / PERCENT_1);
     chaosEntries.add(new ChaosEntry(name, target, probability, metrics));
   }
 
+  /**
+   * Iterate through all the entries and invoke chaos on those wanted
+   */
   public void play() {
-
+    for (ChaosEntry chaosEntry : chaosEntries) {
+      long p = random.nextInt(PERCENT_100);
+      chaosEntry.maybeInvokeChaos(p);
+    }
   }
 
   public RenewingAction<MonkeyPlayAction> getChaosAction(long time, TimeUnit timeUnit) {

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
index 8ccd463..0fdba6b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
@@ -29,17 +29,20 @@ import org.apache.slider.server.appmaster.operations.RMOperationHandler
 @Slf4j
 class MockRMOperationHandler extends RMOperationHandler {
   public List<AbstractRMOperation> operations = [];
-  
+  int requests, releases;
+
   @Override
   public void releaseAssignedContainer(ContainerId containerId) {
     operations.add(new ContainerReleaseOperation(containerId))
     log.info("Releasing container ID " + containerId.getId())
+    releases++;
   }
 
   @Override
   public void addContainerRequest(AMRMClient.ContainerRequest req) {
     operations.add(new ContainerRequestOperation(req))
     log.info("Requesting container role #" + req.priority);
+    requests++;
   }
 
   /**
@@ -47,5 +50,7 @@ class MockRMOperationHandler extends RMOperationHandler {
    */
   public void clear() {
     operations.clear()
+    releases = 0;
+    requests = 0;
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
new file mode 100644
index 0000000..c789011
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.monkey
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.server.appmaster.actions.ActionHalt
+import org.apache.slider.server.appmaster.actions.ActionKillContainer
+import org.apache.slider.server.appmaster.actions.QueueService
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler
+import org.apache.slider.server.appmaster.monkey.ChaosKillAM
+import org.apache.slider.server.appmaster.monkey.ChaosKillContainer
+import org.apache.slider.server.appmaster.monkey.ChaosMonkeyService
+import org.apache.slider.server.appmaster.monkey.ChaosTarget
+import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.junit.Before
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+class TestMockMonkey extends BaseMockAppStateTest {
+
+  /**
+   * This queue service is NOT started; tests need to poll the queue
+   * rather than expect them to execute
+   */
+  QueueService queues = new QueueService();
+  ChaosMonkeyService monkey = new ChaosMonkeyService(metricRegistry,
+  queues)
+
+  @Before
+  public void init() {
+    def configuration = new YarnConfiguration()
+    queues.init(configuration)
+    monkey.init(configuration)
+  }
+  
+  @Test
+  public void testMonkeyStart() throws Throwable {
+    monkey.start()
+    monkey.stop()
+  }
+
+
+  @Test
+  public void testMonkeyPlay() throws Throwable {
+    ChaosCounter counter = new ChaosCounter()
+    monkey.addTarget("target", counter, ChaosMonkeyService.PERCENT_100)
+    
+    monkey.play()
+    assert counter.count == 1
+  }
+
+  @Test
+  public void testMonkeyPlaySometimes() throws Throwable {
+    ChaosCounter counter = new ChaosCounter()
+    ChaosCounter counter2 = new ChaosCounter()
+    monkey.addTarget("target1", counter, ChaosMonkeyService.PERCENT_1 * 50)
+    monkey.addTarget("target2", counter2, ChaosMonkeyService.PERCENT_1 * 25)
+
+    for (int i = 0; i < 100; i++) {
+      monkey.play()
+    }
+    log.info("Counter1 = ${counter.count} counter2 = ${counter2.count}")
+    /*
+     * Relying on probability here to give approximate answers 
+     */
+    assert counter.count > 25 
+    assert counter.count < 75 
+    assert counter2.count < counter.count 
+  }
+
+  @Test
+  public void testAMKiller() throws Throwable {
+
+    def chaos = new ChaosKillAM(queues, -1)
+    chaos.chaosAction();
+    assert queues.scheduledActions.size() == 1
+    def action = queues.scheduledActions.take()
+    assert action instanceof ActionHalt
+  }
+  
+  
+  @Test
+  public void testContainerKillerEmptyApp() throws Throwable {
+
+    
+    def chaos = new ChaosKillContainer(appState,
+        queues,
+        new MockRMOperationHandler())
+    chaos.chaosAction();
+    assert queues.scheduledActions.size() == 0
+  }
+  
+   
+  
+  @Test
+  public void testContainerKiller() throws Throwable {
+    MockRMOperationHandler ops = new MockRMOperationHandler();
+    role0Status.desired = 1
+    List<RoleInstance> instances = createAndStartNodes()
+    assert instances.size() == 1
+    def instance = instances[0]
+    
+    def chaos = new ChaosKillContainer(appState, queues, ops)
+    chaos.chaosAction();
+    assert queues.scheduledActions.size() == 1
+    def action = queues.scheduledActions.take()
+    ActionKillContainer killer = (ActionKillContainer) action
+    assert killer.containerId == instance.containerId;
+    killer.execute(null, queues, appState)
+    assert ops.releases == 1;
+
+    ContainerReleaseOperation operation = (ContainerReleaseOperation) ops.operations[0]
+    assert operation.containerId == instance.containerId
+  }
+  
+  
+
+  /**
+   * Chaos target that just implement a counter
+   */
+  private static class ChaosCounter implements ChaosTarget {
+    int count;
+    
+    @Override
+    void chaosAction() {
+      count++;
+    }
+
+
+    @Override
+    public String toString() {
+      final StringBuilder sb = new StringBuilder(
+          "ChaosCounter{");
+      sb.append("count=").append(count);
+      sb.append('}');
+      return sb.toString();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
index d7b328f..28b484f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
@@ -18,6 +18,7 @@
 
 package org.apache.slider.test
 
+import com.codahale.metrics.MetricRegistry
 import groovy.transform.CompileStatic
 import org.apache.hadoop.fs.FileUtil
 import org.apache.slider.common.SliderXMLConfKeysForTesting
@@ -38,7 +39,11 @@ import java.nio.file.Files
 @CompileStatic
 public abstract class SliderTestBase extends SliderTestUtils {
 
-
+  /**
+   * Singleton metric registry
+   */
+  public static final MetricRegistry metricRegistry = new MetricRegistry()
+  
   @Rule
   public TestName methodName = new TestName();
 


[43/50] [abbrv] git commit: SLIDER-331 fixed incorrect headers, added missing dependencies

Posted by st...@apache.org.
SLIDER-331 fixed incorrect headers, added missing dependencies


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/80eec9b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/80eec9b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/80eec9b1

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 80eec9b105968f5c13f97155f1d241067a6c924d
Parents: a34cca8
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Mon Aug 18 17:07:34 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Mon Aug 18 17:07:34 2014 -0700

----------------------------------------------------------------------
 app-packages/accumulo/pom.xml                   | 11 +++++++++
 pom.xml                                         | 24 ++++++++++++++++---
 slider-assembly/pom.xml                         | 14 -----------
 slider-core/pom.xml                             | 16 ++++++++++++-
 .../accumulo/accumulo-funtests/pom.xml          | 23 ++++++++++++++++++
 slider-providers/hbase/hbase-funtests/pom.xml   |  5 ++++
 src/test/clusters/c6401/slider/log4j.properties | 25 +++++++++++---------
 .../clusters/morzine/slider/log4j.properties    | 25 +++++++++++---------
 .../clusters/offline/slider/log4j.properties    | 25 +++++++++++---------
 .../clusters/remote/slider/log4j.properties     | 25 +++++++++++---------
 .../clusters/sandbox/slider/log4j.properties    | 25 +++++++++++---------
 11 files changed, 145 insertions(+), 73 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80eec9b1/app-packages/accumulo/pom.xml
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml
index 801d90f..02bf668 100644
--- a/app-packages/accumulo/pom.xml
+++ b/app-packages/accumulo/pom.xml
@@ -169,6 +169,12 @@
     </dependency>
     <dependency>
       <groupId>org.apache.slider</groupId>
+      <artifactId>slider-core</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.slider</groupId>
       <artifactId>slider-funtest</artifactId>
       <scope>test</scope>
     </dependency>
@@ -177,6 +183,11 @@
       <artifactId>groovy-all</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
 </project>

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80eec9b1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 223ff2b..22f56b1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -145,9 +145,11 @@
     <jackson.version>1.9.13</jackson.version>
     <jcommander.version>1.30</jcommander.version>
 
+    <jetty.version>6.1.26</jetty.version>
     <jersey.version>1.9</jersey.version>
     <servlet-api.version>2.5</servlet-api.version>
     <jsr311-api.version>1.1.1</jsr311-api.version>
+    <jaxb-api.version>2.2.7</jaxb-api.version>
 
     <junit.version>4.11</junit.version>
     <log4j.version>1.2.17</log4j.version>
@@ -1119,6 +1121,12 @@
       </dependency>
 
       <dependency>
+        <groupId>javax.xml.bind</groupId>
+        <artifactId>jaxb-api</artifactId>
+        <version>${jaxb-api.version}</version>
+      </dependency>
+
+      <dependency>
         <groupId>com.sun.jersey</groupId>
         <artifactId>jersey-client</artifactId>
         <version>${jersey.version}</version>
@@ -1205,9 +1213,19 @@
       <!-- ======================================================== -->
 
       <dependency>
-	      <groupId>org.mortbay.jetty</groupId>
-	      <artifactId>jetty-sslengine</artifactId>
-        <version>6.1.26</version>
+        <groupId>org.mortbay.jetty</groupId>
+        <artifactId>jetty</artifactId>
+        <version>${jetty.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.mortbay.jetty</groupId>
+        <artifactId>jetty-util</artifactId>
+        <version>${jetty.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.mortbay.jetty</groupId>
+        <artifactId>jetty-sslengine</artifactId>
+        <version>${jetty.version}</version>
       </dependency>
 
     </dependencies>

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80eec9b1/slider-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/slider-assembly/pom.xml b/slider-assembly/pom.xml
index d5299ee..e9defb5 100644
--- a/slider-assembly/pom.xml
+++ b/slider-assembly/pom.xml
@@ -141,26 +141,12 @@
         </configuration>
         <executions>
           <execution>
-            <id>clean</id>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <phase>pre-clean</phase>
-          </execution>
-          <execution>
             <id>default</id>
             <goals>
               <goal>enforce</goal>
             </goals>
             <phase>validate</phase>
           </execution>
-          <execution>
-            <id>site</id>
-            <goals>
-              <goal>enforce</goal>
-            </goals>
-            <phase>pre-site</phase>
-          </execution>
         </executions>
       </plugin>
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80eec9b1/slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index d403034..b2bdeed 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -406,6 +406,11 @@
     </dependency>
 
     <dependency>
+      <groupId>javax.xml.bind</groupId>
+      <artifactId>jaxb-api</artifactId>
+    </dependency>
+
+    <dependency>
       <groupId>com.sun.jersey</groupId>
       <artifactId>jersey-client</artifactId>
     </dependency>
@@ -466,8 +471,17 @@
 
     <dependency>
       <groupId>org.mortbay.jetty</groupId>
+      <artifactId>jetty</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.mortbay.jetty</groupId>
+      <artifactId>jetty-util</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.mortbay.jetty</groupId>
       <artifactId>jetty-sslengine</artifactId>
-      <scope>compile</scope>
     </dependency>
 
   </dependencies>

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80eec9b1/slider-providers/accumulo/accumulo-funtests/pom.xml
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/accumulo-funtests/pom.xml b/slider-providers/accumulo/accumulo-funtests/pom.xml
index 0f11618..73d9ae0 100644
--- a/slider-providers/accumulo/accumulo-funtests/pom.xml
+++ b/slider-providers/accumulo/accumulo-funtests/pom.xml
@@ -143,6 +143,16 @@
     <dependency>
       <groupId>org.apache.slider</groupId>
       <artifactId>slider-core</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.slider</groupId>
+      <artifactId>slider-core</artifactId>
       <version>${project.version}</version>
       <type>test-jar</type>
       <scope>test</scope>
@@ -207,9 +217,22 @@
 
     <dependency>
       <groupId>org.apache.accumulo</groupId>
+      <artifactId>accumulo-fate</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.accumulo</groupId>
       <artifactId>accumulo-test</artifactId>
       <scope>test</scope>
     </dependency>
+
+    <dependency>
+      <groupId>org.apache.thrift</groupId>
+      <artifactId>libthrift</artifactId>
+      <version>0.9.0</version>
+      <scope>test</scope>
+    </dependency>
     
   </dependencies>
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80eec9b1/slider-providers/hbase/hbase-funtests/pom.xml
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/pom.xml b/slider-providers/hbase/hbase-funtests/pom.xml
index d5e9d0e..d143408 100644
--- a/slider-providers/hbase/hbase-funtests/pom.xml
+++ b/slider-providers/hbase/hbase-funtests/pom.xml
@@ -140,6 +140,11 @@
     <dependency>
       <groupId>org.apache.slider</groupId>
       <artifactId>slider-core</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.slider</groupId>
+      <artifactId>slider-core</artifactId>
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80eec9b1/src/test/clusters/c6401/slider/log4j.properties
----------------------------------------------------------------------
diff --git a/src/test/clusters/c6401/slider/log4j.properties b/src/test/clusters/c6401/slider/log4j.properties
index d814f14..4682a96 100644
--- a/src/test/clusters/c6401/slider/log4j.properties
+++ b/src/test/clusters/c6401/slider/log4j.properties
@@ -1,15 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#   
-#    http://www.apache.org/licenses/LICENSE-2.0
-#   
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License. See accompanying LICENSE file.
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 #
 # log4j configuration used during build and unit tests
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80eec9b1/src/test/clusters/morzine/slider/log4j.properties
----------------------------------------------------------------------
diff --git a/src/test/clusters/morzine/slider/log4j.properties b/src/test/clusters/morzine/slider/log4j.properties
index d814f14..4682a96 100644
--- a/src/test/clusters/morzine/slider/log4j.properties
+++ b/src/test/clusters/morzine/slider/log4j.properties
@@ -1,15 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#   
-#    http://www.apache.org/licenses/LICENSE-2.0
-#   
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License. See accompanying LICENSE file.
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 #
 # log4j configuration used during build and unit tests
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80eec9b1/src/test/clusters/offline/slider/log4j.properties
----------------------------------------------------------------------
diff --git a/src/test/clusters/offline/slider/log4j.properties b/src/test/clusters/offline/slider/log4j.properties
index d814f14..4682a96 100644
--- a/src/test/clusters/offline/slider/log4j.properties
+++ b/src/test/clusters/offline/slider/log4j.properties
@@ -1,15 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#   
-#    http://www.apache.org/licenses/LICENSE-2.0
-#   
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License. See accompanying LICENSE file.
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 #
 # log4j configuration used during build and unit tests
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80eec9b1/src/test/clusters/remote/slider/log4j.properties
----------------------------------------------------------------------
diff --git a/src/test/clusters/remote/slider/log4j.properties b/src/test/clusters/remote/slider/log4j.properties
index f672472..5b59190 100644
--- a/src/test/clusters/remote/slider/log4j.properties
+++ b/src/test/clusters/remote/slider/log4j.properties
@@ -1,15 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#   
-#    http://www.apache.org/licenses/LICENSE-2.0
-#   
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License. See accompanying LICENSE file.
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 #
 # log4j configuration used during build and unit tests
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80eec9b1/src/test/clusters/sandbox/slider/log4j.properties
----------------------------------------------------------------------
diff --git a/src/test/clusters/sandbox/slider/log4j.properties b/src/test/clusters/sandbox/slider/log4j.properties
index d814f14..4682a96 100644
--- a/src/test/clusters/sandbox/slider/log4j.properties
+++ b/src/test/clusters/sandbox/slider/log4j.properties
@@ -1,15 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#   
-#    http://www.apache.org/licenses/LICENSE-2.0
-#   
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License. See accompanying LICENSE file.
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 #
 # log4j configuration used during build and unit tests
 


[34/50] [abbrv] git commit: SLIDER-320 agent provider registers endpoints and triggers a publish operation, as does app state on an NM container start event. Nothing happens at the moment.

Posted by st...@apache.org.
SLIDER-320 agent provider registers endpoints and triggers a publish operation, as does app state on an NM container start event. Nothing happens at the moment.


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/31bfae26
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/31bfae26
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/31bfae26

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 31bfae26a5ee44794cd72fd681dbf69d237e18f8
Parents: 7e2ce27
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 15 15:18:42 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 15 15:18:42 2014 +0100

----------------------------------------------------------------------
 .../providers/agent/AgentProviderService.java   | 32 +++++++++++++--
 .../server/appmaster/SliderAppMaster.java       | 24 +++++++++--
 .../actions/PublishRegistryDetails.java         |  3 +-
 .../slider/server/appmaster/state/AppState.java | 43 +++++++++++++++++---
 .../appmaster/state/ProviderAppState.java       |  5 +++
 .../server/appmaster/state/RoleInstance.java    | 42 +++++++++++++++++--
 .../state/StateAccessForProviders.java          |  7 ++++
 .../providers/hbase/HBaseProviderService.java   | 11 ++---
 8 files changed, 144 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/31bfae26/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 91bc0a6..680074d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -42,6 +42,7 @@ import org.apache.slider.core.conf.ConfTreeOperations;
 import org.apache.slider.core.conf.MapOperations;
 import org.apache.slider.core.exceptions.BadCommandArgumentsException;
 import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.NoSuchNodeException;
 import org.apache.slider.core.exceptions.SliderException;
 import org.apache.slider.core.launch.CommandLineBuilder;
 import org.apache.slider.core.launch.ContainerLauncher;
@@ -62,7 +63,9 @@ import org.apache.slider.providers.agent.application.metadata.Metainfo;
 import org.apache.slider.providers.agent.application.metadata.OSPackage;
 import org.apache.slider.providers.agent.application.metadata.OSSpecific;
 import org.apache.slider.server.appmaster.actions.ProviderReportedContainerLoss;
+import org.apache.slider.server.appmaster.actions.PublishRegistryDetails;
 import org.apache.slider.server.appmaster.state.ContainerPriority;
+import org.apache.slider.server.appmaster.state.RoleInstance;
 import org.apache.slider.server.appmaster.state.StateAccessForProviders;
 import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType;
 import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
@@ -96,6 +99,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_AGENTS;
@@ -541,14 +545,36 @@ public class AgentProviderService extends AbstractProviderService implements
                                      String roleName,
                                      String containerId,
                                      Map<String, String> ports) {
+    RoleInstance instance;
+    try {
+      instance = getAmState().getOwnedContainer(containerId);
+    } catch (NoSuchNodeException e) {
+      log.warn("Failed to locate instance of container {}: {}", containerId, e);
+      instance = null;
+    }
     for (Map.Entry<String, String> port : ports.entrySet()) {
-      log.info("Recording allocated port for {} as {}", port.getKey(), port.getValue());
-      this.getAllocatedPorts().put(port.getKey(), port.getValue());
-      this.getAllocatedPorts(containerId).put(port.getKey(), port.getValue());
+      String portname = port.getKey();
+      String portNo = port.getValue();
+      log.info("Recording allocated port for {} as {}", portname, portNo);
+      this.getAllocatedPorts().put(portname, portNo);
+      this.getAllocatedPorts(containerId).put(portname, portNo);
+        if (instance!=null) {
+          try {
+            instance.registerPortEndpoint(Integer.valueOf(portNo), portname, "");
+          } catch (NumberFormatException e) {
+            log.warn("Failed to parse {}: {}", portNo, e);
+          }
+        }
     }
 
     // component specific publishes
     processAndPublishComponentSpecificData(ports, containerId, fqdn, roleName);
+    
+    // and update registration entries
+    if (instance != null) {
+      queueAccess.put(new PublishRegistryDetails(instance.getId(), 0,
+          TimeUnit.MILLISECONDS));
+    }
   }
 
   private void updateComponentStatusWithAgentState(

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/31bfae26/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 9a16188..c54a268 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -84,7 +84,6 @@ import org.apache.slider.core.exceptions.SliderException;
 import org.apache.slider.core.exceptions.SliderInternalStateException;
 import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
 import org.apache.slider.core.main.ExitCodeProvider;
-import org.apache.slider.core.main.LauncherExitCodes;
 import org.apache.slider.core.main.RunService;
 import org.apache.slider.core.main.ServiceLauncher;
 import org.apache.slider.core.persist.ConfTreeSerDeser;
@@ -99,6 +98,7 @@ import org.apache.slider.providers.SliderProviderFactory;
 import org.apache.slider.providers.slideram.SliderAMClientProvider;
 import org.apache.slider.providers.slideram.SliderAMProviderService;
 import org.apache.slider.server.appmaster.actions.ActionKillContainer;
+import org.apache.slider.server.appmaster.actions.PublishRegistryDetails;
 import org.apache.slider.server.appmaster.actions.QueueExecutor;
 import org.apache.slider.server.appmaster.actions.ActionHalt;
 import org.apache.slider.server.appmaster.actions.QueueService;
@@ -901,6 +901,20 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
   }
 
   /**
+   * Register/re-register a component (that is already in the app state
+   * @param id the component
+   */
+  public boolean registerComponent(ContainerId id) {
+    RoleInstance instance = appState.getOwnedContainer(id);
+    if (instance == null) {
+      return false;
+    }
+    // this is where component registrations will go
+
+    return true;
+  }
+  
+  /**
    * looks for a specific case where a token file is provided as an environment
    * variable, yet the file is not there.
    * 
@@ -1192,9 +1206,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
     throws IOException, SliderInternalStateException, BadConfigException {
 
     appState.updateResourceDefinitions(resources);
-    appState.resetFailureCounts();
-    // reset the scheduled window resetter...the values
+
+    // reset the scheduled windows...the values
     // may have changed
+    appState.resetFailureCounts();
     
 
 
@@ -1665,6 +1680,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
       //trigger an async container status
       nmClientAsync.getContainerStatusAsync(containerId,
                                             cinfo.container.getNodeId());
+      // push out a registration
+      queue(new PublishRegistryDetails(containerId, 0, TimeUnit.MILLISECONDS));
+      
     } else {
       //this is a hypothetical path not seen. We react by warning
       log.error("Notified of started container that isn't pending {} - releasing",

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/31bfae26/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java
index f96183f..cb6d140 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java
@@ -31,7 +31,7 @@ public class PublishRegistryDetails extends AsyncAction {
 
   public PublishRegistryDetails(ContainerId containerId, long delay,
       TimeUnit timeUnit) {
-    super("PublishEndpointDetails :" + containerId.toString(),
+    super("PublishRegistryDetails :" + containerId.toString(),
         delay, timeUnit);
     this.containerId = containerId;
   }
@@ -40,6 +40,5 @@ public class PublishRegistryDetails extends AsyncAction {
   public void execute(SliderAppMaster appMaster,
       QueueAccess queueService,
       AppState appState) throws Exception {
-    
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/31bfae26/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 2f2eadd..07976ef 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -850,20 +850,53 @@ public class AppState {
 
   /**
    * Lookup live instance by string value of container ID
-   * @param containerId container ID
+   * @param containerId container ID as a string
    * @return the role instance for that container
    * @throws NoSuchNodeException if it does not exist
    */
   public synchronized RoleInstance getLiveInstanceByContainerID(String containerId)
-    throws NoSuchNodeException {
+      throws NoSuchNodeException {
     Collection<RoleInstance> nodes = getLiveNodes().values();
+    return findNodeInCollection(containerId, nodes);
+  }
+
+  /**
+   * Lookup owned instance by string value of container ID
+   * @param containerId container ID as a string
+   * @return the role instance for that container
+   * @throws NoSuchNodeException if it does not exist
+   */
+  public synchronized RoleInstance getOwnedInstanceByContainerID(String containerId)
+      throws NoSuchNodeException {
+    Collection<RoleInstance> nodes = ownedContainers.values();
+    return findNodeInCollection(containerId, nodes);
+  }
+
+  
+  
+  /**
+   * Iterate through a collection of role instances to find one with a
+   * specific (string) container ID
+   * @param containerId container ID as a string
+   * @param nodes collection
+   * @return 
+   * @throws NoSuchNodeException if there was no match
+   */
+  private RoleInstance findNodeInCollection(String containerId,
+      Collection<RoleInstance> nodes) throws NoSuchNodeException {
+    RoleInstance found = null;
     for (RoleInstance node : nodes) {
       if (containerId.equals(node.id)) {
-        return node;
+        found = node;
+        break;
       }
     }
-    //at this point: no node
-    throw new NoSuchNodeException(containerId);
+    if (found != null) {
+      return found;
+    } else {
+      //at this point: no node
+      throw new NoSuchNodeException(containerId);
+    }
   }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/31bfae26/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
index e7384c8..a0871ae 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
@@ -179,6 +179,11 @@ public class ProviderAppState implements StateAccessForProviders {
   }
 
   @Override
+  public RoleInstance getOwnedContainer(String id) throws NoSuchNodeException {
+    return appState.getOwnedInstanceByContainerID(id);
+  }
+
+  @Override
   public List<RoleInstance> cloneLiveContainerInfoList() {
     return appState.cloneLiveContainerInfoList();
   }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/31bfae26/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
index 8b6ed8e..e373843 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
@@ -25,8 +25,12 @@ import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.slider.api.ClusterDescription;
 import org.apache.slider.api.proto.Messages;
 import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.registry.info.RegisteredEndpoint;
 
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 
 /**
  * Tracking information about a container
@@ -82,11 +86,12 @@ public final class RoleInstance implements Cloneable {
   public String host;
   public String hostURL;
 
+
   /**
-   * Any information the provider wishes to retain on the state of
-   * an instance
+   * A list of registered endpoints.
    */
-  public Object providerInfo;
+  private List<RegisteredEndpoint> endpoints =
+      new ArrayList<RegisteredEndpoint>(2);
 
   public RoleInstance(Container container) {
     Preconditions.checkNotNull(container, "Null container");
@@ -188,8 +193,39 @@ public final class RoleInstance implements Cloneable {
   @Override
   public Object clone() throws CloneNotSupportedException {
     RoleInstance cloned = (RoleInstance) super.clone();
+    // clone the endpoint list, but not the values
+    cloned.endpoints = new ArrayList<RegisteredEndpoint>(this.endpoints);
     return cloned;
   }
 
+  /**
+   * Get the list of endpoints. 
+   * @return the endpoint list.
+   */
+  public List<RegisteredEndpoint> getEndpoints() {
+    return endpoints;
+  }
+
+  /**
+   * Add an endpoint registration
+   * @param endpoint
+   */
+  public void addEndpoint(RegisteredEndpoint endpoint) {
+    Preconditions.checkArgument(endpoint != null);
+    endpoints.add(endpoint);
+  }
 
+  /**
+   * Register a port endpoint as an inet-addr formatted endpoint, using the
+   * hostname as the first part of the address
+   * @param port
+   * @param protocol
+   * @param text
+   */
+  public void registerPortEndpoint(int port, String protocol, String text) {
+    InetSocketAddress addr = new InetSocketAddress(host, port);
+    RegisteredEndpoint epr = new RegisteredEndpoint(addr, protocol, text);
+    addEndpoint(epr);
+  }
+  
 }

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/31bfae26/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
index b236dfc..1714f75 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
@@ -160,6 +160,13 @@ public interface StateAccessForProviders {
   RoleInstance getOwnedContainer(ContainerId id);
 
   /**
+   * Get any active container with the given ID
+   * @param id container Id
+   * @return the active container or null if it is not found
+   */
+  RoleInstance getOwnedContainer(String id) throws NoSuchNodeException;
+
+  /**
    * Create a clone of the list of live cluster nodes.
    * @return the list of nodes, may be empty
    */

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/31bfae26/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index 1fb392c..82e535f 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -68,11 +68,8 @@ import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_
  * This class implements the server-side aspects
  * of an HBase Cluster
  */
-public class HBaseProviderService extends AbstractProviderService implements
-                                                                  ProviderCore,
-                                                                  HBaseKeys,
-    SliderKeys,
-    AgentRestOperations{
+public class HBaseProviderService extends AbstractProviderService 
+    implements ProviderCore, HBaseKeys, SliderKeys, AgentRestOperations{
 
   protected static final Logger log =
     LoggerFactory.getLogger(HBaseProviderService.class);
@@ -109,8 +106,8 @@ public class HBaseProviderService extends AbstractProviderService implements
    * @param instanceDefinition the instance definition to validate
    */
   @Override // Client and Server
-  public void validateInstanceDefinition(AggregateConf instanceDefinition) throws
-      SliderException {
+  public void validateInstanceDefinition(AggregateConf instanceDefinition) 
+      throws SliderException {
     clientProvider.validateInstanceDefinition(instanceDefinition);
   }
 


[20/50] [abbrv] git commit: Merge branch 'feature/SLIDER-322_Slider_to_briefly_support_Java_1.6' into develop

Posted by st...@apache.org.
Merge branch 'feature/SLIDER-322_Slider_to_briefly_support_Java_1.6' into develop


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/966c8c83
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/966c8c83
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/966c8c83

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 966c8c83d84bf0a127b646608821ca9316b179ac
Parents: 9aa0299 b6782e1
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 14 18:23:46 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 14 18:23:46 2014 +0100

----------------------------------------------------------------------
 pom.xml                                         |   8 +-
 .../src/main/java/org/apache/slider/Slider.java |   2 +-
 .../apache/slider/api/ClusterDescription.java   |  14 +--
 .../org/apache/slider/client/SliderClient.java  | 100 +++++++++++++++++--
 .../slider/client/SliderYarnClientImpl.java     |   8 +-
 .../common/params/AbstractActionArgs.java       |   6 +-
 .../params/AppAndResouceOptionArgsDelegate.java |   8 +-
 .../org/apache/slider/common/params/ArgOps.java |   8 +-
 .../apache/slider/common/params/CommonArgs.java |   4 +-
 .../common/params/DontSplitArguments.java       |   2 +-
 .../slider/common/tools/ConfigHelper.java       |  60 +++++++----
 .../slider/common/tools/CoreFileSystem.java     |   2 +-
 .../apache/slider/common/tools/SliderUtils.java |  10 +-
 .../org/apache/slider/core/conf/ConfTree.java   |   6 +-
 .../slider/core/conf/ConfTreeOperations.java    |   4 +-
 .../apache/slider/core/conf/MapOperations.java  |   2 +-
 .../slider/core/launch/AbstractLauncher.java    |  10 +-
 .../core/launch/ClasspathConstructor.java       |   2 +-
 .../slider/core/launch/CommandLineBuilder.java  |   2 +-
 .../slider/core/main/ServiceLauncher.java       |   2 +-
 .../slider/core/persist/JsonSerDeser.java       |  27 ++++-
 .../registry/docstore/PublishedConfigSet.java   |   2 +-
 .../docstore/PublishedConfiguration.java        |   4 +-
 .../PublishedConfigurationOutputter.java        |  12 +++
 .../slider/core/registry/docstore/UriMap.java   |   2 +-
 .../slider/core/registry/info/RegistryView.java |   3 +-
 .../core/registry/info/ServiceInstanceData.java |   4 +-
 .../apache/slider/core/zk/ZKIntegration.java    |   2 +-
 .../apache/slider/core/zk/ZookeeperUtils.java   |   6 +-
 .../providers/AbstractProviderService.java      |   4 +-
 .../apache/slider/providers/ProviderUtils.java  |   2 +-
 .../providers/agent/AgentClientProvider.java    |   2 +-
 .../providers/agent/AgentLaunchParameter.java   |   2 +-
 .../providers/agent/AgentProviderService.java   |  47 +++++----
 .../providers/agent/ComponentCommandOrder.java  |   9 +-
 .../agent/application/metadata/Application.java |   8 +-
 .../agent/application/metadata/Component.java   |   2 +-
 .../slideram/SliderAMClientProvider.java        |   4 +-
 .../slideram/SliderAMProviderService.java       |   2 +-
 .../server/appmaster/SliderAppMaster.java       |  37 +++++--
 .../appmaster/actions/ActionKillContainer.java  |   2 +-
 .../server/appmaster/actions/QueueService.java  |   6 +-
 .../appmaster/monkey/ChaosMonkeyService.java    |   2 +-
 .../slider/server/appmaster/state/AppState.java |  56 +++++------
 .../MostRecentContainerReleaseSelector.java     |   2 +-
 .../server/appmaster/state/NodeInstance.java    |   2 +-
 .../slider/server/appmaster/state/NodeMap.java  |   2 +-
 .../state/OutstandingRequestTracker.java        |   6 +-
 .../appmaster/state/ProviderAppState.java       |   4 +-
 .../server/appmaster/state/RoleHistory.java     |  15 ++-
 .../server/appmaster/state/RoleStatus.java      |   2 +-
 .../server/appmaster/web/SliderAMWebApp.java    |   2 +-
 .../web/SliderAmFilterInitializer.java          |   2 +-
 .../server/appmaster/web/SliderAmIpFilter.java  |   8 +-
 .../server/appmaster/web/WebAppApiImpl.java     |   2 +-
 .../web/rest/publisher/PublisherResource.java   |   4 +-
 .../slider/server/avro/RoleHistoryWriter.java   |   6 +-
 .../server/servicemonitor/MonitorUtils.java     |   2 +-
 .../services/curator/RegistryBinderService.java |   6 +-
 .../registry/RegistryRestResources.java         |   3 +-
 .../registry/SliderRegistryService.java         |   2 +-
 .../agent/TestAgentProviderService.java         |  45 +++++----
 .../agent/TestComponentInstanceState.java       |  11 +-
 .../providers/agent/TestHeartbeatMonitor.java   |   4 +-
 .../management/TestAMManagementWebServices.java |   9 +-
 .../publisher/TestAgentProviderService.java     |   2 +-
 .../workflow/ProcessCommandFactory.java         |   6 +-
 .../slider/test/MiniZooKeeperCluster.java       |   6 +-
 .../accumulo/AccumuloProviderService.java       |  19 +++-
 .../providers/hbase/HBaseClientProvider.java    |   6 +-
 .../providers/hbase/HBaseProviderService.java   |  31 +++++-
 71 files changed, 476 insertions(+), 240 deletions(-)
----------------------------------------------------------------------



[48/50] [abbrv] git commit: SLIDER-306 changes pulled in, reviewed tests set up and working

Posted by st...@apache.org.
SLIDER-306 changes pulled in, reviewed tests set up and working


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fa289e8d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fa289e8d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fa289e8d

Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: fa289e8d700aee1fad27670fd6875468038bafde
Parents: 7b589d9
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 19 12:52:16 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 19 12:52:16 2014 +0100

----------------------------------------------------------------------
 .../org/apache/slider/client/SliderClient.java  | 60 +++++++++++++-------
 .../apache/slider/common/tools/SliderUtils.java |  6 +-
 .../slider/agent/actions/TestActionList.groovy  |  9 +--
 .../agent/freezethaw/TestFreezeCommands.groovy  |  3 +-
 .../standalone/TestStandaloneAMDestroy.groovy   | 16 ++++++
 .../slider/client/TestCommonArgParsing.groovy   |  3 +-
 6 files changed, 66 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fa289e8d/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index d608278..dc186ce 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1420,29 +1420,42 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
     String user = UserGroupInformation.getCurrentUser().getUserName();
     List<ApplicationReport> instances = listSliderInstances(user);
     SliderUtils.sortApplicationReport(instances);
+    boolean live = args.live;
+    boolean history = args.history;
     if (isUnset(clustername)) {
+      // no cluster name: list all
       log.info("Instances for {}: {}",
-          (user != null ? user : "all users"),
-          instances.size());
+               (user != null ? user : "all users"),
+               instances.size());
       for (ApplicationReport report : instances) {
-        logAppReport(report, args.live, args.history);
+        logAppReport(report, live, history);
       }
+      // and always succeed
       return EXIT_SUCCESS;
     } else {
+      // cluster name provided
       SliderUtils.validateClusterName(clustername);
-      log.debug("Listing cluster named {}", clustername);
-      if (args.history) {
+      log.debug("Listing cluster named {}, live={}, history={}",
+          clustername, live, history);
+      boolean instanceFound = false;
+      if (history) {
         for (ApplicationReport report : instances) {
           if (report.getName().equals(clustername)) {
-            logAppReport(report, args.live, args.history);
+            logAppReport(report, live, true);
+            instanceFound = true;
           }
         }
-        return EXIT_SUCCESS;
+      } else {
+        // no history flag, only list live value
+        ApplicationReport report =
+            findClusterInInstanceList(instances, clustername);
+        if (report != null) {
+          logAppReport(report, true, true);
+          instanceFound = true;
+        }
       }
-      ApplicationReport report = findClusterInInstanceList(instances,
-          clustername);
-      if (report != null) {
-        logAppReport(report, true, false);
+      // exit code if the instance was found
+      if (instanceFound) {
         return EXIT_SUCCESS;
       } else {
         throw unknownClusterException(clustername);
@@ -1453,21 +1466,30 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
   /**
    * Log the application report at INFO
    * @param report report to log
+   * @param live only list live apps
+   * @param history list historical containers
    */
   public void logAppReport(ApplicationReport report, boolean live, boolean history) {
-    boolean active = report.getYarnApplicationState() == YarnApplicationState.RUNNING
-                || report.getYarnApplicationState() ==
-                   YarnApplicationState.ACCEPTED;
-    if (active && live && !history) {
-      log.info(SliderUtils.appReportToString(report, "\n"));
-    } else if (!active && history && !live) {
-      log.info(SliderUtils.appReportToString(report, "\n"));
-    } else {
+    // app is active if it is accepted or running
+    boolean active = isApplicationActive(report);
+    
+    boolean toLog = (active && live) || (!active && history);
+    if (toLog) {
       log.info(SliderUtils.appReportToString(report, "\n"));
     }
   }
 
   /**
+   * Is an application active: accepted or running
+   * @param report the application report
+   * @return true if it is running or scheduled to run.
+   */
+  public boolean isApplicationActive(ApplicationReport report) {
+    return report.getYarnApplicationState() == YarnApplicationState.RUNNING
+                || report.getYarnApplicationState() == YarnApplicationState.ACCEPTED;
+  }
+
+  /**
    * Implement the islive action: probe for a cluster of the given name existing
    * @return exit code
    */

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fa289e8d/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 3f83855..f740d3d 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -571,9 +571,9 @@ public final class SliderUtils {
    * Sorts the given list of application reports
    * Finished instances are ordered by finished time and running/accepted instances are
    * ordered by start time
-   * Finally Instance are order by finished instances and running instances
+   * Finally Instance are order by finished instances coming after running instances
    *
-   * @param instances
+   * @param instances list of intances
    */
   public static void sortApplicationReport(List<ApplicationReport> instances) {
     if (instances.size() <= 1) {
@@ -610,8 +610,8 @@ public final class SliderUtils {
       Collections.sort(nonLiveInstance, nonLiveInstanceComparator);
     }
     instances.clear();
-    instances.addAll(nonLiveInstance);
     instances.addAll(liveInstance);
+    instances.addAll(nonLiveInstance);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fa289e8d/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
index 5ccaeca..b80658d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
@@ -173,15 +173,11 @@ class TestActionList extends AgentMiniClusterTestBase {
     
     args.live = true;
     args.history = false;
-    try {
-      int exitCode = sliderClient.actionList(clustername, args);
-      fail("expected an exception, got a status code $exitCode")
-    } catch (UnknownApplicationInstanceException expected) {
-    }
+    assert 0 == sliderClient.actionList(clustername, args);
 
     // historical list will work
     args.history = true;
-    assert sliderClient.actionList(clustername, args) == 0;
+    assert 0 == sliderClient.actionList(clustername, args)
 
     // thaw
     ActionThawArgs thawArgs = new ActionThawArgs();
@@ -192,7 +188,6 @@ class TestActionList extends AgentMiniClusterTestBase {
     args.live = true;
     args.history = false;
     assert 0 == sliderClient.actionList(clustername, args);
-    assert launcher.serviceExitCode == 0
 
     //Listing all the instance both history (previously freezed instance) and live
     args.live = true

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fa289e8d/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy
index f5eff25..14541f3 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy
@@ -107,7 +107,8 @@ class TestFreezeCommands extends AgentMiniClusterTestBase {
     ServiceLauncher thawCommand = execSliderCommand(conf, commands);
     assertSucceeded(thawCommand)
     assertSucceeded(execSliderCommand(conf,
-        [SliderActions.ACTION_LIST, clustername]))
+        [SliderActions.ACTION_LIST, clustername, 
+         Arguments.ARG_LIVE]))
     assertSucceeded(execSliderCommand(conf,
         [SliderActions.ACTION_EXISTS, clustername]))
 

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fa289e8d/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
index 17e4ff2..07da54b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
@@ -57,6 +57,22 @@ class TestStandaloneAMDestroy extends AgentMiniClusterTestBase {
         ])
     assert launcher1.serviceExitCode == 0
 
+    // try to list it and expect failures
+    try {
+      launchClientAgainstMiniMR(
+          configuration,
+          [
+              SliderActions.ACTION_LIST,
+              "no-cluster-of-this-name",
+              Arguments.ARG_LIVE, Arguments.ARG_HISTORY
+          ])
+      fail("expected a failure")
+    } catch (UnknownApplicationInstanceException e) {
+      assertExceptionDetails(e,
+          SliderExitCodes.EXIT_UNKNOWN_INSTANCE,
+          ErrorStrings.E_UNKNOWN_INSTANCE)
+    }
+    
     ServiceLauncher<SliderClient> launcher = createStandaloneAM(
         clustername,
         true,

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fa289e8d/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy
index d94cd54..2083576 100644
--- a/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy
@@ -74,7 +74,8 @@ class TestCommonArgParsing implements SliderActions, Arguments {
 
   @Test
   public void testSliderBasePath() throws Throwable {
-    ClientArgs clientArgs = createClientArgs([ACTION_LIST, "--basepath", "/projects/slider/clusters"])
+    ClientArgs clientArgs = createClientArgs([ACTION_LIST,
+        ARG_BASE_PATH,  "/projects/slider/clusters"])
     assert clientArgs.basePath == new Path("/projects/slider/clusters")
   }