You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/08/19 14:22:15 UTC
[01/50] [abbrv] git commit: rm unused import from SliderAppMaster
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options 258fb54c0 -> e8f772e7a (forced update)
rm unused import from SliderAppMaster
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d38b31a7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d38b31a7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d38b31a7
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: d38b31a7a0fc65309dd5723d7baf61093cd63d53
Parents: 7091397
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 12 17:30:50 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 12 17:30:50 2014 +0100
----------------------------------------------------------------------
.../java/org/apache/slider/server/appmaster/SliderAppMaster.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d38b31a7/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 9f4567d..9a2851a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -132,7 +132,6 @@ import org.apache.slider.server.services.utility.WebAppService;
import org.apache.slider.server.services.workflow.ServiceThreadFactory;
import org.apache.slider.server.services.workflow.WorkflowExecutorService;
import org.apache.slider.server.services.workflow.WorkflowRpcService;
-import org.apache.slider.server.services.workflow.WorkflowScheduledExecutorService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
[36/50] [abbrv] git commit: SLIDER-320 TestAgentEcho starts two
containers
Posted by st...@apache.org.
SLIDER-320 TestAgentEcho starts two containers
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4610c25e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4610c25e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4610c25e
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 4610c25e9f57ad980447bdd83f1d2f1563be467b
Parents: 01c7390
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 15 15:33:51 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 15 15:33:51 2014 +0100
----------------------------------------------------------------------
.../groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4610c25e/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
index 304815e..a29c8cb 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
@@ -66,7 +66,7 @@ class TestAgentEcho extends AgentTestBase {
def role = "echo"
Map<String, Integer> roles = [
- (role): 1,
+ (role): 2,
];
ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
roles,
[47/50] [abbrv] git commit: SLIDER-306 test checks are not consistent
with current listing semantics: succeeds if live or not live cluster is in
the history
Posted by st...@apache.org.
SLIDER-306 test checks are not consistent with current listing semantics: succeeds if live or not live cluster is in the history
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7b589d98
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7b589d98
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7b589d98
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 7b589d986e468b83669860e807b74bff0136125b
Parents: f06b37c
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 19 12:38:26 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 19 12:38:26 2014 +0100
----------------------------------------------------------------------
.../slider/agent/actions/TestActionList.groovy | 58 +++++++++++++++++++-
1 file changed, 55 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7b589d98/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
index b7196ac..5ccaeca 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
@@ -23,6 +23,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.agent.AgentMiniClusterTestBase
import org.apache.slider.client.SliderClient
+import org.apache.slider.common.params.ActionListArgs
+import org.apache.slider.common.params.ActionThawArgs
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
@@ -40,7 +42,7 @@ class TestActionList extends AgentMiniClusterTestBase {
@Before
public void setup() {
super.setup()
- createMiniCluster("", configuration, 1, false)
+ createMiniCluster("", configuration, 1, true)
}
/**
@@ -55,6 +57,7 @@ class TestActionList extends AgentMiniClusterTestBase {
testListAllUsersNoClusters()
testListLiveCluster()
testListMissingCluster()
+// testActionListHistory()
}
public void testListThisUserNoClusters() throws Throwable {
@@ -128,7 +131,7 @@ class TestActionList extends AgentMiniClusterTestBase {
}
public void testListMissingCluster() throws Throwable {
- describe("exec the status command against an unknown cluster")
+ describe("exec the list command against an unknown cluster")
ServiceLauncher<SliderClient> launcher
try {
@@ -138,7 +141,7 @@ class TestActionList extends AgentMiniClusterTestBase {
//varargs list of command line params
[
SliderActions.ACTION_LIST,
- createClusterName()
+ "no-instance"
]
)
fail("expected an exception, got a status code " + launcher.serviceExitCode)
@@ -148,4 +151,53 @@ class TestActionList extends AgentMiniClusterTestBase {
}
+ @Test
+ public void testActionListHistory() {
+ String clustername = createClusterName()
+ ServiceLauncher<SliderClient> launcher = createStandaloneAM(
+ clustername,
+ true,
+ true)
+ addToTeardown(launcher)
+ SliderClient sliderClient = launcher.service
+ waitForClusterLive(sliderClient)
+
+ ActionListArgs args = new ActionListArgs();
+ //Listing only live instances
+ args.live = true;
+ assert sliderClient.actionList(clustername, args) == 0;
+ clusterActionFreeze(sliderClient, clustername, "stopping first cluster")
+ waitForAppToFinish(sliderClient)
+
+ //Listing only live instances but prints nothing since instance is freezed/stopped
+
+ args.live = true;
+ args.history = false;
+ try {
+ int exitCode = sliderClient.actionList(clustername, args);
+ fail("expected an exception, got a status code $exitCode")
+ } catch (UnknownApplicationInstanceException expected) {
+ }
+
+ // historical list will work
+ args.history = true;
+ assert sliderClient.actionList(clustername, args) == 0;
+
+ // thaw
+ ActionThawArgs thawArgs = new ActionThawArgs();
+ sliderClient.actionThaw(clustername, thawArgs);
+ waitForClusterLive(sliderClient)
+
+ //Listing only live instances
+ args.live = true;
+ args.history = false;
+ assert 0 == sliderClient.actionList(clustername, args);
+ assert launcher.serviceExitCode == 0
+
+ //Listing all the instance both history (previously freezed instance) and live
+ args.live = true
+ args.history = true
+ assert 0 == sliderClient.actionList("", args);
+ }
+
}
[03/50] [abbrv] git commit: SLIDER-202 initial chaos monkey
Posted by st...@apache.org.
SLIDER-202 initial chaos monkey
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c951e251
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c951e251
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c951e251
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: c951e2519e3d4327bce04bc2ee826d3140f8c0ed
Parents: 8d4217e
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 13 10:18:01 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 13 10:18:01 2014 +0100
----------------------------------------------------------------------
.../server/appmaster/SliderAppMaster.java | 6 ++
.../server/appmaster/monkey/ChaosEntry.java | 35 ++++++++++++
.../appmaster/monkey/ChaosMonkeyService.java | 60 ++++++++++++++++++++
.../server/appmaster/monkey/ChaosTarget.java | 24 ++++++++
4 files changed, 125 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c951e251/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 9a2851a..1135a58 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -18,6 +18,7 @@
package org.apache.slider.server.appmaster;
+import com.codahale.metrics.MetricRegistry;
import com.google.protobuf.BlockingService;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -196,6 +197,11 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
public static final int NUM_RPC_HANDLERS = 5;
public static final int SCHEDULED_EXECUTOR_POOL_SIZE = 1;
+ /**
+ * Singleton of metrics registry
+ */
+ public static final MetricRegistry metrics = new MetricRegistry();
+
/** YARN RPC to communicate with the Resource Manager or Node Manager */
private YarnRPC yarnRPC;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c951e251/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
new file mode 100644
index 0000000..779cca1
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.monkey;
+
+/**
+ * Entry in the chaos list
+ */
+public class ChaosEntry {
+
+ public final String name;
+ public final ChaosTarget target;
+ public final long probability;
+
+ public ChaosEntry(String name, ChaosTarget target, long probability) {
+ this.name = name;
+ this.target = target;
+ this.probability = probability;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c951e251/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
new file mode 100644
index 0000000..7c5638c
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.monkey;
+
+import com.codahale.metrics.MetricRegistry;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.slider.server.appmaster.actions.QueueAccess;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ChaosMonkeyService extends AbstractService {
+ protected static final Logger log =
+ LoggerFactory.getLogger(ChaosMonkeyService.class);
+ private final MetricRegistry metrics;
+ private final QueueAccess queues;
+
+ private static final List<ChaosEntry> chaosEntries =
+ new ArrayList<ChaosEntry>();
+
+ public ChaosMonkeyService(MetricRegistry metrics, QueueAccess queues) {
+ super("ChaosMonkeyService");
+ this.metrics = metrics;
+ this.queues = queues;
+ }
+
+
+ @Override
+ protected void serviceStart() throws Exception {
+ super.serviceStart();
+
+ }
+
+
+ public synchronized void addTarget(ChaosTarget target, long probability) {
+ Preconditions.checkArgument(target != null, "null target");
+ Preconditions.checkArgument(probability > 0, "negative probability");
+ Preconditions.checkArgument(probability > 10000, "probability over 100%");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c951e251/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosTarget.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosTarget.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosTarget.java
new file mode 100644
index 0000000..1c3a9ac
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosTarget.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.monkey;
+
+public interface ChaosTarget {
+
+ public void chaosAction();
+}
[39/50] [abbrv] git commit: SLIDER-324. Exports should be attributed
to individual master components rather than only one master component
Posted by st...@apache.org.
SLIDER-324. Exports should be attributed to individual master components rather than only one master component
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2a25ab91
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2a25ab91
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2a25ab91
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 2a25ab9192bc96a2f6f4409f9eea916e94c9484c
Parents: bc9561d
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Fri Aug 15 17:18:52 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Fri Aug 15 17:18:52 2014 -0700
----------------------------------------------------------------------
app-packages/storm/metainfo.xml | 4 +-
.../providers/agent/AgentProviderService.java | 216 +++++++++++--------
.../application/metadata/MetainfoParser.java | 8 +-
.../agent/TestAgentProviderService.java | 34 ++-
.../metadata/MetainfoParserTest.java | 59 -----
.../metadata/TestMetainfoParser.java | 63 ++++++
6 files changed, 221 insertions(+), 163 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2a25ab91/app-packages/storm/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index d9004ad..dbe8549 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -45,7 +45,7 @@
<value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value>
</export>
<export>
- <name>nimbus.url</name>
+ <name>nimbus.host_port</name>
<value>http://${NIMBUS_HOST}:${site.storm-site.nimbus.thrift.port}</value>
</export>
</exports>
@@ -82,7 +82,7 @@
<name>NIMBUS</name>
<category>MASTER</category>
<autoStartOnFailure>true</autoStartOnFailure>
- <appExports>QuickLinks-nimbus.url,QuickLinks-ganglia.ui,QuickLinks-app.metrics</appExports>
+ <appExports>QuickLinks-nimbus.host_port,QuickLinks-ganglia.ui,QuickLinks-app.metrics</appExports>
<commandScript>
<script>scripts/nimbus.py</script>
<scriptType>PYTHON</scriptType>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2a25ab91/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 91bc0a6..842f4f6 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -56,6 +56,7 @@ import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderUtils;
import org.apache.slider.providers.agent.application.metadata.Application;
import org.apache.slider.providers.agent.application.metadata.Component;
+import org.apache.slider.providers.agent.application.metadata.ComponentExport;
import org.apache.slider.providers.agent.application.metadata.Export;
import org.apache.slider.providers.agent.application.metadata.ExportGroup;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
@@ -137,6 +138,8 @@ public class AgentProviderService extends AbstractProviderService implements
new ConcurrentHashMap<String, ComponentInstanceState>();
private final Map<String, Map<String, String>> componentInstanceData =
new ConcurrentHashMap<String, Map<String, String>>();
+ private final Map<String, Map<String, String>> exportGroups =
+ new ConcurrentHashMap<String, Map<String, String>>();
private final Map<String, Map<String, String>> allocatedPorts =
new ConcurrentHashMap<String, Map<String, String>>();
private final Map<String, String> workFolders =
@@ -451,22 +454,8 @@ public class AgentProviderService extends AbstractProviderService implements
Boolean isMaster = isMaster(roleName);
ComponentInstanceState componentStatus = getComponentStatuses().get(label);
componentStatus.heartbeat(System.currentTimeMillis());
- // If no Master can explicitly publish then publish if its a master
- // Otherwise, wait till the master that can publish is ready
- if (isMaster &&
- (canAnyMasterPublishConfig() == false || canPublishConfig(roleName))) {
- publishConfigAndExportGroups(heartBeat, componentStatus);
- } else {
- // Ack that config has been reported
- List<ComponentStatus> statuses = heartBeat.getComponentStatus();
- if (statuses != null && !statuses.isEmpty()) {
- ComponentStatus status = statuses.get(0);
- if(status.getConfigs().size() > 0) {
- log.info("Config got reported by {} but discarded as component {} cannot publish.", label, roleName);
- componentStatus.setConfigReported(true);
- }
- }
- }
+
+ publishConfigAndExportGroups(heartBeat, componentStatus, roleName);
List<CommandReport> reports = heartBeat.getReports();
if (reports != null && !reports.isEmpty()) {
@@ -513,6 +502,7 @@ public class AgentProviderService extends AbstractProviderService implements
}
}
}
+
// if there is no outstanding command then retrieve config
if (isMaster && componentStatus.getState() == State.STARTED
&& command == Command.NOP) {
@@ -758,39 +748,60 @@ public class AgentProviderService extends AbstractProviderService implements
/**
* Process return status for component instances
+ *
* @param heartBeat
* @param componentStatus
*/
- protected void publishConfigAndExportGroups(HeartBeat heartBeat, ComponentInstanceState componentStatus) {
+ protected void publishConfigAndExportGroups(
+ HeartBeat heartBeat, ComponentInstanceState componentStatus, String roleName) {
List<ComponentStatus> statuses = heartBeat.getComponentStatus();
if (statuses != null && !statuses.isEmpty()) {
log.info("Processing {} status reports.", statuses.size());
- Application application = getMetainfo().getApplication();
- List<ExportGroup> exportGroups = application.getExportGroups();
- boolean hasExportGroups = exportGroups != null && !exportGroups.isEmpty();
- Set<String> exportedConfigs = new HashSet();
- String exportedConfigsStr = application.getExportedConfigs();
- boolean exportedAllConfigs = exportedConfigsStr == null || exportedConfigsStr.isEmpty();
- if (application.getExportedConfigs() != null && application.getExportedConfigs().length() > 0) {
- for (String exportedConfig : application.getExportedConfigs().split(",")) {
- if (exportedConfig.trim().length() > 0) {
- exportedConfigs.add(exportedConfig.trim());
- }
- }
- }
-
for (ComponentStatus status : statuses) {
log.info("Status report: " + status.toString());
+
if (status.getConfigs() != null) {
- for (String key : status.getConfigs().keySet()) {
- if ((!exportedAllConfigs && exportedConfigs.contains(key)) ||
- exportedAllConfigs) {
- Map<String, String> configs = status.getConfigs().get(key);
- publishApplicationInstanceData(key, key, configs.entrySet());
+ Application application = getMetainfo().getApplication();
+
+ if (canAnyMasterPublishConfig() == false || canPublishConfig(roleName)) {
+ // If no Master can explicitly publish then publish if its a master
+ // Otherwise, wait till the master that can publish is ready
+
+ Set<String> exportedConfigs = new HashSet();
+ String exportedConfigsStr = application.getExportedConfigs();
+ boolean exportedAllConfigs = exportedConfigsStr == null || exportedConfigsStr.isEmpty();
+ if (!exportedAllConfigs) {
+ for (String exportedConfig : exportedConfigsStr.split(",")) {
+ if (exportedConfig.trim().length() > 0) {
+ exportedConfigs.add(exportedConfig.trim());
+ }
+ }
+ }
+
+ for (String key : status.getConfigs().keySet()) {
+ if ((!exportedAllConfigs && exportedConfigs.contains(key)) ||
+ exportedAllConfigs) {
+ Map<String, String> configs = status.getConfigs().get(key);
+ publishApplicationInstanceData(key, key, configs.entrySet());
+ }
}
}
- if (hasExportGroups) {
+ List<ExportGroup> exportGroups = application.getExportGroups();
+ boolean hasExportGroups = exportGroups != null && !exportGroups.isEmpty();
+
+ Set<String> appExports = new HashSet();
+ String appExportsStr = getApplicationComponent(roleName).getAppExports();
+ boolean hasNoAppExports = appExportsStr == null || appExportsStr.isEmpty();
+ if (!hasNoAppExports) {
+ for (String appExport : appExportsStr.split(",")) {
+ if (appExport.trim().length() > 0) {
+ appExports.add(appExport.trim());
+ }
+ }
+ }
+
+ if (hasExportGroups && appExports.size() > 0) {
String configKeyFormat = "${site.%s.%s}";
String hostKeyFormat = "${%s_HOST}";
@@ -809,33 +820,63 @@ public class AgentProviderService extends AbstractProviderService implements
}
}
+ Set<String> modifiedGroups = new HashSet<String>();
for (ExportGroup exportGroup : exportGroups) {
List<Export> exports = exportGroup.getExports();
if (exports != null && !exports.isEmpty()) {
String exportGroupName = exportGroup.getName();
- Map<String, String> map = new HashMap<String, String>();
+ Map<String, String> map = getCurrentExports(exportGroupName);
for (Export export : exports) {
- String value = export.getValue();
- // replace host names
- for (String token : replaceTokens.keySet()) {
- if (value.contains(token)) {
- value = value.replace(token, replaceTokens.get(token));
+ if (canBeExported(exportGroupName, export.getName(), appExports)) {
+ String value = export.getValue();
+ // replace host names
+ for (String token : replaceTokens.keySet()) {
+ if (value.contains(token)) {
+ value = value.replace(token, replaceTokens.get(token));
+ }
}
+ map.put(export.getName(), value);
+ log.info("Preparing to publish. Key {} and Value {}", export.getName(), value);
}
- map.put(export.getName(), value);
- log.info("Preparing to publish. Key {} and Value {}", export.getName(), value);
}
- publishApplicationInstanceData(exportGroupName, exportGroupName, map.entrySet());
+ modifiedGroups.add(exportGroupName);
}
}
+ publishModifiedExportGroups(modifiedGroups);
}
- log.info("Received and stored config for {}", heartBeat.getHostname());
+
+ log.info("Received and processed config for {}", heartBeat.getHostname());
componentStatus.setConfigReported(true);
+
}
}
}
}
+ private boolean canBeExported(String exportGroupName, String name, Set<String> appExports) {
+ return appExports.contains(String.format("%s-%s", exportGroupName, name));
+ }
+
+ protected Map<String, String> getCurrentExports(String groupName) {
+ if(!this.exportGroups.containsKey(groupName)) {
+ synchronized (this.exportGroups) {
+ if(!this.exportGroups.containsKey(groupName)) {
+ this.exportGroups.put(groupName, new ConcurrentHashMap<String, String>());
+ }
+ }
+ }
+
+ return this.exportGroups.get(groupName);
+ }
+
+ private void publishModifiedExportGroups(Set<String> modifiedGroups) {
+ synchronized (this.exportGroups) {
+ for(String groupName : modifiedGroups) {
+ publishApplicationInstanceData(groupName, groupName, this.exportGroups.get(groupName).entrySet());
+ }
+ }
+ }
+
/** Publish component instance specific data if the component demands it */
protected void processAndPublishComponentSpecificData(Map<String, String> ports,
String containerId,
@@ -848,9 +889,9 @@ public class AgentProviderService extends AbstractProviderService implements
Application application = getMetainfo().getApplication();
for (Component component : application.getComponents()) {
if (component.getName().equals(roleName)) {
- if (component.getExports().size() > 0) {
+ if (component.getComponentExports().size() > 0) {
- for (Export export : component.getExports()) {
+ for (ComponentExport export : component.getComponentExports()) {
String templateToExport = export.getValue();
for (String portName : ports.keySet()) {
boolean publishData = false;
@@ -901,26 +942,37 @@ public class AgentProviderService extends AbstractProviderService implements
}
/**
- * Extract script path from the application metainfo
- *
+ * Return Component based on name
* @param roleName
- *
* @return
*/
- protected String getScriptPathFromMetainfo(String roleName) {
- String scriptPath = null;
+ protected Component getApplicationComponent(String roleName) {
Application application = getMetainfo().getApplication();
if (application == null) {
log.error("Malformed app definition: Expect application as the top level element for metainfo.xml");
- return scriptPath;
- }
- for (Component component : application.getComponents()) {
- if (component.getName().equals(roleName)) {
- scriptPath = component.getCommandScript().getScript();
- break;
+ } else {
+ for (Component component : application.getComponents()) {
+ if (component.getName().equals(roleName)) {
+ return component;
+ }
}
}
- return scriptPath;
+ return null;
+ }
+
+ /**
+ * Extract script path from the application metainfo
+ *
+ * @param roleName
+ *
+ * @return
+ */
+ protected String getScriptPathFromMetainfo(String roleName) {
+ Component component = getApplicationComponent(roleName);
+ if (component != null) {
+ return component.getCommandScript().getScript();
+ }
+ return null;
}
/**
@@ -931,18 +983,10 @@ public class AgentProviderService extends AbstractProviderService implements
* @return
*/
protected boolean isMaster(String roleName) {
- Application application = getMetainfo().getApplication();
- if (application == null) {
- log.error("Malformed app definition: Expect application as the top level element for metainfo.xml");
- } else {
- for (Component component : application.getComponents()) {
- if (component.getName().equals(roleName)) {
- if (component.getCategory().equals("MASTER")) {
- return true;
- } else {
- return false;
- }
- }
+ Component component = getApplicationComponent(roleName);
+ if (component != null) {
+ if (component.getCategory().equals("MASTER")) {
+ return true;
}
}
return false;
@@ -956,34 +1000,24 @@ public class AgentProviderService extends AbstractProviderService implements
* @return
*/
protected boolean canPublishConfig(String roleName) {
- Application application = getMetainfo().getApplication();
- if (application == null) {
- log.error("Malformed app definition: Expect application as the top level element for metainfo.xml");
- } else {
- for (Component component : application.getComponents()) {
- if (component.getName().equals(roleName)) {
- return Boolean.TRUE.toString().equals(component.getPublishConfig());
- }
- }
+ Component component = getApplicationComponent(roleName);
+ if (component != null) {
+ return Boolean.TRUE.toString().equals(component.getPublishConfig());
}
return false;
}
/**
* Checks if the role is marked auto-restart
+ *
* @param roleName
+ *
* @return
*/
protected boolean isMarkedAutoRestart(String roleName) {
- Application application = getMetainfo().getApplication();
- if (application == null) {
- log.error("Malformed app definition: Expect application as the top level element for metainfo.xml");
- } else {
- for (Component component : application.getComponents()) {
- if (component.getName().equals(roleName)) {
- return component.getRequiresAutoRestart();
- }
- }
+ Component component = getApplicationComponent(roleName);
+ if (component != null) {
+ return component.getRequiresAutoRestart();
}
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2a25ab91/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
index 4b91ff7..bc93d6f 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
@@ -61,10 +61,10 @@ public class MetainfoParser {
digester.addBeanPropertySetter("*/component/maxInstanceCount");
digester.addBeanPropertySetter("*/component/autoStartOnFailure");
digester.addBeanPropertySetter("*/component/appExports");
- digester.addObjectCreate("*/component/componentExport", ComponentExport.class);
- digester.addBeanPropertySetter("*/component/componentExport/name");
- digester.addBeanPropertySetter("*/component/componentExport/value");
- digester.addSetNext("*/component/componentExport", "addComponentExport");
+ digester.addObjectCreate("*/componentExport", ComponentExport.class);
+ digester.addBeanPropertySetter("*/componentExport/name");
+ digester.addBeanPropertySetter("*/componentExport/value");
+ digester.addSetNext("*/componentExport", "addComponentExport");
digester.addSetNext("*/component", "addComponent");
digester.addObjectCreate("*/commandScript", CommandScript.class);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2a25ab91/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 1ea11a6..6ed950f 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -136,8 +136,18 @@ public class TestAgentProviderService {
+ " </commandOrders>\n"
+ " <components>\n"
+ " <component>\n"
+ + " <name>HBASE_REST</name>\n"
+ + " <category>MASTER</category>\n"
+ + " <commandScript>\n"
+ + " <script>scripts/hbase_rest.py</script>\n"
+ + " <scriptType>PYTHON</scriptType>\n"
+ + " <timeout>600</timeout>\n"
+ + " </commandScript>\n"
+ + " </component>\n"
+ + " <component>\n"
+ " <name>HBASE_MASTER</name>\n"
+ " <category>MASTER</category>\n"
+ + " <publishConfig>true</publishConfig>\n"
+ " <autoStartOnFailure>true</autoStartOnFailure>\n"
+ " <appExports>QuickLinks-JMX_Endpoint,QuickLinks-Master_Status</appExports>\n"
+ " <minInstanceCount>1</minInstanceCount>\n"
@@ -166,7 +176,7 @@ public class TestAgentProviderService {
+ " <name>PropertyB</name>\n"
+ " <value>AConstant</value>\n"
+ " </componentExport>\n"
- + " </componentExport>\n"
+ + " </componentExports>\n"
+ " </component>\n"
+ " </components>\n"
+ " <osSpecifics>\n"
@@ -458,15 +468,15 @@ public class TestAgentProviderService {
doReturn(metainfo).when(mockAps).getMetainfo();
doReturn(roleClusterNodeMap).when(mockAps).getRoleClusterNodeMapping();
- mockAps.publishConfigAndExportGroups(hb, componentStatus);
- assert componentStatus.getConfigReported() == true;
+ mockAps.publishConfigAndExportGroups(hb, componentStatus, "HBASE_MASTER");
+ Assert.assertTrue(componentStatus.getConfigReported());
ArgumentCaptor<Collection> entriesCaptor = ArgumentCaptor.
forClass(Collection.class);
Mockito.verify(mockAps, Mockito.times(3)).publishApplicationInstanceData(
anyString(),
anyString(),
entriesCaptor.capture());
- assert entriesCaptor.getAllValues().size() == 3;
+ Assert.assertEquals(3, entriesCaptor.getAllValues().size());
for (Collection coll : entriesCaptor.getAllValues()) {
Set<Map.Entry<String, String>> entrySet = (Set<Map.Entry<String, String>>) coll;
for (Map.Entry entry : entrySet) {
@@ -476,6 +486,16 @@ public class TestAgentProviderService {
}
}
}
+
+ Map<String, String> exports = mockAps.getCurrentExports("QuickLinks");
+ Assert.assertEquals(2, exports.size());
+ Assert.assertEquals(exports.get("JMX_Endpoint"), "http://HOST1:60012/jmx");
+
+ mockAps.publishConfigAndExportGroups(hb, componentStatus, "HBASE_REST");
+ Mockito.verify(mockAps, Mockito.times(3)).publishApplicationInstanceData(
+ anyString(),
+ anyString(),
+ entriesCaptor.capture());
}
@Test
@@ -487,7 +507,7 @@ public class TestAgentProviderService {
log.info("Service: " + application.toString());
Assert.assertEquals(application.getName(), "HBASE");
Assert.assertEquals(application.getExportedConfigs(), "hbase-site,global");
- Assert.assertEquals(application.getComponents().size(), 2);
+ Assert.assertEquals(application.getComponents().size(), 3);
List<Component> components = application.getComponents();
int found = 0;
for (Component component : components) {
@@ -596,9 +616,9 @@ public class TestAgentProviderService {
Assert.assertTrue(mockAps.isMaster(role_hm));
Assert.assertFalse(mockAps.isMaster(role_hrs));
- Assert.assertFalse(mockAps.canPublishConfig(role_hm));
+ Assert.assertTrue(mockAps.canPublishConfig(role_hm));
Assert.assertFalse(mockAps.canPublishConfig(role_hrs));
- Assert.assertFalse(mockAps.canAnyMasterPublishConfig());
+ Assert.assertTrue(mockAps.canAnyMasterPublishConfig());
Assert.assertTrue(mockAps2.isMaster(role_hm));
Assert.assertFalse(mockAps2.isMaster(role_hrs));
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2a25ab91/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/MetainfoParserTest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/MetainfoParserTest.java b/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/MetainfoParserTest.java
deleted file mode 100644
index 61c53df..0000000
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/MetainfoParserTest.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- *
- */
-public class MetainfoParserTest {
- protected static final Logger log =
- LoggerFactory.getLogger(MetainfoParserTest.class);
- public static final String METAINFO_XML =
- "/org/apache/slider/providers/agent/application/metadata/metainfo.xml";
-
- @Test
- public void testParse() throws IOException {
-
- InputStream resStream = this.getClass().getResourceAsStream(
- METAINFO_XML);
- MetainfoParser parser = new MetainfoParser();
- Metainfo metainfo = parser.parse(resStream);
- Assert.assertNotNull(metainfo);
- Assert.assertNotNull(metainfo.getApplication());
- Application application = metainfo.getApplication();
- assert "STORM".equals(application.getName());
- assert 5 == application.getComponents().size();
- OSPackage pkg = application.getOSSpecifics().get(0).getPackages().get(0);
- assert "tarball".equals(pkg.getType());
- assert "files/apache-storm-0.9.1.2.1.1.0-237.tar.gz".equals(pkg.getName());
- boolean found = false;
- for (Component comp : application.getComponents()) {
- if (comp != null && comp.getName().equals("NIMBUS")) {
- found = true;
- }
- }
- assert found;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2a25ab91/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java b/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java
new file mode 100644
index 0000000..98f0afb
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ *
+ */
+public class TestMetainfoParser {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestMetainfoParser.class);
+ public static final String METAINFO_XML =
+ "/org/apache/slider/providers/agent/application/metadata/metainfo.xml";
+
+ @Test
+ public void testParse() throws IOException {
+
+ InputStream resStream = this.getClass().getResourceAsStream(
+ METAINFO_XML);
+ MetainfoParser parser = new MetainfoParser();
+ Metainfo metainfo = parser.parse(resStream);
+ Assert.assertNotNull(metainfo);
+ Assert.assertNotNull(metainfo.getApplication());
+ Application application = metainfo.getApplication();
+ assert "STORM".equals(application.getName());
+ assert 5 == application.getComponents().size();
+ OSPackage pkg = application.getOSSpecifics().get(0).getPackages().get(0);
+ assert "tarball".equals(pkg.getType());
+ assert "files/apache-storm-0.9.1.2.1.1.0-237.tar.gz".equals(pkg.getName());
+ boolean found = false;
+ for (Component comp : application.getComponents()) {
+ if (comp != null && comp.getName().equals("NIMBUS")) {
+ found = true;
+ Assert.assertEquals(0, comp.getComponentExports().size());
+ }
+ if (comp != null && comp.getName().equals("SUPERVISOR")) {
+ Assert.assertEquals(1, comp.getComponentExports().size());
+ }
+ }
+ assert found;
+ }
+}
[49/50] [abbrv] git commit: SLIDER-306 list tests
Posted by st...@apache.org.
SLIDER-306 list tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9645d56e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9645d56e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9645d56e
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 9645d56e339bcc423c6c84bf4ceba8e549c45246
Parents: fa289e8
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 19 13:15:39 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 19 13:15:39 2014 +0100
----------------------------------------------------------------------
.../apache/slider/agent/actions/TestActionList.groovy | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9645d56e/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
index b80658d..787d96d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
@@ -57,7 +57,7 @@ class TestActionList extends AgentMiniClusterTestBase {
testListAllUsersNoClusters()
testListLiveCluster()
testListMissingCluster()
-// testActionListHistory()
+ testActionListHistory()
}
public void testListThisUserNoClusters() throws Throwable {
@@ -151,7 +151,6 @@ class TestActionList extends AgentMiniClusterTestBase {
}
- @Test
public void testActionListHistory() {
String clustername = createClusterName()
ServiceLauncher<SliderClient> launcher = createStandaloneAM(
@@ -173,9 +172,14 @@ class TestActionList extends AgentMiniClusterTestBase {
args.live = true;
args.history = false;
- assert 0 == sliderClient.actionList(clustername, args);
+ try {
+ sliderClient.actionList(clustername, args);
+ fail("expected a failure")
+ } catch (UnknownApplicationInstanceException e) {
+
+ }
- // historical list will work
+ // historical list will work
args.history = true;
assert 0 == sliderClient.actionList(clustername, args)
[28/50] [abbrv] git commit: fix pom.xml for UTF-8 inputs and outputs
(remove 1 maven warning)
Posted by st...@apache.org.
fix pom.xml for UTF-8 inputs and outputs (remove 1 maven warning)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/15da2799
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/15da2799
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/15da2799
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 15da2799ea6f6f5107d19ff8f693f47b5c8307ae
Parents: 599e972
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 15 13:06:31 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 15 13:06:31 2014 +0100
----------------------------------------------------------------------
pom.xml | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/15da2799/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 43f9231..223ff2b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -93,6 +93,8 @@
<properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<!--
Java versions
[50/50] [abbrv] git commit: SLIDER-306 list exit code/exception
depends on output of report is o
Posted by st...@apache.org.
SLIDER-306 list exit code/exception depends on output of report is o
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e8f772e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e8f772e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e8f772e7
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: e8f772e7a1411384c46fe9e0d2930aa0a1b54660
Parents: 9645d56
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 19 13:16:20 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 19 13:16:20 2014 +0100
----------------------------------------------------------------------
.../main/java/org/apache/slider/client/SliderClient.java | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e8f772e7/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index dc186ce..b0aa561 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1450,8 +1450,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
ApplicationReport report =
findClusterInInstanceList(instances, clustername);
if (report != null) {
- logAppReport(report, true, true);
- instanceFound = true;
+ instanceFound = logAppReport(report, true, false);
}
}
// exit code if the instance was found
@@ -1468,8 +1467,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @param report report to log
* @param live only list live apps
* @param history list historical containers
+ * @return whether the report was logged or not
*/
- public void logAppReport(ApplicationReport report, boolean live, boolean history) {
+ public boolean logAppReport(ApplicationReport report,
+ boolean live,
+ boolean history) {
// app is active if it is accepted or running
boolean active = isApplicationActive(report);
@@ -1477,6 +1479,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
if (toLog) {
log.info(SliderUtils.appReportToString(report, "\n"));
}
+ return toLog;
}
/**
[19/50] [abbrv] git commit: SLIDER-322 POM-> java6
Posted by st...@apache.org.
SLIDER-322 POM-> java6
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b6782e1d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b6782e1d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b6782e1d
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: b6782e1dc0f2393746b91170e5e02a0c07616d2c
Parents: 280afd8
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 14 17:47:24 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 14 17:47:24 2014 +0100
----------------------------------------------------------------------
pom.xml | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b6782e1d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0816479..43f9231 100644
--- a/pom.xml
+++ b/pom.xml
@@ -97,7 +97,7 @@
<!--
Java versions
-->
- <project.java.src.version>1.7</project.java.src.version>
+ <project.java.src.version>1.6</project.java.src.version>
<enforced.java.version>${project.java.src.version}</enforced.java.version>
<groovy.version>2.2.2</groovy.version>
@@ -227,8 +227,8 @@
<version>${maven-compiler-plugin.version}</version>
<configuration>
<compilerId>groovy-eclipse-compiler</compilerId>
- <source>1.7</source>
- <target>1.7</target>
+ <source>${project.java.src.version}</source>
+ <target>${project.java.src.version}</target>
<!-- set verbose to be true if you want lots of uninteresting messages -->
<!-- <verbose>true</verbose> -->
</configuration>
@@ -248,7 +248,7 @@
<configuration>
<rules>
<requireJavaVersion>
- <version>[1.7,)</version>
+ <version>[${project.java.src.version},)</version>
</requireJavaVersion>
</rules>
</configuration>
[04/50] [abbrv] git commit: SLIDER-202 Monkey implemented but not
conditional/integrated
Posted by st...@apache.org.
SLIDER-202 Monkey implemented but not conditional/integrated
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b111268a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b111268a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b111268a
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: b111268af07a75797c9f56b3ab96c0dd052a62a8
Parents: c951e25
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 13 12:00:27 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 13 12:00:27 2014 +0100
----------------------------------------------------------------------
.../server/appmaster/SliderAppMaster.java | 52 +++++++++++----
.../server/appmaster/actions/ActionHalt.java | 4 +-
.../appmaster/actions/ActionKillContainer.java | 51 +++++++++++++++
.../server/appmaster/monkey/ChaosEntry.java | 53 ++++++++++++++-
.../server/appmaster/monkey/ChaosKillAM.java | 48 ++++++++++++++
.../appmaster/monkey/ChaosKillContainer.java | 68 ++++++++++++++++++++
.../appmaster/monkey/ChaosMonkeyService.java | 41 ++++++++----
.../appmaster/monkey/MonkeyPlayAction.java | 48 ++++++++++++++
.../slider/server/appmaster/state/AppState.java | 14 +++-
.../standalone/TestStandaloneAgentAM.groovy | 2 +-
10 files changed, 346 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b111268a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 1135a58..ee0e59e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -97,6 +97,7 @@ import org.apache.slider.providers.ProviderService;
import org.apache.slider.providers.SliderProviderFactory;
import org.apache.slider.providers.slideram.SliderAMClientProvider;
import org.apache.slider.providers.slideram.SliderAMProviderService;
+import org.apache.slider.server.appmaster.actions.ActionKillContainer;
import org.apache.slider.server.appmaster.actions.QueueExecutor;
import org.apache.slider.server.appmaster.actions.ActionHalt;
import org.apache.slider.server.appmaster.actions.QueueService;
@@ -104,6 +105,9 @@ import org.apache.slider.server.appmaster.actions.ActionStopSlider;
import org.apache.slider.server.appmaster.actions.AsyncAction;
import org.apache.slider.server.appmaster.actions.RenewingAction;
import org.apache.slider.server.appmaster.actions.ResetFailureWindow;
+import org.apache.slider.server.appmaster.monkey.ChaosKillAM;
+import org.apache.slider.server.appmaster.monkey.ChaosKillContainer;
+import org.apache.slider.server.appmaster.monkey.ChaosMonkeyService;
import org.apache.slider.server.appmaster.operations.AsyncRMOperationHandler;
import org.apache.slider.server.appmaster.operations.ProviderNotifyingOperationHandler;
import org.apache.slider.server.appmaster.rpc.RpcBinder;
@@ -112,7 +116,6 @@ import org.apache.slider.server.appmaster.rpc.SliderClusterProtocolPBImpl;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
import org.apache.slider.server.appmaster.state.AppState;
import org.apache.slider.server.appmaster.state.ContainerAssignment;
-import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
import org.apache.slider.server.appmaster.state.ProviderAppState;
import org.apache.slider.server.appmaster.operations.RMOperationHandler;
import org.apache.slider.server.appmaster.state.RoleInstance;
@@ -147,7 +150,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
@@ -301,6 +303,11 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
private ContainerId appMasterContainerID;
/**
+ * Monkey Service -may be null
+ */
+ private ChaosMonkeyService monkey;
+
+ /**
* ProviderService of this cluster
*/
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
@@ -780,6 +787,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// starts the node review process
launchProviderService(instanceDefinition, confDir);
+ // chaos monkey
+ maybeStartMonkey();
try {
//now block waiting to be told to exit the process
@@ -1103,7 +1112,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
//for all the operations, exec them
- rmOperationHandler.execute(operations);
+ executeRMOperations(operations);
log.info("Diagnostics: " + getContainerDiagnosticInfo());
}
@@ -1221,7 +1230,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// tell the provider
providerRMOperationHandler.execute(allOperations);
//now apply the operations
- rmOperationHandler.execute(allOperations);
+ executeRMOperations(allOperations);
return !allOperations.isEmpty();
} catch (TriggerClusterTeardownException e) {
@@ -1237,7 +1246,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
*/
private void releaseAllContainers() {
//now apply the operations
- rmOperationHandler.execute(appState.releaseAllContainers());
+ executeRMOperations(appState.releaseAllContainers());
}
/**
@@ -1441,19 +1450,17 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//throws NoSuchNodeException if it is missing
RoleInstance instance =
appState.getLiveInstanceByContainerID(containerID);
- List<AbstractRMOperation> opsList =
- new LinkedList<>();
- ContainerReleaseOperation release =
- new ContainerReleaseOperation(instance.getId());
- opsList.add(release);
- //now apply the operations
- rmOperationHandler.execute(opsList);
+ queue(new ActionKillContainer(instance.getId(), 0, TimeUnit.MILLISECONDS));
Messages.KillContainerResponseProto.Builder builder =
Messages.KillContainerResponseProto.newBuilder();
builder.setSuccess(true);
return builder.build();
}
+ public void executeRMOperations(List<AbstractRMOperation> operations) {
+ rmOperationHandler.execute(operations);
+ }
+
@Override
public Messages.AMSuicideResponseProto amSuicide(
Messages.AMSuicideRequestProto request)
@@ -1462,7 +1469,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
String text = request.getText();
int delay = request.getDelay();
log.info("AM Suicide with signal {}, message {} delay = {}", signal, text, delay);
- ActionHalt action = new ActionHalt(signal, text, delay);
+ ActionHalt action = new ActionHalt(signal, text, delay,
+ TimeUnit.MILLISECONDS);
schedule(action);
return Messages.AMSuicideResponseProto.getDefaultInstance();
}
@@ -1537,7 +1545,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
containerId);
RoleInstance activeContainer = appState.getActiveContainer(containerId);
if (activeContainer != null) {
- rmOperationHandler.execute(appState.releaseContainer(containerId));
+ executeRMOperations(appState.releaseContainer(containerId));
// ask for more containers if needed
log.info("Container released; triggering review");
reviewRequestAndReleaseNodes();
@@ -1725,6 +1733,22 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
signalAMComplete(exitCode, exception.toString());
}
+ public boolean maybeStartMonkey() {
+ monkey = new ChaosMonkeyService(metrics, actionQueues);
+ int amKillProbability = 100;
+ int containerKillProbability = 200;
+ monkey.addTarget("AM killer",
+ new ChaosKillAM(this, actionQueues, -1),
+ amKillProbability);
+ monkey.addTarget("Container killer",
+ new ChaosKillContainer(this, actionQueues),
+ amKillProbability);
+ initAndAddService(monkey);
+ // and schedule it
+ schedule(monkey.getChaosAction(60, TimeUnit.SECONDS));
+ return true;
+ }
+
/**
* This is the main entry point for the service launcher.
* @param args command line arguments.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b111268a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
index b46a791..c21e249 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
@@ -22,6 +22,8 @@ import org.apache.hadoop.util.ExitUtil;
import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.state.AppState;
+import java.util.concurrent.TimeUnit;
+
/**
* Exit a JVM halt.
* @see ExitUtil#halt(int, String)
@@ -34,7 +36,7 @@ public class ActionHalt extends AsyncAction {
public ActionHalt(
int status,
String text,
- long delay) {
+ long delay, TimeUnit timeUnit) {
super("Halt", delay, ActionAttributes.HALTS_CLUSTER);
this.status = status;
this.text = text;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b111268a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
new file mode 100644
index 0000000..7668bd5
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
+import org.apache.slider.server.appmaster.state.AppState;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+public class ActionKillContainer extends AsyncAction {
+
+ private final ContainerId containerId;
+
+ public ActionKillContainer(ContainerId containerId, long delay,
+ TimeUnit timeUnit) {
+ super("kill container", delay, timeUnit);
+ this.containerId = containerId;
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster,
+ QueueAccess queueService,
+ AppState appState) throws Exception {
+ List<AbstractRMOperation> opsList = new LinkedList<>();
+ ContainerReleaseOperation release = new ContainerReleaseOperation(containerId);
+ opsList.add(release);
+ //now apply the operations
+ appMaster.executeRMOperations(opsList);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b111268a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
index 779cca1..13bda57 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
@@ -18,18 +18,67 @@
package org.apache.slider.server.appmaster.monkey;
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.MetricRegistry;
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* Entry in the chaos list
*/
public class ChaosEntry {
-
+
+ protected static final Logger log =
+ LoggerFactory.getLogger(ChaosEntry.class);
public final String name;
public final ChaosTarget target;
public final long probability;
- public ChaosEntry(String name, ChaosTarget target, long probability) {
+ private final MetricRegistry metrics;
+ private final Counter invocationCounter;
+
+
+ /**
+ * Constructor -includes validaton of all arguments
+ * @param name
+ * @param target
+ * @param probability
+ */
+ public ChaosEntry(String name, ChaosTarget target, long probability,
+ MetricRegistry metrics) {
+ Preconditions.checkArgument(!StringUtils.isEmpty(name), "missing name");
+ Preconditions.checkArgument(target != null, "null target");
+ Preconditions.checkArgument(probability > 0, "negative probability");
+ Preconditions.checkArgument(probability < 10000, "probability over 100%");
this.name = name;
this.target = target;
this.probability = probability;
+ this.metrics = metrics;
+ invocationCounter =
+ metrics.counter(MetricRegistry.name(ChaosEntry.class, name));
+ }
+
+ /**
+ * Trigger the chaos action
+ */
+ public void invokeChaos() {
+ log.info("Invoking {}", name);
+ invocationCounter.inc();
+ target.chaosAction();
+ }
+
+ /**
+ * Invoke Chaos if the trigger value is in range of the probability
+ * @param trigger trigger value, 0-10K
+ * @return true if the chaos method was invoked
+ */
+ public boolean maybeInvokeChaos(long trigger) {
+ if (probability < 0) {
+ invokeChaos();
+ return true;
+ }
+ return false;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b111268a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java
new file mode 100644
index 0000000..c464ac3
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.monkey;
+
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.actions.ActionHalt;
+import org.apache.slider.server.appmaster.actions.QueueAccess;
+
+import java.util.concurrent.TimeUnit;
+
+public class ChaosKillAM implements ChaosTarget {
+
+ private final SliderAppMaster appMaster;
+ private final QueueAccess queues;
+ private final int exitCode;
+
+ public ChaosKillAM(SliderAppMaster appMaster,
+ QueueAccess queues, int exitCode) {
+ this.appMaster = appMaster;
+ this.queues = queues;
+ this.exitCode = exitCode;
+ }
+
+ /**
+ * Trigger a delayed halt
+ */
+ @Override
+ public void chaosAction() {
+ queues.schedule(new ActionHalt(exitCode, "Chaos invoked halt", 1000,
+ TimeUnit.MILLISECONDS));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b111268a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
new file mode 100644
index 0000000..0ffcc8a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.monkey;
+
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.actions.ActionKillContainer;
+import org.apache.slider.server.appmaster.actions.QueueAccess;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Kill a container
+ */
+public class ChaosKillContainer implements ChaosTarget {
+ protected static final Logger log =
+ LoggerFactory.getLogger(ChaosKillContainer.class);
+ private final SliderAppMaster appMaster;
+ private final QueueAccess queues;
+ private final Random random = new Random();
+
+ public ChaosKillContainer(SliderAppMaster appMaster,
+ QueueAccess queues) {
+ this.appMaster = appMaster;
+ this.queues = queues;
+ }
+
+ /**
+ * Trigger a container kill halt
+ */
+
+ @Override
+ public void chaosAction() {
+ List<RoleInstance> liveContainers =
+ appMaster.getAppState().cloneLiveContainerInfoList();
+ int size = liveContainers.size();
+ if (size == 0) {
+ log.info("No containers to kill");
+ return;
+ }
+ int target = random.nextInt(size);
+ RoleInstance roleInstance = liveContainers.get(target);
+ log.info("Killing {}", roleInstance);
+
+ queues.schedule(new ActionKillContainer(roleInstance.getId(),
+ 100, TimeUnit.MILLISECONDS));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b111268a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
index 7c5638c..c525d02 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
@@ -19,15 +19,20 @@
package org.apache.slider.server.appmaster.monkey;
import com.codahale.metrics.MetricRegistry;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.service.AbstractService;
+import org.apache.slider.server.appmaster.actions.AsyncAction;
import org.apache.slider.server.appmaster.actions.QueueAccess;
+import org.apache.slider.server.appmaster.actions.RenewingAction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.TimeUnit;
+/**
+ * A chaos monkey service which will invoke ChaosTarget events
+ */
public class ChaosMonkeyService extends AbstractService {
protected static final Logger log =
LoggerFactory.getLogger(ChaosMonkeyService.class);
@@ -36,25 +41,33 @@ public class ChaosMonkeyService extends AbstractService {
private static final List<ChaosEntry> chaosEntries =
new ArrayList<ChaosEntry>();
-
+
public ChaosMonkeyService(MetricRegistry metrics, QueueAccess queues) {
super("ChaosMonkeyService");
this.metrics = metrics;
this.queues = queues;
}
-
- @Override
- protected void serviceStart() throws Exception {
- super.serviceStart();
-
+
+ public synchronized void addTarget(String name,
+ ChaosTarget target,
+ long probability) {
+
+ chaosEntries.add(new ChaosEntry(name, target, probability, metrics));
+ }
+
+ public void play() {
+
}
-
-
- public synchronized void addTarget(ChaosTarget target, long probability) {
- Preconditions.checkArgument(target != null, "null target");
- Preconditions.checkArgument(probability > 0, "negative probability");
- Preconditions.checkArgument(probability > 10000, "probability over 100%");
+
+ public RenewingAction<MonkeyPlayAction> getChaosAction(long time, TimeUnit timeUnit) {
+ RenewingAction<MonkeyPlayAction> action = new RenewingAction<>(
+ new MonkeyPlayAction(this, 0, TimeUnit.MILLISECONDS),
+ time,
+ time,
+ timeUnit,
+ 0
+ );
+ return action;
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b111268a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/MonkeyPlayAction.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/MonkeyPlayAction.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/MonkeyPlayAction.java
new file mode 100644
index 0000000..20e4466
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/MonkeyPlayAction.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.monkey;
+
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.actions.AsyncAction;
+import org.apache.slider.server.appmaster.actions.QueueAccess;
+import org.apache.slider.server.appmaster.state.AppState;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Queueable action which calls {@link ChaosMonkeyService#play()} when
+ * executed.
+ */
+public class MonkeyPlayAction extends AsyncAction {
+
+ private final ChaosMonkeyService monkey;
+
+ public MonkeyPlayAction(ChaosMonkeyService monkey, long delay,
+ TimeUnit timeUnit) {
+ super("chaos monkey", delay, timeUnit);
+ this.monkey = monkey;
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster,
+ QueueAccess queueService,
+ AppState appState) throws Exception {
+ monkey.play();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b111268a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 48f97bb..d616678 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -101,7 +101,7 @@ public class AppState {
* Flag set to indicate the application is live -this only happens
* after the buildInstance operation
*/
- boolean applicationLive = false;
+ private boolean applicationLive = false;
/**
* The definition of the instance. Flexing updates the resources section
@@ -811,6 +811,10 @@ public class AppState {
return activeContainers.remove(id);
}
+ /**
+ * Clone the live container list. This is synchronized.
+ * @return a snapshot of the live node list
+ */
public synchronized List<RoleInstance> cloneLiveContainerInfoList() {
List<RoleInstance> allRoleInstances;
Collection<RoleInstance> values = getLiveNodes().values();
@@ -818,8 +822,12 @@ public class AppState {
return allRoleInstances;
}
-
-
+ /**
+ * Lookup live instance by string value of container ID
+ * @param containerId container ID
+ * @return the role instance for that container
+ * @throws NoSuchNodeException if it does not exist
+ */
public synchronized RoleInstance getLiveInstanceByContainerID(String containerId)
throws NoSuchNodeException {
Collection<RoleInstance> nodes = getLiveNodes().values();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b111268a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
index 8498ee2..73e9b07 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
@@ -37,10 +37,10 @@ import org.junit.Test
@CompileStatic
@Slf4j
class TestStandaloneAgentAM extends AgentMiniClusterTestBase {
+
@Test
public void testStandaloneAgentAM() throws Throwable {
-
describe "create a masterless AM then get the service and look it up via the AM"
//launch fake master
[16/50] [abbrv] git commit: Add get-hbase-site.py - python version of
get-hbase-site.sh
Posted by st...@apache.org.
Add get-hbase-site.py - python version of get-hbase-site.sh
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9aa02997
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9aa02997
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9aa02997
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 9aa029973fffc7e9de80c1fd85b1c81b5c8ff36f
Parents: 52f1f6f
Author: tedyu <yu...@gmail.com>
Authored: Thu Aug 14 09:01:32 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Thu Aug 14 09:01:32 2014 -0700
----------------------------------------------------------------------
app-packages/hbase/get-hbase-site.py | 48 +++++++++++++++++++++++++++++++
1 file changed, 48 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9aa02997/app-packages/hbase/get-hbase-site.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/get-hbase-site.py b/app-packages/hbase/get-hbase-site.py
new file mode 100644
index 0000000..9760781
--- /dev/null
+++ b/app-packages/hbase/get-hbase-site.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""Gets hbase-site.xml from running HBase instance
+First argument is the name of cluster instance
+"""
+
+import sys
+import urllib2
+import subprocess
+
+f=subprocess.Popen("slider status "+sys.argv[1], shell=True, stdout=subprocess.PIPE).stdout
+for line in f:
+ pos = line.find("info.am.web.url")
+ if pos > 0 :
+ part = line[(pos+20) :]
+ endPos = part.find("\"")
+ url = part[: (endPos-1)]
+ url = url + "/ws/v1/slider/publisher/slider/hbase-site.xml"
+ print url
+ response = urllib2.urlopen(url)
+ html = response.read()
+
+ fout=open("hbase-site.xml", "w")
+ fout.write(html)
+ fout.close()
+ f.close()
+
+ sys.exit(0)
+
+print "info.am.web.url key was not found for " + sys.argv[1]
+sys.exit(1)
[31/50] [abbrv] git commit: Merge branch 'develop' into
feature/SLIDER-320-endpoints-in-roleinstances
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-320-endpoints-in-roleinstances
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/95dcbc9c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/95dcbc9c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/95dcbc9c
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 95dcbc9c670bf919016a5266d3100b4f2b14f2fa
Parents: 74cca8a 4ea66fa
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 15 14:05:41 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 15 14:05:41 2014 +0100
----------------------------------------------------------------------
app-packages/hbase/get-hbase-site.py | 48 ++++++
app-packages/memcached-win/appConfig.json | 2 +-
.../memcached-win/package/scripts/memcached.py | 5 +-
.../memcached-win/package/scripts/params.py | 1 +
app-packages/memcached/appConfig.json | 1 +
.../memcached/package/scripts/memcached.py | 5 +-
.../memcached/package/scripts/params.py | 2 +
app-packages/storm/appConfig.json | 6 +-
app-packages/storm/metainfo.xml | 12 ++
app-packages/storm/package/scripts/service.py | 24 +--
.../storm/package/scripts/status_params.py | 1 +
pom.xml | 10 +-
.../src/main/python/agent/ActionQueue.py | 26 ++-
.../src/main/python/agent/CommandStatusDict.py | 5 +
slider-agent/src/main/python/agent/Constants.py | 1 +
.../src/main/python/agent/Controller.py | 50 +++++-
.../python/agent/CustomServiceOrchestrator.py | 81 +++++++---
slider-agent/src/main/python/agent/Heartbeat.py | 15 +-
slider-agent/src/main/python/agent/Register.py | 7 +-
.../core/providers/system.py | 3 +-
.../core/providers/windows/system.py | 11 +-
.../core/resources/system.py | 4 +
.../python/resource_management/core/shell.py | 14 +-
.../src/test/python/agent/TestActionQueue.py | 10 +-
.../test/python/agent/TestCommandStatusDict.py | 94 +++++++++++
.../src/test/python/agent/TestController.py | 137 +++++++++++++++-
.../agent/TestCustomServiceOrchestrator.py | 18 ++-
.../src/test/python/agent/TestHeartbeat.py | 107 ++++++++++++-
.../src/test/python/agent/TestRegistration.py | 8 +-
.../src/main/java/org/apache/slider/Slider.java | 2 +-
.../apache/slider/api/ClusterDescription.java | 14 +-
.../java/org/apache/slider/api/OptionKeys.java | 1 -
.../org/apache/slider/client/SliderClient.java | 100 +++++++++++-
.../slider/client/SliderYarnClientImpl.java | 8 +-
.../common/params/AbstractActionArgs.java | 6 +-
.../params/AppAndResouceOptionArgsDelegate.java | 8 +-
.../org/apache/slider/common/params/ArgOps.java | 8 +-
.../apache/slider/common/params/CommonArgs.java | 4 +-
.../common/params/DontSplitArguments.java | 2 +-
.../slider/common/tools/ConfigHelper.java | 60 +++++--
.../slider/common/tools/CoreFileSystem.java | 4 +-
.../apache/slider/common/tools/SliderUtils.java | 10 +-
.../org/apache/slider/core/conf/ConfTree.java | 6 +-
.../slider/core/conf/ConfTreeOperations.java | 4 +-
.../apache/slider/core/conf/MapOperations.java | 2 +-
.../slider/core/launch/AbstractLauncher.java | 10 +-
.../core/launch/ClasspathConstructor.java | 2 +-
.../slider/core/launch/CommandLineBuilder.java | 2 +-
.../slider/core/main/ServiceLauncher.java | 2 +-
.../slider/core/persist/JsonSerDeser.java | 27 +++-
.../registry/docstore/PublishedConfigSet.java | 2 +-
.../docstore/PublishedConfiguration.java | 4 +-
.../PublishedConfigurationOutputter.java | 12 ++
.../slider/core/registry/docstore/UriMap.java | 2 +-
.../slider/core/registry/info/RegistryView.java | 3 +-
.../core/registry/info/ServiceInstanceData.java | 4 +-
.../apache/slider/core/zk/ZKIntegration.java | 2 +-
.../apache/slider/core/zk/ZookeeperUtils.java | 6 +-
.../providers/AbstractProviderService.java | 4 +-
.../apache/slider/providers/ProviderUtils.java | 2 +-
.../providers/agent/AgentClientProvider.java | 8 +-
.../providers/agent/AgentLaunchParameter.java | 2 +-
.../providers/agent/AgentProviderService.java | 159 +++++++++++++------
.../providers/agent/ComponentCommandOrder.java | 9 +-
.../agent/application/metadata/Application.java | 8 +-
.../agent/application/metadata/Component.java | 15 +-
.../application/metadata/MetainfoParser.java | 1 +
.../slideram/SliderAMClientProvider.java | 32 ++--
.../slideram/SliderAMProviderService.java | 2 +-
.../server/appmaster/SliderAppMaster.java | 37 +++--
.../appmaster/actions/ActionKillContainer.java | 2 +-
.../server/appmaster/actions/QueueService.java | 6 +-
.../appmaster/monkey/ChaosMonkeyService.java | 2 +-
.../slider/server/appmaster/state/AppState.java | 56 +++----
.../MostRecentContainerReleaseSelector.java | 2 +-
.../server/appmaster/state/NodeInstance.java | 2 +-
.../slider/server/appmaster/state/NodeMap.java | 2 +-
.../state/OutstandingRequestTracker.java | 6 +-
.../appmaster/state/ProviderAppState.java | 4 +-
.../server/appmaster/state/RoleHistory.java | 15 +-
.../server/appmaster/state/RoleStatus.java | 2 +-
.../server/appmaster/web/SliderAMWebApp.java | 2 +-
.../web/SliderAmFilterInitializer.java | 2 +-
.../server/appmaster/web/SliderAmIpFilter.java | 8 +-
.../server/appmaster/web/WebAppApiImpl.java | 2 +-
.../appmaster/web/rest/agent/HeartBeat.java | 10 --
.../web/rest/agent/HeartBeatResponse.java | 11 ++
.../appmaster/web/rest/agent/Register.java | 58 +++++--
.../web/rest/publisher/PublisherResource.java | 4 +-
.../slider/server/avro/RoleHistoryWriter.java | 6 +-
.../server/servicemonitor/MonitorUtils.java | 2 +-
.../services/curator/RegistryBinderService.java | 6 +-
.../registry/RegistryRestResources.java | 3 +-
.../registry/SliderRegistryService.java | 2 +-
.../agent/TestAgentProviderService.java | 96 +++++++----
.../agent/TestComponentInstanceState.java | 11 +-
.../providers/agent/TestHeartbeatMonitor.java | 4 +-
.../management/TestAMManagementWebServices.java | 9 +-
.../publisher/TestAgentProviderService.java | 2 +-
.../workflow/ProcessCommandFactory.java | 6 +-
.../slider/test/MiniZooKeeperCluster.java | 6 +-
.../funtest/framework/CommandTestBase.groovy | 16 +-
.../lifecycle/AgentClusterLifecycleIT.groovy | 11 +-
.../accumulo/AccumuloProviderService.java | 19 ++-
.../funtest/HBaseClusterLifecycleIT.groovy | 16 +-
.../hbase/funtest/HBaseIntegrationIT.groovy | 5 +-
.../providers/hbase/funtest/HBaseLoadIT.groovy | 7 +
.../providers/hbase/HBaseClientProvider.java | 6 +-
.../providers/hbase/HBaseProviderService.java | 31 +++-
.../clusters/sandbox/slider/slider-client.xml | 2 +-
110 files changed, 1350 insertions(+), 416 deletions(-)
----------------------------------------------------------------------
[24/50] [abbrv] git commit: SLIDER-323. Allocated ports must be
reported back upon AM restart SLIDER-262. Slider agent should provide process
supervision such as auto-restart
Posted by st...@apache.org.
SLIDER-323. Allocated ports must be reported back upon AM restart
SLIDER-262. Slider agent should provide process supervision such as auto-restart
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/47822dff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/47822dff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/47822dff
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 47822dffa3e086c198fed820ec1e4b099a453659
Parents: 18b0545
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Aug 14 17:36:59 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Aug 14 17:57:43 2014 -0700
----------------------------------------------------------------------
.../src/main/python/agent/ActionQueue.py | 11 ++-
.../src/main/python/agent/CommandStatusDict.py | 5 +-
.../src/main/python/agent/Controller.py | 27 ++++---
.../python/agent/CustomServiceOrchestrator.py | 8 +-
slider-agent/src/main/python/agent/Heartbeat.py | 9 ++-
slider-agent/src/main/python/agent/Register.py | 7 +-
.../src/test/python/agent/TestActionQueue.py | 2 -
.../src/test/python/agent/TestController.py | 29 +++++--
.../agent/TestCustomServiceOrchestrator.py | 2 +
.../src/test/python/agent/TestHeartbeat.py | 82 +++++++++++++++++++-
.../src/test/python/agent/TestRegistration.py | 8 +-
.../providers/agent/AgentProviderService.java | 51 +++++++++---
.../appmaster/web/rest/agent/HeartBeat.java | 10 ---
.../appmaster/web/rest/agent/Register.java | 58 ++++++++++----
.../agent/TestAgentProviderService.java | 17 ++++
15 files changed, 254 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-agent/src/main/python/agent/ActionQueue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/ActionQueue.py b/slider-agent/src/main/python/agent/ActionQueue.py
index fba9aa2..4c45a76 100644
--- a/slider-agent/src/main/python/agent/ActionQueue.py
+++ b/slider-agent/src/main/python/agent/ActionQueue.py
@@ -122,7 +122,8 @@ class ActionQueue(threading.Thread):
taskId = command['taskId']
- reportResult = not command[Constants.AUTO_GENERATED]
+ # if auto generated then do not report result
+ reportResult = CommandStatusDict.shouldReportResult(command)
# Preparing 'IN_PROGRESS' report
in_progress_status = self.commandStatuses.generate_report_template(command)
@@ -140,7 +141,8 @@ class ActionQueue(threading.Thread):
if ActionQueue.STORE_APPLIED_CONFIG in command['commandParams']:
store_config = 'true' == command['commandParams'][ActionQueue.STORE_APPLIED_CONFIG]
store_command = False
- if ActionQueue.AUTO_RESTART in command['roleParams']:
+ if 'roleParams' in command and ActionQueue.AUTO_RESTART in command['roleParams']:
+ logger.info("Component has indicated auto-restart. Saving details from START command.")
store_command = 'true' == command['roleParams'][ActionQueue.AUTO_RESTART]
@@ -195,10 +197,7 @@ class ActionQueue(threading.Thread):
cluster = command['clusterName']
service = command['serviceName']
component = command['componentName']
- reportResult = True
- if Constants.AUTO_GENERATED in command:
- reportResult = not command[Constants.AUTO_GENERATED]
-
+ reportResult = CommandStatusDict.shouldReportResult(command)
component_status = self.customServiceOrchestrator.requestComponentStatus(command)
result = {"componentName": component,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-agent/src/main/python/agent/CommandStatusDict.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CommandStatusDict.py b/slider-agent/src/main/python/agent/CommandStatusDict.py
index f20e0fa..bcbce9e 100644
--- a/slider-agent/src/main/python/agent/CommandStatusDict.py
+++ b/slider-agent/src/main/python/agent/CommandStatusDict.py
@@ -114,7 +114,7 @@ class CommandStatusDict():
grep = Grep()
output = grep.tail(tmpout, Grep.OUTPUT_LAST_LINES)
inprogress = self.generate_report_template(command)
- reportResult = not command[Constants.AUTO_GENERATED]
+ reportResult = CommandStatusDict.shouldReportResult(command)
inprogress.update({
'stdout': grep.filterMarkup(output),
'stderr': tmperr,
@@ -142,3 +142,6 @@ class CommandStatusDict():
return stub
+ @staticmethod
+ def shouldReportResult(command):
+ return not (Constants.AUTO_GENERATED in command and command[Constants.AUTO_GENERATED])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-agent/src/main/python/agent/Controller.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Controller.py b/slider-agent/src/main/python/agent/Controller.py
index 6bdfc26..1e27efa 100644
--- a/slider-agent/src/main/python/agent/Controller.py
+++ b/slider-agent/src/main/python/agent/Controller.py
@@ -118,7 +118,11 @@ class Controller(threading.Thread):
while not self.isRegistered:
try:
- data = json.dumps(self.register.build(id))
+ data = json.dumps(self.register.build(
+ self.componentActualState,
+ self.componentExpectedState,
+ self.actionQueue.customServiceOrchestrator.allocated_ports,
+ id))
logger.info("Registering with the server at " + self.registerUrl +
" with data " + pprint.pformat(data))
response = self.sendRequest(self.registerUrl, data)
@@ -211,7 +215,7 @@ class Controller(threading.Thread):
try:
if not retry:
data = json.dumps(
- self.heartbeat.build(commandResult, self.componentActualState,
+ self.heartbeat.build(commandResult,
self.responseId, self.hasMappedComponents))
self.updateStateBasedOnResult(commandResult)
logger.debug("Sending request: " + data)
@@ -225,8 +229,11 @@ class Controller(threading.Thread):
serverId = int(response['responseId'])
+ restartEnabled = False
if 'restartEnabled' in response:
- restartEnabled = 'true' == response['restartEnabled']
+ restartEnabled = response['restartEnabled']
+ if restartEnabled:
+ logger.info("Component auto-restart is enabled.")
if 'hasMappedComponents' in response.keys():
self.hasMappedComponents = response['hasMappedComponents'] != False
@@ -262,15 +269,16 @@ class Controller(threading.Thread):
pass
# Add a start command
- if self.componentActualState == State.INSTALLED and \
+ if self.componentActualState == State.FAILED and \
self.componentExpectedState == State.STARTED and restartEnabled:
stored_command = self.actionQueue.customServiceOrchestrator.stored_command
if len(stored_command) > 0:
- auto_start_command = self.create_start_command(stored_command)
- if auto_start_command:
- logger.info("Automatically adding a start command.")
- self.updateStateBasedOnCommand([auto_start_command], False)
- self.addToQueue([auto_start_command])
+ auto_start_command = self.create_start_command(stored_command)
+ if auto_start_command:
+ logger.info("Automatically adding a start command.")
+ logger.debug("Auto start command: " + pprint.pformat(auto_start_command))
+ self.updateStateBasedOnCommand([auto_start_command], False)
+ self.addToQueue([auto_start_command])
pass
# Add a status command
@@ -388,6 +396,7 @@ class Controller(threading.Thread):
if "healthStatus" in commandResult:
if commandResult["healthStatus"] == "INSTALLED":
+ # Mark it FAILED as its a failure remedied by auto-start or container restart
self.componentActualState = State.FAILED
self.failureCount += 1
self.logStates()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 6b2ace5..15f1664 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -57,6 +57,7 @@ class CustomServiceOrchestrator():
'status_command_stderr.txt'))
self.public_fqdn = hostname.public_hostname()
self.stored_command = {}
+ self.allocated_ports = {}
# Clean up old status command files if any
try:
os.unlink(self.status_commands_stdout)
@@ -69,7 +70,7 @@ class CustomServiceOrchestrator():
def runCommand(self, command, tmpoutfile, tmperrfile,
override_output_files=True, store_command=False):
- allocated_port = {}
+ allocated_ports = {}
try:
script_type = command['commandParams']['script_type']
script = command['commandParams']['script']
@@ -86,7 +87,7 @@ class CustomServiceOrchestrator():
# We don't support anything else yet
message = "Unknown script type {0}".format(script_type)
raise AgentException(message)
- json_path = self.dump_command_to_json(command, allocated_port, store_command)
+ json_path = self.dump_command_to_json(command, allocated_ports, store_command)
py_file_list = [script_tuple]
# filter None values
filtered_py_file_list = [i for i in py_file_list if i]
@@ -132,7 +133,8 @@ class CustomServiceOrchestrator():
}
if Constants.EXIT_CODE in ret and ret[Constants.EXIT_CODE] == 0:
- ret[Constants.ALLOCATED_PORTS] = allocated_port
+ ret[Constants.ALLOCATED_PORTS] = allocated_ports
+ self.allocated_ports = allocated_ports
# Irrespective of the outcome report the folder paths
if command_name == 'INSTALL':
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-agent/src/main/python/agent/Heartbeat.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Heartbeat.py b/slider-agent/src/main/python/agent/Heartbeat.py
index aa403d4..b107d92 100644
--- a/slider-agent/src/main/python/agent/Heartbeat.py
+++ b/slider-agent/src/main/python/agent/Heartbeat.py
@@ -36,7 +36,7 @@ class Heartbeat:
self.config = config
self.reports = []
- def build(self, commandResult, componentActualState, id='-1',
+ def build(self, commandResult, id='-1',
componentsMapped=False):
timestamp = int(time.time() * 1000)
queueResult = self.actionQueue.result()
@@ -49,8 +49,7 @@ class Heartbeat:
'timestamp': timestamp,
'hostname': self.config.getLabel(),
'nodeStatus': nodeStatus,
- 'fqdn': hostname.public_hostname(),
- 'agentState': componentActualState
+ 'fqdn': hostname.public_hostname()
}
commandsInProgress = False
@@ -62,6 +61,10 @@ class Heartbeat:
if report['reportResult']:
del report['reportResult']
heartbeat['reports'].append(report)
+ else:
+ # dropping the result but only recording the status
+ commandResult["commandStatus"] = report["status"]
+ pass
if len(heartbeat['reports']) > 0:
# There may be IN_PROGRESS tasks
commandsInProgress = True
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-agent/src/main/python/agent/Register.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Register.py b/slider-agent/src/main/python/agent/Register.py
index 7c7ff06..b59154f 100644
--- a/slider-agent/src/main/python/agent/Register.py
+++ b/slider-agent/src/main/python/agent/Register.py
@@ -29,7 +29,7 @@ class Register:
def __init__(self, config):
self.config = config
- def build(self, id='-1'):
+ def build(self, actualState, expectedState, allocated_ports, id='-1'):
timestamp = int(time.time() * 1000)
version = self.read_agent_version()
@@ -38,7 +38,10 @@ class Register:
'timestamp': timestamp,
'hostname': self.config.getLabel(),
'publicHostname': hostname.public_hostname(),
- 'agentVersion': version
+ 'agentVersion': version,
+ 'actualState': actualState,
+ 'expectedState': expectedState,
+ 'allocatedPorts': allocated_ports
}
return register
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-agent/src/test/python/agent/TestActionQueue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestActionQueue.py b/slider-agent/src/test/python/agent/TestActionQueue.py
index 4a53c2e..8071ee8 100644
--- a/slider-agent/src/test/python/agent/TestActionQueue.py
+++ b/slider-agent/src/test/python/agent/TestActionQueue.py
@@ -61,8 +61,6 @@ class TestActionQueue(TestCase):
'serviceName': u'HBASE',
'configurations': {'global': {}},
'configurationTags': {'global': {'tag': 'v1'}},
- 'auto_generated': False,
- 'roleParams': {'auto_restart':'false'},
'commandParams': {'script_type': 'PYTHON',
'script': 'scripts/abc.py',
'command_timeout': '600'}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-agent/src/test/python/agent/TestController.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestController.py b/slider-agent/src/test/python/agent/TestController.py
index 91e12f1..401d69a 100644
--- a/slider-agent/src/test/python/agent/TestController.py
+++ b/slider-agent/src/test/python/agent/TestController.py
@@ -55,6 +55,7 @@ class TestController(unittest.TestCase):
self.controller = Controller.Controller(config)
self.controller.netutil.MINIMUM_INTERVAL_BETWEEN_HEARTBEATS = 0.1
self.controller.netutil.HEARTBEAT_NOT_IDDLE_INTERVAL_SEC = 0.1
+ self.controller.actionQueue = ActionQueue.ActionQueue(config, self.controller)
@patch("json.dumps")
@@ -289,7 +290,7 @@ class TestController(unittest.TestCase):
self.controller.sendRequest = sendRequest
self.controller.responseId = 1
- response = {"responseId":"2", "restartAgent":"false"}
+ response = {"responseId":"2", "restartAgent": False}
loadsMock.return_value = response
def one_heartbeat(*args, **kwargs):
@@ -651,8 +652,9 @@ class TestController(unittest.TestCase):
self.controller.sendRequest = sendRequest
self.controller.responseId = 1
- response = {"responseId":"2", "restartAgent":"false", "restartEnabled":'true'}
- loadsMock.return_value = response
+ response1 = {"responseId": "2", "restartAgent": False, "restartEnabled": True}
+ response2 = {"responseId": "2", "restartAgent": False, "restartEnabled": False}
+ loadsMock.side_effect = [response1, response2, response1]
def one_heartbeat(*args, **kwargs):
self.controller.DEBUG_STOP_HEARTBEATING = True
@@ -665,9 +667,9 @@ class TestController(unittest.TestCase):
# one successful request, after stop
self.controller.actionQueue = actionQueue
- self.controller.componentActualState = State.INSTALLED
+ self.controller.componentActualState = State.FAILED
self.controller.componentExpectedState = State.STARTED
- self.assertTrue(self.controller.componentActualState, State.INSTALLED)
+ self.assertTrue(self.controller.componentActualState, State.FAILED)
self.controller.actionQueue.customServiceOrchestrator.stored_command = {
'commandType': 'EXECUTION_COMMAND',
'role': u'HBASE',
@@ -699,6 +701,23 @@ class TestController(unittest.TestCase):
'commandId': '8-1',
'auto_generated': True}])])
self.controller.config = original_value
+
+ # restartEnabled = False
+ self.controller.componentActualState = State.FAILED
+ self.controller.heartbeatWithServer()
+
+ self.assertTrue(sendRequest.called)
+ self.assertTrue(self.controller.componentActualState, State.FAILED)
+ self.assertTrue(self.controller.componentExpectedState, State.STARTED)
+
+ # restartEnabled = True
+ self.controller.componentActualState = State.INSTALLED
+ self.controller.componentExpectedState = State.INSTALLED
+ self.controller.heartbeatWithServer()
+
+ self.assertTrue(sendRequest.called)
+ self.assertTrue(self.controller.componentActualState, State.INSTALLED)
+ self.assertTrue(self.controller.componentExpectedState, State.INSTALLED)
pass
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index 6408809..e545afe 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -230,6 +230,8 @@ class TestCustomServiceOrchestrator(TestCase):
self.assertEqual(ret['allocated_ports'], {'a.a.port': '10233'})
self.assertTrue(run_file_mock.called)
self.assertEqual(run_file_mock.call_count, 1)
+ self.assertEqual(orchestrator.allocated_ports, {'a.a.port': '10233'})
+ self.assertEqual(orchestrator.stored_command, {})
@patch.object(socket, "close")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-agent/src/test/python/agent/TestHeartbeat.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestHeartbeat.py b/slider-agent/src/test/python/agent/TestHeartbeat.py
index 70870ec..b012218 100644
--- a/slider-agent/src/test/python/agent/TestHeartbeat.py
+++ b/slider-agent/src/test/python/agent/TestHeartbeat.py
@@ -50,7 +50,7 @@ class TestHeartbeat(TestCase):
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
heartbeat = Heartbeat(actionQueue, config)
- result = heartbeat.build({}, State.STARTED, 100)
+ result = heartbeat.build({}, 100)
print "Heartbeat: " + str(result)
self.assertEquals(result['hostname'] != '', True,
"hostname should not be empty")
@@ -63,7 +63,6 @@ class TestHeartbeat(TestCase):
self.assertEquals(len(result['nodeStatus']), 2)
self.assertEquals(result['nodeStatus']['cause'], "NONE")
self.assertEquals(result['nodeStatus']['status'], "HEALTHY")
- self.assertEquals(result['agentState'], State.STARTED)
# result may or may NOT have an agentEnv structure in it
self.assertEquals((len(result) is 6) or (len(result) is 7), True)
self.assertEquals(not heartbeat.reports, True,
@@ -147,7 +146,7 @@ class TestHeartbeat(TestCase):
}
heartbeat = Heartbeat(actionQueue, config)
# State.STARTED results in agentState to be set to 4 (enum order)
- hb = heartbeat.build({}, State.STARTED, 10)
+ hb = heartbeat.build({}, 10)
hb['hostname'] = 'hostname'
hb['timestamp'] = 'timestamp'
hb['fqdn'] = 'fqdn'
@@ -155,7 +154,7 @@ class TestHeartbeat(TestCase):
{'status': 'HEALTHY',
'cause': 'NONE'},
'timestamp': 'timestamp', 'hostname': 'hostname', 'fqdn': 'fqdn',
- 'agentState': 4, 'responseId': 10, 'reports': [
+ 'responseId': 10, 'reports': [
{'status': 'IN_PROGRESS', 'roleCommand': u'INSTALL',
'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1',
'stderr': 'Read from /tmp/errors-3.txt',
@@ -178,6 +177,81 @@ class TestHeartbeat(TestCase):
self.assertEqual.__self__.maxDiff = None
self.assertEquals(hb, expected)
+ @patch.object(ActionQueue, "result")
+ def test_build_result2(self, result_mock):
+ config = AgentConfig("", "")
+ config.set('agent', 'prefix', 'tmp')
+ dummy_controller = MagicMock()
+ actionQueue = ActionQueue(config, dummy_controller)
+ result_mock.return_value = {
+ 'reports': [{'status': 'IN_PROGRESS',
+ 'stderr': 'Read from /tmp/errors-3.txt',
+ 'stdout': 'Read from /tmp/output-3.txt',
+ 'clusterName': u'cc',
+ 'roleCommand': u'INSTALL',
+ 'serviceName': u'HDFS',
+ 'role': u'DATANODE',
+ 'actionId': '1-1',
+ 'taskId': 3,
+ 'exitcode': 777,
+ 'reportResult' : False}
+ ],
+ 'componentStatus': []
+ }
+ heartbeat = Heartbeat(actionQueue, config)
+
+ commandResult = {}
+ hb = heartbeat.build(commandResult, 10)
+ hb['hostname'] = 'hostname'
+ hb['timestamp'] = 'timestamp'
+ hb['fqdn'] = 'fqdn'
+ expected = {'nodeStatus':
+ {'status': 'HEALTHY',
+ 'cause': 'NONE'},
+ 'timestamp': 'timestamp', 'hostname': 'hostname', 'fqdn': 'fqdn',
+ 'responseId': 10, 'reports': []}
+ self.assertEqual.__self__.maxDiff = None
+ self.assertEquals(hb, expected)
+ self.assertEquals(commandResult, {'commandStatus': 'IN_PROGRESS'})
+
+ @patch.object(ActionQueue, "result")
+ def test_build_result3(self, result_mock):
+ config = AgentConfig("", "")
+ config.set('agent', 'prefix', 'tmp')
+ dummy_controller = MagicMock()
+ actionQueue = ActionQueue(config, dummy_controller)
+ result_mock.return_value = {
+ 'reports': [{'status': 'COMPLETED',
+ 'stderr': 'Read from /tmp/errors-3.txt',
+ 'stdout': 'Read from /tmp/output-3.txt',
+ 'clusterName': u'cc',
+ 'roleCommand': u'INSTALL',
+ 'serviceName': u'HDFS',
+ 'role': u'DATANODE',
+ 'actionId': '1-1',
+ 'taskId': 3,
+ 'exitcode': 777,
+ 'reportResult' : False}
+ ],
+ 'componentStatus': []
+ }
+ heartbeat = Heartbeat(actionQueue, config)
+
+ commandResult = {}
+ hb = heartbeat.build(commandResult, 10)
+ hb['hostname'] = 'hostname'
+ hb['timestamp'] = 'timestamp'
+ hb['fqdn'] = 'fqdn'
+ expected = {'nodeStatus':
+ {'status': 'HEALTHY',
+ 'cause': 'NONE'},
+ 'timestamp': 'timestamp', 'hostname': 'hostname', 'fqdn': 'fqdn',
+ 'responseId': 10, 'reports': []}
+ self.assertEqual.__self__.maxDiff = None
+ self.assertEquals(hb, expected)
+ self.assertEquals(commandResult, {'commandStatus': 'COMPLETED'})
+
+
if __name__ == "__main__":
logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-agent/src/test/python/agent/TestRegistration.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestRegistration.py b/slider-agent/src/test/python/agent/TestRegistration.py
index 5245af9..f91fe29 100644
--- a/slider-agent/src/test/python/agent/TestRegistration.py
+++ b/slider-agent/src/test/python/agent/TestRegistration.py
@@ -26,6 +26,7 @@ import tempfile
from mock.mock import patch
from mock.mock import MagicMock
from Register import Register
+from Controller import State
from AgentConfig import AgentConfig
class TestRegistration(TestCase):
@@ -47,14 +48,17 @@ class TestRegistration(TestCase):
text_file.write("1.3.0")
register = Register(config)
- data = register.build(1)
+ data = register.build(State.INIT, State.INIT, {}, 1)
#print ("Register: " + pprint.pformat(data))
self.assertEquals(data['hostname'] != "", True, "hostname should not be empty")
self.assertEquals(data['publicHostname'] != "", True, "publicHostname should not be empty")
self.assertEquals(data['responseId'], 1)
self.assertEquals(data['timestamp'] > 1353678475465L, True, "timestamp should not be empty")
self.assertEquals(data['agentVersion'], '1.3.0', "agentVersion should not be empty")
- self.assertEquals(len(data), 5)
+ self.assertEquals(data['actualState'], State.INIT, "actualState should not be empty")
+ self.assertEquals(data['expectedState'], State.INIT, "expectedState should not be empty")
+ self.assertEquals(data['allocatedPorts'], {}, "allocatedPorts should be empty")
+ self.assertEquals(len(data), 8)
self.assertEquals(os.path.join(tmpdir, "app/definition"), config.getResolvedPath("app_pkg_dir"))
self.assertEquals(os.path.join(tmpdir, "app/install"), config.getResolvedPath("app_install_dir"))
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index b22ef39..8ce1f18 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -398,9 +398,19 @@ public class AgentProviderService extends AbstractProviderService implements
log.info("Handling registration: " + registration);
RegistrationResponse response = new RegistrationResponse();
String label = registration.getHostname();
+ State agentState = registration.getActualState();
if (getComponentStatuses().containsKey(label)) {
response.setResponseStatus(RegistrationStatus.OK);
- getComponentStatuses().get(label).heartbeat(System.currentTimeMillis());
+ ComponentInstanceState componentStatus = getComponentStatuses().get(label);
+ componentStatus.heartbeat(System.currentTimeMillis());
+ updateComponentStatusWithAgentState(componentStatus, agentState);
+
+ Map<String, String> ports = registration.getAllocatedPorts();
+ if (ports != null && !ports.isEmpty()) {
+ String roleName = getRoleName(label);
+ String containerId = getContainerId(label);
+ processAllocatedPorts(registration.getPublicHostname(), roleName, containerId, ports);
+ }
} else {
response.setResponseStatus(RegistrationStatus.FAILED);
response.setLog("Label not recognized.");
@@ -424,9 +434,8 @@ public class AgentProviderService extends AbstractProviderService implements
String label = heartBeat.getHostname();
String roleName = getRoleName(label);
- State agentState = heartBeat.getAgentState();
-
String containerId = getContainerId(label);
+
StateAccessForProviders accessor = getAmState();
String scriptPath = getScriptPathFromMetainfo(roleName);
@@ -441,13 +450,22 @@ public class AgentProviderService extends AbstractProviderService implements
Boolean isMaster = isMaster(roleName);
ComponentInstanceState componentStatus = getComponentStatuses().get(label);
- updateComponentStatusWithAgentState(componentStatus, agentState);
componentStatus.heartbeat(System.currentTimeMillis());
// If no Master can explicitly publish then publish if its a master
// Otherwise, wait till the master that can publish is ready
if (isMaster &&
(canAnyMasterPublishConfig() == false || canPublishConfig(roleName))) {
publishConfigAndExportGroups(heartBeat, componentStatus);
+ } else {
+ // Ack that config has been reported
+ List<ComponentStatus> statuses = heartBeat.getComponentStatus();
+ if (statuses != null && !statuses.isEmpty()) {
+ ComponentStatus status = statuses.get(0);
+ if(status.getConfigs().size() > 0) {
+ log.info("Config got reported by {} but discarded as component {} cannot publish.", label, roleName);
+ componentStatus.setConfigReported(true);
+ }
+ }
}
List<CommandReport> reports = heartBeat.getReports();
@@ -455,14 +473,7 @@ public class AgentProviderService extends AbstractProviderService implements
CommandReport report = reports.get(0);
Map<String, String> ports = report.getAllocatedPorts();
if (ports != null && !ports.isEmpty()) {
- for (Map.Entry<String, String> port : ports.entrySet()) {
- log.info("Recording allocated port for {} as {}", port.getKey(), port.getValue());
- this.getAllocatedPorts().put(port.getKey(), port.getValue());
- this.getAllocatedPorts(containerId).put(port.getKey(), port.getValue());
- }
-
- // component specific publishes
- processAndPublishComponentSpecificData(ports, containerId, heartBeat.getFqdn(), roleName);
+ processAllocatedPorts(heartBeat.getFqdn(), roleName, containerId, ports);
}
CommandResult result = CommandResult.getCommandResult(report.getStatus());
Command command = Command.getCommand(report.getRoleCommand());
@@ -506,6 +517,7 @@ public class AgentProviderService extends AbstractProviderService implements
if (isMaster && componentStatus.getState() == State.STARTED
&& command == Command.NOP) {
if (!componentStatus.getConfigReported()) {
+ log.info("Requesting applied config for {} on {}.", roleName, containerId);
addGetConfigCommand(roleName, containerId, response);
}
}
@@ -525,6 +537,20 @@ public class AgentProviderService extends AbstractProviderService implements
return response;
}
+ protected void processAllocatedPorts(String fqdn,
+ String roleName,
+ String containerId,
+ Map<String, String> ports) {
+ for (Map.Entry<String, String> port : ports.entrySet()) {
+ log.info("Recording allocated port for {} as {}", port.getKey(), port.getValue());
+ this.getAllocatedPorts().put(port.getKey(), port.getValue());
+ this.getAllocatedPorts(containerId).put(port.getKey(), port.getValue());
+ }
+
+ // component specific publishes
+ processAndPublishComponentSpecificData(ports, containerId, fqdn, roleName);
+ }
+
private void updateComponentStatusWithAgentState(
ComponentInstanceState componentStatus, State agentState) {
if (agentState != null) {
@@ -803,6 +829,7 @@ public class AgentProviderService extends AbstractProviderService implements
}
}
}
+ log.info("Received and stored config for {}", heartBeat.getHostname());
componentStatus.setConfigReported(true);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java
index 62df18d..a08d46d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java
@@ -45,7 +45,6 @@ public class HeartBeat {
HostStatus nodeStatus;
private AgentEnv agentEnv = null;
private String fqdn;
- private State agentState;
public long getResponseId() {
return responseId;
@@ -125,14 +124,6 @@ public class HeartBeat {
this.mounts = mounts;
}
- public State getAgentState() {
- return agentState;
- }
-
- public void setAgentState(State agentState) {
- this.agentState = agentState;
- }
-
@Override
public String toString() {
return "HeartBeat{" +
@@ -142,7 +133,6 @@ public class HeartBeat {
", reports=" + reports +
", componentStatus=" + componentStatus +
", nodeStatus=" + nodeStatus +
- ", agentState=" + agentState +
'}';
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
index 9299a16..a44c3a4 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
@@ -16,15 +16,14 @@
*/
package org.apache.slider.server.appmaster.web.rest.agent;
+import org.apache.slider.providers.agent.State;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.annotate.JsonProperty;
import org.codehaus.jackson.map.annotate.JsonSerialize;
-/**
- *
- * Data model for agent to send heartbeat to ambari and/or app master.
- *
- */
+import java.util.Map;
+
+/** Data model for agent to send heartbeat to ambari and/or app master. */
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class Register {
@@ -36,6 +35,9 @@ public class Register {
private String publicHostname;
private AgentEnv agentEnv;
private String agentVersion;
+ private State actualState;
+ private State expectedState;
+ private Map<String, String> allocatedPorts;
@JsonProperty("responseId")
public int getResponseId() {
@@ -44,13 +46,17 @@ public class Register {
@JsonProperty("responseId")
public void setResponseId(int responseId) {
- this.responseId=responseId;
+ this.responseId = responseId;
}
public long getTimestamp() {
return timestamp;
}
+ public void setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ }
+
public String getHostname() {
return hostname;
}
@@ -67,10 +73,6 @@ public class Register {
this.hardwareProfile = hardwareProfile;
}
- public void setTimestamp(long timestamp) {
- this.timestamp = timestamp;
- }
-
public String getPublicHostname() {
return publicHostname;
}
@@ -103,15 +105,45 @@ public class Register {
this.currentPingPort = currentPingPort;
}
+ public State getActualState() {
+ return actualState;
+ }
+
+ public void setActualState(State actualState) {
+ this.actualState = actualState;
+ }
+
+ public State getExpectedState() {
+ return expectedState;
+ }
+
+ public void setExpectedState(State expectedState) {
+ this.expectedState = expectedState;
+ }
+
+ /** @return the allocated ports, or <code>null</code> if none are present */
+ @JsonProperty("allocatedPorts")
+ public Map<String, String> getAllocatedPorts() {
+ return allocatedPorts;
+ }
+
+ /** @param ports allocated ports */
+ @JsonProperty("allocatedPorts")
+ public void setAllocatedPorts(Map<String, String> ports) {
+ this.allocatedPorts = ports;
+ }
+
@Override
public String toString() {
String ret = "responseId=" + responseId + "\n" +
"timestamp=" + timestamp + "\n" +
- "hostname=" + hostname + "\n" +
- "currentPingPort=" + currentPingPort + "\n";
+ "hostname=" + hostname + "\n" +
+ "expectedState=" + expectedState + "\n" +
+ "actualState=" + actualState + "\n";
- if (hardwareProfile != null)
+ if (hardwareProfile != null) {
ret = ret + "hardwareprofile=" + this.hardwareProfile.toString();
+ }
return ret;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/47822dff/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 1aeedbc..20c16ff 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -87,6 +87,7 @@ import static org.easymock.EasyMock.replay;
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyCollection;
+import static org.mockito.Matchers.anyMap;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doReturn;
@@ -271,6 +272,12 @@ public class TestAgentProviderService {
} catch (SliderException e) {
}
+ doNothing().when(mockAps).processAllocatedPorts(
+ anyString(),
+ anyString(),
+ anyString(),
+ anyMap()
+ );
expect(access.isApplicationLive()).andReturn(true).anyTimes();
ClusterDescription desc = new ClusterDescription();
desc.setOption(OptionKeys.ZOOKEEPER_QUORUM, "host1:2181");
@@ -305,10 +312,20 @@ public class TestAgentProviderService {
Register reg = new Register();
reg.setResponseId(0);
reg.setHostname("mockcontainer_1___HBASE_MASTER");
+ Map<String,String> ports = new HashMap();
+ ports.put("a","100");
+ reg.setAllocatedPorts(ports);
RegistrationResponse resp = mockAps.handleRegistration(reg);
Assert.assertEquals(0, resp.getResponseId());
Assert.assertEquals(RegistrationStatus.OK, resp.getResponseStatus());
+ Mockito.verify(mockAps, Mockito.times(1)).processAllocatedPorts(
+ anyString(),
+ anyString(),
+ anyString(),
+ anyMap()
+ );
+
HeartBeat hb = new HeartBeat();
hb.setResponseId(1);
hb.setHostname("mockcontainer_1___HBASE_MASTER");
[14/50] [abbrv] SLIDER-322 move all <> constructors to typed
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index 6653dba..df4ab8e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -264,7 +264,7 @@ public final class RoleStatus implements Cloneable {
* @return a map for use in statistics reports
*/
public Map<String, Integer> buildStatistics() {
- Map<String, Integer> stats = new HashMap<>();
+ Map<String, Integer> stats = new HashMap<String, Integer>();
stats.put(StatusKeys.STATISTICS_CONTAINERS_ACTIVE_REQUESTS, getRequested());
stats.put(StatusKeys.STATISTICS_CONTAINERS_COMPLETED, getCompleted());
stats.put(StatusKeys.STATISTICS_CONTAINERS_DESIRED, getDesired());
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
index 4f290af..9192efe 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
@@ -96,7 +96,7 @@ public class SliderAMWebApp extends WebApp {
String regex = "(?!/ws)";
serveRegex(regex).with(SliderDefaultWrapperServlet.class);
- Map<String, String> params = new HashMap<>();
+ Map<String, String> params = new HashMap<String, String>();
params.put(ResourceConfig.FEATURE_IMPLICIT_VIEWABLES, "true");
params.put(ServletContainer.FEATURE_FILTER_FORWARD_ON_404, "true");
params.put(ResourceConfig.FEATURE_XMLROOTELEMENT_PROCESSING, "true");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
index 606c05d..5fffa4a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmFilterInitializer.java
@@ -43,7 +43,7 @@ public class SliderAmFilterInitializer extends FilterInitializer {
@Override
public void initFilter(FilterContainer container, Configuration conf) {
configuration = conf;
- Map<String, String> params = new HashMap<>();
+ Map<String, String> params = new HashMap<String, String>();
String proxy = WebAppUtils.getProxyHostAndPort(conf);
String[] parts = proxy.split(":");
params.put(SliderAmIpFilter.PROXY_HOST, parts[0]);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
index ad5e219..aad443e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
@@ -70,7 +70,7 @@ public class SliderAmIpFilter implements Filter {
synchronized(this) {
if(proxyAddresses == null || (lastUpdate + updateInterval) >= now) {
try {
- proxyAddresses = new HashSet<>();
+ proxyAddresses = new HashSet<String>();
for(InetAddress add : InetAddress.getAllByName(proxyHost)) {
if (log.isDebugEnabled()) {
log.debug("proxy address is: " + add.getHostAddress());
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
index 4eebd45..4d595a9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
@@ -126,7 +126,7 @@ public class WebAppApiImpl implements WebAppApi {
*/
private TreeMap<String, RoleStatus> getRoleStatusesByName(Map<Integer, ProviderRole> rolesById,
Map<Integer, RoleStatus> statusById) {
- TreeMap<String, RoleStatus> statusByName = new TreeMap<>();
+ TreeMap<String, RoleStatus> statusByName = new TreeMap<String, RoleStatus>();
for (Entry<Integer, ProviderRole> role : rolesById.entrySet()) {
final RoleStatus status = statusById.get(role.getKey());
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
index a439d9b..5d8b657 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
@@ -110,7 +110,7 @@ public class PublisherResource {
@Produces({MediaType.APPLICATION_JSON})
public Set<URL> getAMClassPath() {
URL[] urls = ((URLClassLoader) getClass().getClassLoader()).getURLs();
- return new LinkedHashSet<>(Arrays.asList(urls));
+ return new LinkedHashSet<URL>(Arrays.asList(urls));
}
@GET
@@ -231,7 +231,7 @@ public class PublisherResource {
propertyName, config);
throw new NotFoundException("Property not found: " + propertyName);
}
- Map<String,String> rtnVal = new HashMap<>();
+ Map<String, String> rtnVal = new HashMap<String, String>();
rtnVal.put(propertyName, propVal);
return rtnVal;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java b/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
index 2120be5..422ffeb 100644
--- a/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
+++ b/slider-core/src/main/java/org/apache/slider/server/avro/RoleHistoryWriter.java
@@ -90,7 +90,7 @@ public class RoleHistoryWriter {
throws IOException {
try {
DatumWriter<RoleHistoryRecord> writer =
- new SpecificDatumWriter<>(RoleHistoryRecord.class);
+ new SpecificDatumWriter<RoleHistoryRecord>(RoleHistoryRecord.class);
int roles = history.getRoleSize();
RoleHistoryHeader header = new RoleHistoryHeader();
@@ -184,7 +184,7 @@ public class RoleHistoryWriter {
BadConfigException {
try {
DatumReader<RoleHistoryRecord> reader =
- new SpecificDatumReader<>(RoleHistoryRecord.class);
+ new SpecificDatumReader<RoleHistoryRecord>(RoleHistoryRecord.class);
Decoder decoder =
DecoderFactory.get().jsonDecoder(RoleHistoryRecord.getClassSchema(),
in);
@@ -336,7 +336,7 @@ public class RoleHistoryWriter {
PathFilter filter = new GlobFilter(SliderKeys.HISTORY_FILENAME_GLOB_PATTERN);
FileStatus[] stats = fs.listStatus(dir, filter);
- List<Path> paths = new ArrayList<>(stats.length);
+ List<Path> paths = new ArrayList<Path>(stats.length);
for (FileStatus stat : stats) {
log.debug("Possible entry: {}", stat.toString());
if (stat.isFile() && (includeEmptyFiles || stat.getLen() > 0)) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorUtils.java b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorUtils.java
index 3b2c6d7..a4447e3 100644
--- a/slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/server/servicemonitor/MonitorUtils.java
@@ -50,7 +50,7 @@ public final class MonitorUtils {
* @return a list view with no empty strings
*/
public static List<String> prepareArgs(String[] args) {
- List<String> argsList = new ArrayList<>(args.length);
+ List<String> argsList = new ArrayList<String>(args.length);
StringBuilder argsStr = new StringBuilder("Arguments: [");
for (String arg : args) {
argsStr.append('"').append(arg).append("\" ");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java b/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
index 14b78a7..b3e2ff2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/curator/RegistryBinderService.java
@@ -54,10 +54,10 @@ public class RegistryBinderService<Payload> extends CuratorService {
private final ServiceDiscovery<Payload> discovery;
private final Map<String, ServiceInstance<Payload>> entries =
- new HashMap<>();
+ new HashMap<String, ServiceInstance<Payload>>();
private JsonSerDeser<CuratorServiceInstance<Payload>> deser =
- new JsonSerDeser<>(CuratorServiceInstance.class);
+ new JsonSerDeser<CuratorServiceInstance<Payload>>(CuratorServiceInstance.class);
/**
* Create an instance
@@ -221,7 +221,7 @@ public class RegistryBinderService<Payload> extends CuratorService {
try {
List<String> instanceIDs = instanceIDs(servicetype);
List<CuratorServiceInstance<Payload>> instances =
- new ArrayList<>(instanceIDs.size());
+ new ArrayList<CuratorServiceInstance<Payload>>(instanceIDs.size());
for (String instanceID : instanceIDs) {
CuratorServiceInstance<Payload> instance =
queryForInstance(servicetype, instanceID);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryRestResources.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryRestResources.java b/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryRestResources.java
index 4938adf..e4e8523 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryRestResources.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/registry/RegistryRestResources.java
@@ -100,7 +100,8 @@ public class RegistryRestResources extends DiscoveryResource<ServiceInstanceData
try {
List<CuratorServiceInstance<ServiceInstanceData>>
instances = registry.listInstances(name);
- return Response.ok(new CuratorServiceInstances<>(instances)).build();
+ return Response.ok(
+ new CuratorServiceInstances<ServiceInstanceData>(instances)).build();
} catch (Exception e) {
log.error("Error during generation of response", e);
return Response.serverError().build();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/services/registry/SliderRegistryService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/registry/SliderRegistryService.java b/slider-core/src/main/java/org/apache/slider/server/services/registry/SliderRegistryService.java
index 1904df5..ca4d180 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/registry/SliderRegistryService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/registry/SliderRegistryService.java
@@ -53,7 +53,7 @@ public class SliderRegistryService
IOException {
List<CuratorServiceInstance<ServiceInstanceData>> services =
listInstances(serviceType);
- List<ServiceInstanceData> payloads = new ArrayList<>(services.size());
+ List<ServiceInstanceData> payloads = new ArrayList<ServiceInstanceData>(services.size());
for (CuratorServiceInstance<ServiceInstanceData> instance : services) {
payloads.add(instance.payload);
}
[12/50] [abbrv] git commit: SLIDER-318 rm slider-assembly module that
isn't used
Posted by st...@apache.org.
SLIDER-318 rm slider-assembly module that isn't used
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/52f1f6fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/52f1f6fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/52f1f6fb
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 52f1f6fbe041a8bc604802e9eaaed2376b0e8ed5
Parents: 564462e
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 14 13:56:07 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 14 13:56:07 2014 +0100
----------------------------------------------------------------------
pom.xml | 1 -
slider-assembly/src/main/bash/README.md | 102 +++++++++++
slider-assembly/src/main/bash/slider-client.xml | 83 +++++++++
slider-assembly/src/main/bash/slider_destroy | 64 +++++++
slider-assembly/src/main/bash/slider_setup | 173 +++++++++++++++++++
slider-assembly/src/main/bash/slider_setup.conf | 60 +++++++
slider-install/README.md | 102 -----------
slider-install/pom.xml | 111 ------------
slider-install/src/main/bash/slider-client.xml | 83 ---------
slider-install/src/main/bash/slider_destroy | 64 -------
slider-install/src/main/bash/slider_setup | 173 -------------------
slider-install/src/main/bash/slider_setup.conf | 60 -------
12 files changed, 482 insertions(+), 594 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ed823c4..0816479 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,7 +45,6 @@
<module>slider-providers/hbase/hbase-funtests</module>
<module>slider-providers/accumulo/slider-accumulo-provider</module>
<module>slider-providers/accumulo/accumulo-funtests</module>
- <module>slider-install</module>
</modules>
<licenses>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-assembly/src/main/bash/README.md
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/bash/README.md b/slider-assembly/src/main/bash/README.md
new file mode 100644
index 0000000..a4b7b08
--- /dev/null
+++ b/slider-assembly/src/main/bash/README.md
@@ -0,0 +1,102 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+
+slider_setup
+============
+
+Tools for installing, starting, and destroying HBase, Accumulo, and Storm slider apps on YARN.
+
+**WARNING: This is intended for POC/sandbox testing, may not be idempotent so DO NOT use on an existing Production cluster!!!**
+
+Setup
+-----
+1. Clone the repo
+2. Set the necessary cluster variables in `slider_setup.conf`, it shouldn't be necessary to change any other values but thoe ones below
+
+
+ # Zookeeper nodes
+ ZK_QUORUM="zk1:2181,zk2:2181,zk3:2181"
+
+ # Resource Manager address (yarn.resourcemanager.address)
+ RM_ADDRESS="rm1:8050"
+
+ # Resource Manager scheduler address (yarn.resourcemanager.scheduler.address)
+ RM_SCHED_ADDRESS="rm1:8030"
+
+ # Default FS (fs.defaultFS)
+ DEFAULT_FS="hdfs://nn1:8020"
+
+Running
+-------
+* slider_setup is the main script and handles the following
+ 1. Pulls down slider and extracts the contents to the SLIDER_INST_DIR
+ 2. Modifies slider-client.xml with cluster related info
+ 3. Pulls down the slider enabled version of the specified product
+ 4. Creates necessary directories and copies required files to HDFS
+ 5. For HBase, creates the app dir in HDFS
+ 6. Submits the slider base application to the YARN cluster
+
+* The following args are required
+ * -f - The path to the slider_setup.conf that has been modified with cluster info
+ * -p - The product to run (hbase, accumulo, or storm are all that are supported at this time)
+ * -w - The number of "worker" nodes. This has different meaning depending on product.
+ * HBase - number of region servers
+ * Accumulo - number of tablet servers
+ * Storm - number of supervisors
+ * -n - The name of the app, this will be the display name in the resource manager and is used by the teardown process
+
+* HBase Example:
+
+
+ ./slider_setup -f slider_setup.conf -p hbase -w 5 -n hbase-slider
+
+* Accumulo Example:
+
+
+ ./slider_setup -f slider_setup.conf -p accumulo -w 3 -n accumulo-slider
+
+* Storm Example:
+
+
+ ./slider_setup -f slider_setup.conf -p storm -w 3 -n storm-slider
+
+Tear Down
+---------
+
+* slider_destroy will do the following
+ 1. Freeze the slider application based on provided name
+ 2. Destory the slider application based on provided name
+
+* The following args are required
+ * `-f` - The path to the `slider_setup.conf` that has been modified with cluster info
+ * `-n` - The name of the app, this was provided to the slider_setup tool
+
+* HBase Example:
+
+
+ ./slider_destroy -f slider_setup.conf -n hbase-slider
+
+* Accumulo Example:
+
+
+ ./slider_destroy -f slider_setup.conf -n accumulo-slider
+
+* Storm Example:
+
+
+ ./slider_destroy -f slider_setup.conf -n storm-slider
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-assembly/src/main/bash/slider-client.xml
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/bash/slider-client.xml b/slider-assembly/src/main/bash/slider-client.xml
new file mode 100644
index 0000000..f7060c1
--- /dev/null
+++ b/slider-assembly/src/main/bash/slider-client.xml
@@ -0,0 +1,83 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!--
+ Properties set here are picked up in the client.
+ They are not passed to the AM -though the filesystem
+ binding details (URL And principal) are added to the
+ hbase-site.xml file when a cluster is created.
+-->
+<configuration>
+
+ <property>
+ <name>yarn.log-aggregation-enable</name>
+ <value>true</value>
+ </property>
+
+
+ <property>
+ <name>slider.yarn.queue</name>
+ <value>default</value>
+ <description>YARN queue for the Application Master</description>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value>@@RM_ADDRESS@@</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.scheduler.address</name>
+ <value>@@RM_SCHED_ADDRESS@@</value>
+ </property>
+
+ <property>
+ <name>fs.defaultFS</name>
+ <value>@@DEFAULT_FS@@</value>
+ </property>
+
+ <property>
+ <name>yarn.application.classpath</name>
+ <value>@@YARN_CP@@</value>
+ </property>
+
+ <property>
+ <name>slider.zookeeper.quorum</name>
+ <value>@@ZK_QUORUM@@</value>
+ </property>
+
+<!--
+ <property>
+ <name>yarn.resourcemanager.principal</name>
+ <value>yarn/master@MINICLUSTER</value>
+ </property>
+
+ <property>
+ <name>slider.security.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.kerberos.principal</name>
+ <value>hdfs/master@MINICLUSTER</value>
+ </property>
+-->
+
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-assembly/src/main/bash/slider_destroy
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/bash/slider_destroy b/slider-assembly/src/main/bash/slider_destroy
new file mode 100755
index 0000000..9039751
--- /dev/null
+++ b/slider-assembly/src/main/bash/slider_destroy
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Base Vars
+#
+SCRIPT_NAME=`basename $0`
+SCRIPT_DIR=`cd $(dirname $0) && pwd`
+
+#
+# Functions
+#
+usage() {
+ echo "Usage: $SCRIPT_NAME -f </path/to/config> -n <app name>"
+ exit 1
+}
+
+#
+# Parse cmd line args
+#
+while getopts "f:n:" opt; do
+ case "$opt" in
+ f) config=$OPTARG;;
+ n) app_name=$OPTARG;;
+ *) usage;;
+ esac
+done
+shift $((OPTIND-1))
+
+if [ -z "$config" ] || [ -z "$app_name" ]; then
+ usage
+fi
+
+#
+# Source the config
+#
+source $config
+
+#
+# Main
+#
+echo -e "\n## Freezing app $app_name"
+sudo -u yarn $SLIDER_INST_DIR/bin/slider freeze $app_name --manager $RM_ADDRESS || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Destroying app $app_name"
+sudo -u yarn $SLIDER_INST_DIR/bin/slider destroy $app_name --manager $RM_ADDRESS || exit 1
+echo "SUCCESS"
+
+exit 0
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-assembly/src/main/bash/slider_setup
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/bash/slider_setup b/slider-assembly/src/main/bash/slider_setup
new file mode 100755
index 0000000..2c15c95
--- /dev/null
+++ b/slider-assembly/src/main/bash/slider_setup
@@ -0,0 +1,173 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+#
+# Base Vars
+#
+SCRIPT_NAME=`basename $0`
+SCRIPT_DIR=`cd $(dirname $0) && pwd`
+
+#
+# Functions
+#
+usage() {
+ echo "Usage: $SCRIPT_NAME -f </path/to/config> -p <product (hbase or storm)> -w <worker count> -n <app name>"
+ exit 1
+}
+
+#
+# Parse cmd line args
+#
+while getopts "f:p:w:n:" opt; do
+ case "$opt" in
+ f) config=$OPTARG;;
+ p) product=$OPTARG;;
+ w) worker_cnt=$OPTARG;;
+ n) app_name=$OPTARG;;
+ *) usage;;
+ esac
+done
+shift $((OPTIND-1))
+
+if [ -z "$config" ] || [ -z "$product" ] || [ -z "$worker_cnt" ] || [ -z "$app_name" ]; then
+ usage
+fi
+
+# Only support know products for now
+if [ $product != "hbase" ] && [ $product != "storm" ] && [ $product != "accumulo"]; then
+ echo "ERROR: Only HBase, Storm, and Accumulo are currently supported"
+ usage
+fi
+
+#
+# Source the config
+#
+source $config
+
+#
+# Product URLs
+#
+if [ $product = "hbase" ]; then
+ app_url=$HBASE_APP_URL
+elif [ $product = "storm" ]; then
+ app_url=$STORM_APP_URL
+elif [ $product = "accumulo" ]; then
+ app_url=$ACCUMULO_APP_URL
+fi
+
+#
+# Main
+#
+echo -e "\n## Creating slider install dir: $SLIDER_INST_DIR"
+mkdir -p $SLIDER_INST_DIR || exit 1
+chown yarn:hadoop $SLIDER_INST_DIR || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Downloading slider from: $SLIDER_URL"
+if [ -f /tmp/${SLIDER_VER}*tar ]; then
+ rm -f /tmp/${SLIDER_VER}*tar
+fi
+cd /tmp && wget $SLIDER_URL || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Extracting slider to $SLIDER_INST_DIR"
+tar -xf /tmp/${SLIDER_VER}*tar --strip-components=1 -C $SLIDER_INST_DIR || exit 1
+chown -R yarn:hadoop $SLIDER_INST_DIR || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Setting conf values"
+(cd $SCRIPT_DIR && cp slider-client.xml $SLIDER_INST_DIR/conf) || exit 1
+sed -i 's|@@RM_ADDRESS@@|'$RM_ADDRESS'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
+sed -i 's|@@RM_SCHED_ADDRESS@@|'$RM_SCHED_ADDRESS'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
+sed -i 's|@@DEFAULT_FS@@|'$DEFAULT_FS'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
+sed -i 's|@@YARN_CP@@|'$YARN_CP'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
+sed -i 's|@@ZK_QUORUM@@|'$ZK_QUORUM'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Setting PATH to include the JDK bin: $JDK_BIN"
+export PATH=$PATH:$JDK_BIN
+echo "SUCCESS"
+
+echo -e "\n## Checking version of Hadoop slider was compiled against"
+hadoop_compiled_ver=`$SLIDER_INST_DIR/bin/slider version | grep "Compiled against Hadoop" | awk '{print $NF}'`
+if [ "$hadoop_compiled_ver" != "2.4.0" ]; then
+ echo "ERROR: Compiled against Hadoop version $hadoop_compiled_ver instead of 2.4.0" && exit 1
+else
+ echo "Compiled against Hadoop version: $hadoop_compiled_ver"
+fi
+echo "SUCCESS"
+
+echo -e "\n## Setting up HDFS directories for slider"
+sudo -u hdfs hdfs dfs -mkdir -p /slider || exit 1
+sudo -u hdfs hdfs dfs -chown yarn:hdfs /slider || exit 1
+sudo -u hdfs hdfs dfs -mkdir -p /user/yarn || exit 1
+sudo -u hdfs hdfs dfs -chown yarn:hdfs /user/yarn || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Loading the Slider agent"
+sudo -u yarn hdfs dfs -mkdir -p /slider/agent/conf || exit 1
+sudo -u yarn hdfs dfs -copyFromLocal $SLIDER_INST_DIR/agent/* /slider/agent
+echo "SUCCESS"
+
+echo -e "\n## Downloading $product to $SLIDER_INST_DIR/apps/$product"
+sudo -u yarn mkdir -p $SLIDER_INST_DIR/apps/$product || exit 1
+(cd $SLIDER_INST_DIR/apps/$product && sudo -u yarn wget $app_url) || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Extracting $product in $SLIDER_INST_DIR/apps/$product"
+(cd $SLIDER_INST_DIR/apps/$product && sudo -u yarn unzip -o $product*zip) || exit 1
+echo "SUCCESS"
+
+echo -e "\n## Adding $product to HDFS slider dir"
+sudo -u yarn hdfs dfs -copyFromLocal $SLIDER_INST_DIR/apps/$product/$product*zip /slider
+echo "SUCCESS"
+
+echo -e "\n## Setting number of workers in $SLIDER_INST_DIR/apps/$product/resources.json"
+if [ $product = "hbase" ]; then
+ component="HBASE_REGIONSERVER"
+elif [ $product = "storm" ]; then
+ component="SUPERVISOR"
+elif [ $product = "accumulo" ]; then
+ component="ACCUMULO_TSERVER"
+fi
+# Update the resource.json file with worker_cnt
+python << END
+import json
+with open("$SLIDER_INST_DIR/apps/$product/resources.json", "r+") as f:
+ data = json.load(f)
+ data["components"]["$component"]["yarn.component.instances"] = "$worker_cnt"
+
+with open("$SLIDER_INST_DIR/apps/$product/resources.json", "w+") as f:
+ f.write(json.dumps(data, sort_keys = False, indent = 4))
+END
+echo "SUCCESS"
+
+# Handle HBase HDFS dir needs
+if [ $product = "hbase" ]; then
+ echo -e "\n## Creating hbase HDFS dir /apps/hbase"
+ sudo -u hdfs hdfs dfs -mkdir -p /apps/hbase || exit 1
+ sudo -u hdfs hdfs dfs -chown yarn:hdfs /apps/hbase || exit 1
+ echo "SUCCESS"
+fi
+
+echo -e "\n##Starting app $product with $worker_cnt workers via slider"
+sudo -u yarn $SLIDER_INST_DIR/bin/slider create $app_name \
+ --image $DEFAULT_FS/slider/agent/slider-agent.tar.gz \
+ --template $SLIDER_INST_DIR/apps/$product/appConfig.json \
+ --resources $SLIDER_INST_DIR/apps/$product/resources.json || exit 1
+echo "SUCCESS"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-assembly/src/main/bash/slider_setup.conf
----------------------------------------------------------------------
diff --git a/slider-assembly/src/main/bash/slider_setup.conf b/slider-assembly/src/main/bash/slider_setup.conf
new file mode 100644
index 0000000..fed3f4c
--- /dev/null
+++ b/slider-assembly/src/main/bash/slider_setup.conf
@@ -0,0 +1,60 @@
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Change values below to match the cluster being deployed on
+#
+
+# Zookeeper nodes
+ZK_QUORUM="zk1:2181,zk2:2181,zk3:2181"
+
+# Resource Manager address (yarn.resourcemanager.address)
+RM_ADDRESS="rm1:8050"
+
+# Resource Manager scheduler address (yarn.resourcemanager.scheduler.address)
+RM_SCHED_ADDRESS="rm1:8030"
+
+# Default FS (fs.defaultFS)
+DEFAULT_FS="hdfs://nn1:8020"
+
+
+#
+# Should not be necessary to change anything below except on new version releases
+#
+
+# Slider version
+SLIDER_VER="slider-0.30"
+
+# Slider URL
+SLIDER_URL="http://public-repo-1.hortonworks.com/slider/0.30/slider-0.30-all.tar"
+
+# HBase Slider App URL
+HBASE_APP_URL="http://public-repo-1.hortonworks.com/slider/0.30/apps/hbase_v096.zip"
+
+# Storm Slider App URL
+STORM_APP_URL="http://public-repo-1.hortonworks.com/slider/0.30/apps/storm_v091.zip"
+
+# Accumulo Slider App URL
+ACCUMULO_APP_URL="http://public-repo-1.hortonworks.com/slider/0.30/apps/accumulo_v151.zip"
+
+# Where to install Slider
+SLIDER_INST_DIR="/usr/lib/slider"
+
+# JDK bin to add to PATH
+JDK_BIN="/usr/jdk64/jdk1.7.0_45/bin"
+
+# Yarn container classpath (yarn.application.classpath)
+YARN_CP="/etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*"
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-install/README.md
----------------------------------------------------------------------
diff --git a/slider-install/README.md b/slider-install/README.md
deleted file mode 100644
index a4b7b08..0000000
--- a/slider-install/README.md
+++ /dev/null
@@ -1,102 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-
-slider_setup
-============
-
-Tools for installing, starting, and destroying HBase, Accumulo, and Storm slider apps on YARN.
-
-**WARNING: This is intended for POC/sandbox testing, may not be idempotent so DO NOT use on an existing Production cluster!!!**
-
-Setup
------
-1. Clone the repo
-2. Set the necessary cluster variables in `slider_setup.conf`, it shouldn't be necessary to change any other values but thoe ones below
-
-
- # Zookeeper nodes
- ZK_QUORUM="zk1:2181,zk2:2181,zk3:2181"
-
- # Resource Manager address (yarn.resourcemanager.address)
- RM_ADDRESS="rm1:8050"
-
- # Resource Manager scheduler address (yarn.resourcemanager.scheduler.address)
- RM_SCHED_ADDRESS="rm1:8030"
-
- # Default FS (fs.defaultFS)
- DEFAULT_FS="hdfs://nn1:8020"
-
-Running
--------
-* slider_setup is the main script and handles the following
- 1. Pulls down slider and extracts the contents to the SLIDER_INST_DIR
- 2. Modifies slider-client.xml with cluster related info
- 3. Pulls down the slider enabled version of the specified product
- 4. Creates necessary directories and copies required files to HDFS
- 5. For HBase, creates the app dir in HDFS
- 6. Submits the slider base application to the YARN cluster
-
-* The following args are required
- * -f - The path to the slider_setup.conf that has been modified with cluster info
- * -p - The product to run (hbase, accumulo, or storm are all that are supported at this time)
- * -w - The number of "worker" nodes. This has different meaning depending on product.
- * HBase - number of region servers
- * Accumulo - number of tablet servers
- * Storm - number of supervisors
- * -n - The name of the app, this will be the display name in the resource manager and is used by the teardown process
-
-* HBase Example:
-
-
- ./slider_setup -f slider_setup.conf -p hbase -w 5 -n hbase-slider
-
-* Accumulo Example:
-
-
- ./slider_setup -f slider_setup.conf -p accumulo -w 3 -n accumulo-slider
-
-* Storm Example:
-
-
- ./slider_setup -f slider_setup.conf -p storm -w 3 -n storm-slider
-
-Tear Down
----------
-
-* slider_destroy will do the following
- 1. Freeze the slider application based on provided name
- 2. Destory the slider application based on provided name
-
-* The following args are required
- * `-f` - The path to the `slider_setup.conf` that has been modified with cluster info
- * `-n` - The name of the app, this was provided to the slider_setup tool
-
-* HBase Example:
-
-
- ./slider_destroy -f slider_setup.conf -n hbase-slider
-
-* Accumulo Example:
-
-
- ./slider_destroy -f slider_setup.conf -n accumulo-slider
-
-* Storm Example:
-
-
- ./slider_destroy -f slider_setup.conf -n storm-slider
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-install/pom.xml
----------------------------------------------------------------------
diff --git a/slider-install/pom.xml b/slider-install/pom.xml
deleted file mode 100644
index 8dd2ec6..0000000
--- a/slider-install/pom.xml
+++ /dev/null
@@ -1,111 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <artifactId>slider-install</artifactId>
- <name>Slider Install</name>
- <packaging>pom</packaging>
- <description>
-
- Builds any RPMs and other install packaging for Slider.
- This is downstream of the slider assembly and the app packages, so
- that anything from these can be installed in the RPM.
- </description>
- <parent>
- <groupId>org.apache.slider</groupId>
- <artifactId>slider</artifactId>
- <version>0.41.0-incubating-SNAPSHOT</version>
- </parent>
-
- <properties>
- <basedir>/usr/local/slider</basedir>
- <confdir>${basedir}/conf</confdir>
- <bindir>${basedir}/bin</bindir>
- </properties>
-
- <build>
- <plugins>
- <!--read in a build.properties file if defined-->
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>properties-maven-plugin</artifactId>
- <version>${maven.properties.version}</version>
- <executions>
- <execution>
- <phase>initialize</phase>
- <goals>
- <goal>read-project-properties</goal>
- </goals>
- <configuration>
- <quiet>true</quiet>
- <files>
- <file>build.properties</file>
- <file>../build.properties</file>
- </files>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
-
-
- </build>
-
- <reporting>
- <plugins>
-
-
-
- </plugins>
- </reporting>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.slider</groupId>
- <artifactId>slider-core</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <!--
- needed to order the build and ensure the agent tar is found
- the test scope ensures that it isn't copied into the lib dir
- -->
- <dependency>
- <groupId>org.apache.slider</groupId>
- <artifactId>slider-agent</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <type>tar.gz</type>
- </dependency>
-
- <dependency>
- <groupId>com.beust</groupId>
- <artifactId>jcommander</artifactId>
- </dependency>
-
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- </dependency>
-
- </dependencies>
-
-
-</project>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-install/src/main/bash/slider-client.xml
----------------------------------------------------------------------
diff --git a/slider-install/src/main/bash/slider-client.xml b/slider-install/src/main/bash/slider-client.xml
deleted file mode 100644
index f7060c1..0000000
--- a/slider-install/src/main/bash/slider-client.xml
+++ /dev/null
@@ -1,83 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!--
- Properties set here are picked up in the client.
- They are not passed to the AM -though the filesystem
- binding details (URL And principal) are added to the
- hbase-site.xml file when a cluster is created.
--->
-<configuration>
-
- <property>
- <name>yarn.log-aggregation-enable</name>
- <value>true</value>
- </property>
-
-
- <property>
- <name>slider.yarn.queue</name>
- <value>default</value>
- <description>YARN queue for the Application Master</description>
- </property>
-
- <property>
- <name>yarn.resourcemanager.address</name>
- <value>@@RM_ADDRESS@@</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.scheduler.address</name>
- <value>@@RM_SCHED_ADDRESS@@</value>
- </property>
-
- <property>
- <name>fs.defaultFS</name>
- <value>@@DEFAULT_FS@@</value>
- </property>
-
- <property>
- <name>yarn.application.classpath</name>
- <value>@@YARN_CP@@</value>
- </property>
-
- <property>
- <name>slider.zookeeper.quorum</name>
- <value>@@ZK_QUORUM@@</value>
- </property>
-
-<!--
- <property>
- <name>yarn.resourcemanager.principal</name>
- <value>yarn/master@MINICLUSTER</value>
- </property>
-
- <property>
- <name>slider.security.enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>dfs.namenode.kerberos.principal</name>
- <value>hdfs/master@MINICLUSTER</value>
- </property>
--->
-
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-install/src/main/bash/slider_destroy
----------------------------------------------------------------------
diff --git a/slider-install/src/main/bash/slider_destroy b/slider-install/src/main/bash/slider_destroy
deleted file mode 100755
index 9039751..0000000
--- a/slider-install/src/main/bash/slider_destroy
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#
-# Base Vars
-#
-SCRIPT_NAME=`basename $0`
-SCRIPT_DIR=`cd $(dirname $0) && pwd`
-
-#
-# Functions
-#
-usage() {
- echo "Usage: $SCRIPT_NAME -f </path/to/config> -n <app name>"
- exit 1
-}
-
-#
-# Parse cmd line args
-#
-while getopts "f:n:" opt; do
- case "$opt" in
- f) config=$OPTARG;;
- n) app_name=$OPTARG;;
- *) usage;;
- esac
-done
-shift $((OPTIND-1))
-
-if [ -z "$config" ] || [ -z "$app_name" ]; then
- usage
-fi
-
-#
-# Source the config
-#
-source $config
-
-#
-# Main
-#
-echo -e "\n## Freezing app $app_name"
-sudo -u yarn $SLIDER_INST_DIR/bin/slider freeze $app_name --manager $RM_ADDRESS || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Destroying app $app_name"
-sudo -u yarn $SLIDER_INST_DIR/bin/slider destroy $app_name --manager $RM_ADDRESS || exit 1
-echo "SUCCESS"
-
-exit 0
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-install/src/main/bash/slider_setup
----------------------------------------------------------------------
diff --git a/slider-install/src/main/bash/slider_setup b/slider-install/src/main/bash/slider_setup
deleted file mode 100755
index 2c15c95..0000000
--- a/slider-install/src/main/bash/slider_setup
+++ /dev/null
@@ -1,173 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-#
-# Base Vars
-#
-SCRIPT_NAME=`basename $0`
-SCRIPT_DIR=`cd $(dirname $0) && pwd`
-
-#
-# Functions
-#
-usage() {
- echo "Usage: $SCRIPT_NAME -f </path/to/config> -p <product (hbase or storm)> -w <worker count> -n <app name>"
- exit 1
-}
-
-#
-# Parse cmd line args
-#
-while getopts "f:p:w:n:" opt; do
- case "$opt" in
- f) config=$OPTARG;;
- p) product=$OPTARG;;
- w) worker_cnt=$OPTARG;;
- n) app_name=$OPTARG;;
- *) usage;;
- esac
-done
-shift $((OPTIND-1))
-
-if [ -z "$config" ] || [ -z "$product" ] || [ -z "$worker_cnt" ] || [ -z "$app_name" ]; then
- usage
-fi
-
-# Only support know products for now
-if [ $product != "hbase" ] && [ $product != "storm" ] && [ $product != "accumulo"]; then
- echo "ERROR: Only HBase, Storm, and Accumulo are currently supported"
- usage
-fi
-
-#
-# Source the config
-#
-source $config
-
-#
-# Product URLs
-#
-if [ $product = "hbase" ]; then
- app_url=$HBASE_APP_URL
-elif [ $product = "storm" ]; then
- app_url=$STORM_APP_URL
-elif [ $product = "accumulo" ]; then
- app_url=$ACCUMULO_APP_URL
-fi
-
-#
-# Main
-#
-echo -e "\n## Creating slider install dir: $SLIDER_INST_DIR"
-mkdir -p $SLIDER_INST_DIR || exit 1
-chown yarn:hadoop $SLIDER_INST_DIR || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Downloading slider from: $SLIDER_URL"
-if [ -f /tmp/${SLIDER_VER}*tar ]; then
- rm -f /tmp/${SLIDER_VER}*tar
-fi
-cd /tmp && wget $SLIDER_URL || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Extracting slider to $SLIDER_INST_DIR"
-tar -xf /tmp/${SLIDER_VER}*tar --strip-components=1 -C $SLIDER_INST_DIR || exit 1
-chown -R yarn:hadoop $SLIDER_INST_DIR || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Setting conf values"
-(cd $SCRIPT_DIR && cp slider-client.xml $SLIDER_INST_DIR/conf) || exit 1
-sed -i 's|@@RM_ADDRESS@@|'$RM_ADDRESS'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
-sed -i 's|@@RM_SCHED_ADDRESS@@|'$RM_SCHED_ADDRESS'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
-sed -i 's|@@DEFAULT_FS@@|'$DEFAULT_FS'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
-sed -i 's|@@YARN_CP@@|'$YARN_CP'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
-sed -i 's|@@ZK_QUORUM@@|'$ZK_QUORUM'|g' $SLIDER_INST_DIR/conf/slider-client.xml || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Setting PATH to include the JDK bin: $JDK_BIN"
-export PATH=$PATH:$JDK_BIN
-echo "SUCCESS"
-
-echo -e "\n## Checking version of Hadoop slider was compiled against"
-hadoop_compiled_ver=`$SLIDER_INST_DIR/bin/slider version | grep "Compiled against Hadoop" | awk '{print $NF}'`
-if [ "$hadoop_compiled_ver" != "2.4.0" ]; then
- echo "ERROR: Compiled against Hadoop version $hadoop_compiled_ver instead of 2.4.0" && exit 1
-else
- echo "Compiled against Hadoop version: $hadoop_compiled_ver"
-fi
-echo "SUCCESS"
-
-echo -e "\n## Setting up HDFS directories for slider"
-sudo -u hdfs hdfs dfs -mkdir -p /slider || exit 1
-sudo -u hdfs hdfs dfs -chown yarn:hdfs /slider || exit 1
-sudo -u hdfs hdfs dfs -mkdir -p /user/yarn || exit 1
-sudo -u hdfs hdfs dfs -chown yarn:hdfs /user/yarn || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Loading the Slider agent"
-sudo -u yarn hdfs dfs -mkdir -p /slider/agent/conf || exit 1
-sudo -u yarn hdfs dfs -copyFromLocal $SLIDER_INST_DIR/agent/* /slider/agent
-echo "SUCCESS"
-
-echo -e "\n## Downloading $product to $SLIDER_INST_DIR/apps/$product"
-sudo -u yarn mkdir -p $SLIDER_INST_DIR/apps/$product || exit 1
-(cd $SLIDER_INST_DIR/apps/$product && sudo -u yarn wget $app_url) || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Extracting $product in $SLIDER_INST_DIR/apps/$product"
-(cd $SLIDER_INST_DIR/apps/$product && sudo -u yarn unzip -o $product*zip) || exit 1
-echo "SUCCESS"
-
-echo -e "\n## Adding $product to HDFS slider dir"
-sudo -u yarn hdfs dfs -copyFromLocal $SLIDER_INST_DIR/apps/$product/$product*zip /slider
-echo "SUCCESS"
-
-echo -e "\n## Setting number of workers in $SLIDER_INST_DIR/apps/$product/resources.json"
-if [ $product = "hbase" ]; then
- component="HBASE_REGIONSERVER"
-elif [ $product = "storm" ]; then
- component="SUPERVISOR"
-elif [ $product = "accumulo" ]; then
- component="ACCUMULO_TSERVER"
-fi
-# Update the resource.json file with worker_cnt
-python << END
-import json
-with open("$SLIDER_INST_DIR/apps/$product/resources.json", "r+") as f:
- data = json.load(f)
- data["components"]["$component"]["yarn.component.instances"] = "$worker_cnt"
-
-with open("$SLIDER_INST_DIR/apps/$product/resources.json", "w+") as f:
- f.write(json.dumps(data, sort_keys = False, indent = 4))
-END
-echo "SUCCESS"
-
-# Handle HBase HDFS dir needs
-if [ $product = "hbase" ]; then
- echo -e "\n## Creating hbase HDFS dir /apps/hbase"
- sudo -u hdfs hdfs dfs -mkdir -p /apps/hbase || exit 1
- sudo -u hdfs hdfs dfs -chown yarn:hdfs /apps/hbase || exit 1
- echo "SUCCESS"
-fi
-
-echo -e "\n##Starting app $product with $worker_cnt workers via slider"
-sudo -u yarn $SLIDER_INST_DIR/bin/slider create $app_name \
- --image $DEFAULT_FS/slider/agent/slider-agent.tar.gz \
- --template $SLIDER_INST_DIR/apps/$product/appConfig.json \
- --resources $SLIDER_INST_DIR/apps/$product/resources.json || exit 1
-echo "SUCCESS"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52f1f6fb/slider-install/src/main/bash/slider_setup.conf
----------------------------------------------------------------------
diff --git a/slider-install/src/main/bash/slider_setup.conf b/slider-install/src/main/bash/slider_setup.conf
deleted file mode 100644
index fed3f4c..0000000
--- a/slider-install/src/main/bash/slider_setup.conf
+++ /dev/null
@@ -1,60 +0,0 @@
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Change values below to match the cluster being deployed on
-#
-
-# Zookeeper nodes
-ZK_QUORUM="zk1:2181,zk2:2181,zk3:2181"
-
-# Resource Manager address (yarn.resourcemanager.address)
-RM_ADDRESS="rm1:8050"
-
-# Resource Manager scheduler address (yarn.resourcemanager.scheduler.address)
-RM_SCHED_ADDRESS="rm1:8030"
-
-# Default FS (fs.defaultFS)
-DEFAULT_FS="hdfs://nn1:8020"
-
-
-#
-# Should not be necessary to change anything below except on new version releases
-#
-
-# Slider version
-SLIDER_VER="slider-0.30"
-
-# Slider URL
-SLIDER_URL="http://public-repo-1.hortonworks.com/slider/0.30/slider-0.30-all.tar"
-
-# HBase Slider App URL
-HBASE_APP_URL="http://public-repo-1.hortonworks.com/slider/0.30/apps/hbase_v096.zip"
-
-# Storm Slider App URL
-STORM_APP_URL="http://public-repo-1.hortonworks.com/slider/0.30/apps/storm_v091.zip"
-
-# Accumulo Slider App URL
-ACCUMULO_APP_URL="http://public-repo-1.hortonworks.com/slider/0.30/apps/accumulo_v151.zip"
-
-# Where to install Slider
-SLIDER_INST_DIR="/usr/lib/slider"
-
-# JDK bin to add to PATH
-JDK_BIN="/usr/jdk64/jdk1.7.0_45/bin"
-
-# Yarn container classpath (yarn.application.classpath)
-YARN_CP="/etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*"
-
[37/50] [abbrv] git commit: Merge branch
'feature/SLIDER-320-endpoints-in-roleinstances' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-320-endpoints-in-roleinstances' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8050466f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8050466f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8050466f
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 8050466fc3d395e8a31fc760c612b9140e94e3a6
Parents: 4ea66fa 4610c25
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 15 16:11:25 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 15 16:11:25 2014 +0100
----------------------------------------------------------------------
.../providers/agent/AgentProviderService.java | 32 +++++-
.../server/appmaster/SliderAppMaster.java | 46 ++++++--
.../actions/RegisterComponentInstance.java | 48 ++++++++
.../actions/UnregisterComponentInstance.java | 46 ++++++++
.../slider/server/appmaster/state/AppState.java | 115 ++++++++++++++-----
.../appmaster/state/ProviderAppState.java | 17 ++-
.../server/appmaster/state/RoleInstance.java | 42 ++++++-
.../state/StateAccessForProviders.java | 13 ++-
.../appmaster/web/view/ContainerStatsBlock.java | 2 +-
.../server/appmaster/web/view/IndexBlock.java | 3 +-
.../slider/providers/agent/TestAgentEcho.groovy | 2 +-
.../providers/hbase/HBaseProviderService.java | 11 +-
12 files changed, 312 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
[22/50] [abbrv] git commit: SLIDER-262. Slider agent should provide
process supervision such as auto-restart.
Posted by st...@apache.org.
SLIDER-262. Slider agent should provide process supervision such as auto-restart.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/806ebb27
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/806ebb27
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/806ebb27
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 806ebb273f7b66027c771c2f13e2c8ef95cbe829
Parents: 506eb79
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Aug 13 07:53:31 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Aug 14 17:56:52 2014 -0700
----------------------------------------------------------------------
.../src/main/python/agent/ActionQueue.py | 1 +
.../src/main/python/agent/CommandStatusDict.py | 2 +
.../src/main/python/agent/Controller.py | 11 +-
.../src/test/python/agent/TestActionQueue.py | 7 +-
.../test/python/agent/TestCommandStatusDict.py | 94 +++++++++++++++
.../src/test/python/agent/TestController.py | 116 ++++++++++++++++++-
6 files changed, 224 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/806ebb27/slider-agent/src/main/python/agent/ActionQueue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/ActionQueue.py b/slider-agent/src/main/python/agent/ActionQueue.py
index 774d1e6..fba9aa2 100644
--- a/slider-agent/src/main/python/agent/ActionQueue.py
+++ b/slider-agent/src/main/python/agent/ActionQueue.py
@@ -135,6 +135,7 @@ class ActionQueue(threading.Thread):
'reportResult': reportResult
})
self.commandStatuses.put_command_status(command, in_progress_status, reportResult)
+
store_config = False
if ActionQueue.STORE_APPLIED_CONFIG in command['commandParams']:
store_config = 'true' == command['commandParams'][ActionQueue.STORE_APPLIED_CONFIG]
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/806ebb27/slider-agent/src/main/python/agent/CommandStatusDict.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CommandStatusDict.py b/slider-agent/src/main/python/agent/CommandStatusDict.py
index 9261e29..f20e0fa 100644
--- a/slider-agent/src/main/python/agent/CommandStatusDict.py
+++ b/slider-agent/src/main/python/agent/CommandStatusDict.py
@@ -114,12 +114,14 @@ class CommandStatusDict():
grep = Grep()
output = grep.tail(tmpout, Grep.OUTPUT_LAST_LINES)
inprogress = self.generate_report_template(command)
+ reportResult = not command[Constants.AUTO_GENERATED]
inprogress.update({
'stdout': grep.filterMarkup(output),
'stderr': tmperr,
'structuredOut': tmpstructuredout,
Constants.EXIT_CODE: 777,
'status': ActionQueue.IN_PROGRESS_STATUS,
+ 'reportResult': reportResult
})
return inprogress
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/806ebb27/slider-agent/src/main/python/agent/Controller.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Controller.py b/slider-agent/src/main/python/agent/Controller.py
index 77dcafb..6bdfc26 100644
--- a/slider-agent/src/main/python/agent/Controller.py
+++ b/slider-agent/src/main/python/agent/Controller.py
@@ -268,7 +268,8 @@ class Controller(threading.Thread):
if len(stored_command) > 0:
auto_start_command = self.create_start_command(stored_command)
if auto_start_command:
- self.updateStateBasedOnCommand([auto_start_command])
+ logger.info("Automatically adding a start command.")
+ self.updateStateBasedOnCommand([auto_start_command], False)
self.addToQueue([auto_start_command])
pass
@@ -343,21 +344,25 @@ class Controller(threading.Thread):
pass
logger.info("Controller stopped heart-beating.")
+
def create_start_command(self, stored_command):
taskId = int(stored_command['taskId'])
taskId = taskId + 1
stored_command['taskId'] = taskId
stored_command['commandId'] = "{0}-1".format(taskId)
stored_command[Constants.AUTO_GENERATED] = True
+ return stored_command
pass
- def updateStateBasedOnCommand(self, commands):
+
+ def updateStateBasedOnCommand(self, commands, createStatus=True):
for command in commands:
if command["roleCommand"] == "START":
self.componentExpectedState = State.STARTED
self.componentActualState = State.STARTING
self.failureCount = 0
- self.statusCommand = self.createStatusCommand(command)
+ if createStatus:
+ self.statusCommand = self.createStatusCommand(command)
if command["roleCommand"] == "INSTALL":
self.componentExpectedState = State.INSTALLED
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/806ebb27/slider-agent/src/test/python/agent/TestActionQueue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestActionQueue.py b/slider-agent/src/test/python/agent/TestActionQueue.py
index f0935a9..4a53c2e 100644
--- a/slider-agent/src/test/python/agent/TestActionQueue.py
+++ b/slider-agent/src/test/python/agent/TestActionQueue.py
@@ -274,6 +274,7 @@ class TestActionQueue(TestCase):
def test_execute_command(self, status_update_callback_mock, open_mock, json_load_mock,
resolve_script_path_mock):
+ self.assertEqual.__self__.maxDiff = None
tempdir = tempfile.gettempdir()
config = MagicMock()
config.get.return_value = "something"
@@ -344,7 +345,8 @@ class TestActionQueue(TestCase):
'role': u'HBASE_MASTER',
'actionId': '1-1',
'taskId': 3,
- 'exitcode': 777}
+ 'exitcode': 777,
+ 'reportResult': True}
self.assertEqual(report['reports'][0], expected)
# Continue command execution
unfreeze_flag.set()
@@ -406,7 +408,8 @@ class TestActionQueue(TestCase):
'actionId': '1-1',
'taskId': 3,
'structuredOut': '',
- 'exitcode': 13}
+ 'exitcode': 13,
+ 'reportResult': True}
self.assertEqual(len(report['reports']), 1)
self.assertEqual(report['reports'][0], expected)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/806ebb27/slider-agent/src/test/python/agent/TestCommandStatusDict.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestCommandStatusDict.py b/slider-agent/src/test/python/agent/TestCommandStatusDict.py
new file mode 100644
index 0000000..ee91de6
--- /dev/null
+++ b/slider-agent/src/test/python/agent/TestCommandStatusDict.py
@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import StringIO
+import ssl
+import unittest, threading
+import sys
+from CommandStatusDict import CommandStatusDict
+from mock.mock import patch, MagicMock, call, Mock
+import logging
+from threading import Event
+
+class TestCommandStatusDict(unittest.TestCase):
+
+ logger = logging.getLogger()
+
+ auto_hbase_install_command = {
+ 'commandType': 'EXECUTION_COMMAND',
+ 'role': u'HBASE',
+ 'roleCommand': u'INSTALL',
+ 'commandId': '1-1',
+ 'taskId': 7,
+ "componentName": "HBASE_MASTER",
+ 'clusterName': u'cc',
+ 'serviceName': u'HDFS',
+ 'auto_generated': True
+ }
+
+ @patch("__builtin__.open")
+ def test_generate_progress_report(self, open_mock):
+ csd = CommandStatusDict(None)
+ report = {}
+ report['tmpout'] = None
+ report['tmperr'] = None
+ report['structuredOut'] = None
+
+ # Make file read calls visible
+ def open_side_effect(file, mode):
+ if mode == 'r':
+ file_mock = MagicMock()
+ file_mock.read.return_value = "Read from " + str(file)
+ return file_mock
+ else:
+ return self.original_open(file, mode)
+
+ open_mock.side_effect = open_side_effect
+
+ inprogress = csd.generate_in_progress_report(self.auto_hbase_install_command, report)
+ expected = {
+ 'status': 'IN_PROGRESS',
+ 'stderr': 'Read from None',
+ 'stdout': 'Read from None',
+ 'clusterName': u'cc',
+ 'structuredOut': '{}',
+ 'reportResult': False,
+ 'roleCommand': u'INSTALL',
+ 'serviceName': u'HDFS',
+ 'role': u'HBASE',
+ 'actionId': '1-1',
+ 'taskId': 7,
+ 'exitcode': 777}
+ self.assertEqual(inprogress, expected)
+
+ self.auto_hbase_install_command['auto_generated'] = False
+ inprogress = csd.generate_in_progress_report(self.auto_hbase_install_command, report)
+ expected['reportResult'] = True
+ self.assertEqual(inprogress, expected)
+ pass
+
+if __name__ == "__main__":
+ logging.basicConfig(format='%(asctime)s %(message)s',level=logging.DEBUG)
+ unittest.main()
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/806ebb27/slider-agent/src/test/python/agent/TestController.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestController.py b/slider-agent/src/test/python/agent/TestController.py
index af62c67..91e12f1 100644
--- a/slider-agent/src/test/python/agent/TestController.py
+++ b/slider-agent/src/test/python/agent/TestController.py
@@ -42,7 +42,7 @@ class TestController(unittest.TestCase):
@patch.object(hostname, "hostname")
def setUp(self, hostname_method, NetUtil_mock, lockMock, threadMock):
- Controller.logger = MagicMock()
+ #Controller.logger = MagicMock()
lockMock.return_value = MagicMock()
NetUtil_mock.return_value = MagicMock()
hostname_method.return_value = "test_hostname"
@@ -589,9 +589,121 @@ class TestController(unittest.TestCase):
self.controller.config = original_value
pass
+ def test_create_start_command(self):
+ stored_command = {
+ 'commandType': 'EXECUTION_COMMAND',
+ 'role': u'HBASE_MASTER',
+ "componentName": "HBASE_MASTER",
+ 'roleCommand': u'INSTALL',
+ 'commandId': '1-1',
+ 'taskId': 3,
+ 'clusterName': u'cc',
+ 'serviceName': u'HBASE',
+ 'configurations': {'global': {}},
+ 'configurationTags': {'global': {'tag': 'v1'}},
+ 'auto_generated': False,
+ 'roleParams': {'auto_restart':'false'},
+ 'commandParams': {'script_type': 'PYTHON',
+ 'script': 'scripts/abc.py',
+ 'command_timeout': '600'}
+ }
+
+ expected = {
+ 'commandType': 'EXECUTION_COMMAND',
+ 'role': u'HBASE_MASTER',
+ "componentName": "HBASE_MASTER",
+ 'roleCommand': u'INSTALL',
+ 'commandId': '4-1',
+ 'taskId': 4,
+ 'clusterName': u'cc',
+ 'serviceName': u'HBASE',
+ 'configurations': {'global': {}},
+ 'configurationTags': {'global': {'tag': 'v1'}},
+ 'auto_generated': False,
+ 'roleParams': {'auto_restart':'false'},
+ 'commandParams': {'script_type': 'PYTHON',
+ 'script': 'scripts/abc.py',
+ 'command_timeout': '600'},
+ 'auto_generated': True
+ }
+
+ modified_command = self.controller.create_start_command(stored_command)
+ self.assertEqual.__self__.maxDiff = None
+ self.assertEqual(modified_command, expected)
+
+ @patch.object(Controller.Controller, "createStatusCommand")
+ @patch.object(threading._Event, "wait")
+ @patch("time.sleep")
+ @patch("json.loads")
+ @patch("json.dumps")
+ def test_auto_start(self, dumpsMock, loadsMock, timeMock, waitMock, mock_createStatusCommand):
+ original_value = self.controller.config
+ self.controller.config = AgentConfig("", "")
+ out = StringIO.StringIO()
+ sys.stdout = out
+
+ heartbeat = MagicMock()
+ self.controller.heartbeat = heartbeat
+
+ dumpsMock.return_value = "data"
+
+ sendRequest = MagicMock(name="sendRequest")
+ self.controller.sendRequest = sendRequest
+
+ self.controller.responseId = 1
+ response = {"responseId":"2", "restartAgent":"false", "restartEnabled":'true'}
+ loadsMock.return_value = response
+
+ def one_heartbeat(*args, **kwargs):
+ self.controller.DEBUG_STOP_HEARTBEATING = True
+ return "data"
+
+ sendRequest.side_effect = one_heartbeat
+
+ actionQueue = MagicMock()
+ actionQueue.isIdle.return_value = True
+
+ # one successful request, after stop
+ self.controller.actionQueue = actionQueue
+ self.controller.componentActualState = State.INSTALLED
+ self.controller.componentExpectedState = State.STARTED
+ self.assertTrue(self.controller.componentActualState, State.INSTALLED)
+ self.controller.actionQueue.customServiceOrchestrator.stored_command = {
+ 'commandType': 'EXECUTION_COMMAND',
+ 'role': u'HBASE',
+ 'roleCommand': u'START',
+ 'commandId': '7-1',
+ 'taskId': 7,
+ "componentName": "HBASE_MASTER",
+ 'clusterName': u'cc',
+ 'serviceName': u'HDFS'
+ }
+ addToQueue = MagicMock(name="addToQueue")
+ self.controller.addToQueue = addToQueue
+
+ self.controller.heartbeatWithServer()
+ self.assertTrue(sendRequest.called)
+
+ self.assertTrue(self.controller.componentActualState, State.STARTING)
+ self.assertTrue(self.controller.componentExpectedState, State.STARTED)
+ self.assertEquals(self.controller.failureCount, 0)
+ self.assertFalse(mock_createStatusCommand.called)
+ addToQueue.assert_has_calls([call([{
+ 'commandType': 'EXECUTION_COMMAND',
+ 'clusterName': u'cc',
+ 'serviceName': u'HDFS',
+ 'role': u'HBASE',
+ 'taskId': 8,
+ 'roleCommand': u'START',
+ 'componentName': 'HBASE_MASTER',
+ 'commandId': '8-1',
+ 'auto_generated': True}])])
+ self.controller.config = original_value
+ pass
+
if __name__ == "__main__":
- logging.basicConfig(format='%(asctime)s %(message)s',level=logging.DEBUG)
+ logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)
unittest.main()
[13/50] [abbrv] git commit: SLIDER-320 Action PublishEndpointDetails
Posted by st...@apache.org.
SLIDER-320 Action PublishEndpointDetails
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/74cca8a9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/74cca8a9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/74cca8a9
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 74cca8a9f351fbc25e7a617eacce8a330224e4a9
Parents: 52f1f6f
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 14 16:19:34 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 14 16:19:34 2014 +0100
----------------------------------------------------------------------
.../actions/PublishEndpointDetails.java | 45 ++++++++++++++++++++
1 file changed, 45 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/74cca8a9/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishEndpointDetails.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishEndpointDetails.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishEndpointDetails.java
new file mode 100644
index 0000000..67198a2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishEndpointDetails.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.state.AppState;
+
+import java.util.concurrent.TimeUnit;
+
+public class PublishEndpointDetails extends AsyncAction {
+
+
+ public final ContainerId containerId;
+
+ public PublishEndpointDetails(ContainerId containerId, long delay,
+ TimeUnit timeUnit) {
+ super("PublishEndpointDetails :" + containerId.toString(),
+ delay, timeUnit);
+ this.containerId = containerId;
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster,
+ QueueAccess queueService,
+ AppState appState) throws Exception {
+
+ }
+}
[33/50] [abbrv] git commit: SLIDER-320 rename activeContainers in
appState (and all related methods) to ownedContainers ... differentiates from
livecontainers better
Posted by st...@apache.org.
SLIDER-320 rename activeContainers in appState (and all related methods) to ownedContainers ... differentiates from livecontainers better
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7e2ce276
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7e2ce276
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7e2ce276
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 7e2ce2764153fea9a586d809fcf1ebd4f593255e
Parents: 41e0c5b
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 15 14:29:32 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 15 14:29:32 2014 +0100
----------------------------------------------------------------------
.../server/appmaster/SliderAppMaster.java | 2 +-
.../slider/server/appmaster/state/AppState.java | 72 +++++++++++++-------
.../appmaster/state/ProviderAppState.java | 12 ++--
.../state/StateAccessForProviders.java | 6 +-
.../appmaster/web/view/ContainerStatsBlock.java | 2 +-
.../server/appmaster/web/view/IndexBlock.java | 3 +-
6 files changed, 61 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e2ce276/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index acd6870..9a16188 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -1565,7 +1565,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
throws SliderException {
log.info("containerLostContactWithProvider: container {} lost",
containerId);
- RoleInstance activeContainer = appState.getActiveContainer(containerId);
+ RoleInstance activeContainer = appState.getOwnedContainer(containerId);
if (activeContainer != null) {
executeRMOperations(appState.releaseContainer(containerId));
// ask for more containers if needed
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e2ce276/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 0d2afcc..2f2eadd 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -163,7 +163,7 @@ public class AppState {
* Hash map of the containers we have. This includes things that have
* been allocated but are not live; it is a superset of the live list
*/
- private final ConcurrentMap<ContainerId, RoleInstance> activeContainers =
+ private final ConcurrentMap<ContainerId, RoleInstance> ownedContainers =
new ConcurrentHashMap<ContainerId, RoleInstance>();
/**
@@ -792,23 +792,49 @@ public class AppState {
}
- public synchronized List<RoleInstance> cloneActiveContainerList() {
- Collection<RoleInstance> values = activeContainers.values();
+ /**
+ * Clone the list of active (==owned) containers
+ * @return the list of role instances representing all owned containers
+ */
+ public synchronized List<RoleInstance> cloneOwnedContainerList() {
+ Collection<RoleInstance> values = ownedContainers.values();
return new ArrayList<RoleInstance>(values);
}
-
- public int getNumActiveContainers() {
- return activeContainers.size();
+ /**
+ * Get the number of active (==owned) containers
+ * @return
+ */
+ public int getNumOwnedContainers() {
+ return ownedContainers.size();
}
+ /**
+ * Look up an active container: any container that the AM has, even
+ * if it is not currently running/live
+ */
+ public RoleInstance getOwnedContainer(ContainerId id) {
+ return ownedContainers.get(id);
+ }
- public RoleInstance getActiveContainer(ContainerId id) {
- return activeContainers.get(id);
+ /**
+ * Remove an owned container
+ * @param id container ID
+ * @return the instance removed
+ */
+ private RoleInstance removeOwnedContainer(ContainerId id) {
+ return ownedContainers.remove(id);
}
-
- private RoleInstance removeActiveContainer(ContainerId id) {
- return activeContainers.remove(id);
+
+ /**
+ * set/update an owned container
+ * @param id container ID
+ * @param instance
+ * @return
+ */
+ private RoleInstance putOwnedContainer(ContainerId id,
+ RoleInstance instance) {
+ return ownedContainers.put(id, instance);
}
/**
@@ -885,7 +911,7 @@ public class AppState {
boolean active) {
List<RoleInstance> nodes = new ArrayList<RoleInstance>();
Collection<RoleInstance> allRoleInstances;
- allRoleInstances = active? activeContainers.values() : liveNodes.values();
+ allRoleInstances = active? ownedContainers.values() : liveNodes.values();
for (RoleInstance node : allRoleInstances) {
if (node.roleId == roleId) {
nodes.add(node);
@@ -943,7 +969,7 @@ public class AppState {
instance.container = container;
instance.createTime = now();
getStartingNodes().put(container.getId(), instance);
- activeContainers.put(container.getId(), instance);
+ putOwnedContainer(container.getId(), instance);
roleHistory.onContainerStartSubmitted(container, instance);
}
@@ -960,7 +986,7 @@ public class AppState {
throws SliderInternalStateException {
ContainerId id = container.getId();
//look up the container
- RoleInstance instance = getActiveContainer(id);
+ RoleInstance instance = getOwnedContainer(id);
if (instance == null) {
throw new SliderInternalStateException(
"No active container with ID " + id);
@@ -1110,7 +1136,7 @@ public class AppState {
@VisibleForTesting
public RoleInstance innerOnNodeManagerContainerStarted(ContainerId containerId) {
incStartedCountainerCount();
- RoleInstance instance = activeContainers.get(containerId);
+ RoleInstance instance = getOwnedContainer(containerId);
if (instance == null) {
//serious problem
throw new YarnRuntimeException("Container not in active containers start "+
@@ -1146,7 +1172,7 @@ public class AppState {
*/
public synchronized void onNodeManagerContainerStartFailed(ContainerId containerId,
Throwable thrown) {
- activeContainers.remove(containerId);
+ removeOwnedContainer(containerId);
incFailedCountainerCount();
incStartFailedCountainerCount();
RoleInstance instance = getStartingNodes().remove(containerId);
@@ -1246,7 +1272,7 @@ public class AppState {
} else {
//a container has failed
result.containerFailed = true;
- roleInstance = activeContainers.remove(containerId);
+ roleInstance = removeOwnedContainer(containerId);
if (roleInstance != null) {
//it was active, move it to failed
incFailedCountainerCount();
@@ -1320,14 +1346,14 @@ public class AppState {
}
// and the active node list if present
- removeActiveContainer(containerId);
+ removeOwnedContainer(containerId);
// finally, verify the node doesn't exist any more
assert !containersBeingReleased.containsKey(
containerId) : "container still in release queue";
assert !getLiveNodes().containsKey(
containerId) : " container still in live nodes";
- assert getActiveContainer(containerId) ==
+ assert getOwnedContainer(containerId) ==
null : "Container still in active container list";
return result;
@@ -1623,7 +1649,7 @@ public class AppState {
public List<AbstractRMOperation> releaseContainer(ContainerId containerId)
throws SliderInternalStateException {
List<AbstractRMOperation> operations = new ArrayList<AbstractRMOperation>();
- List<RoleInstance> activeRoleInstances = cloneActiveContainerList();
+ List<RoleInstance> activeRoleInstances = cloneOwnedContainerList();
for (RoleInstance role : activeRoleInstances) {
if (role.container.getId().equals(containerId)) {
containerReleaseSubmitted(role.container);
@@ -1645,7 +1671,7 @@ public class AppState {
* that can be released.
*/
private RoleInstance findRoleInstanceOnHost(NodeInstance node, int roleId) {
- Collection<RoleInstance> targets = cloneActiveContainerList();
+ Collection<RoleInstance> targets = cloneOwnedContainerList();
String hostname = node.hostname;
for (RoleInstance ri : targets) {
if (hostname.equals(RoleHistoryUtils.hostnameOf(ri.container))
@@ -1663,7 +1689,7 @@ public class AppState {
*/
public synchronized List<AbstractRMOperation> releaseAllContainers() {
- Collection<RoleInstance> targets = cloneActiveContainerList();
+ Collection<RoleInstance> targets = cloneOwnedContainerList();
log.info("Releasing {} containers", targets.size());
List<AbstractRMOperation> operations =
new ArrayList<AbstractRMOperation>(targets.size());
@@ -1817,7 +1843,7 @@ public class AppState {
instance.container = container;
instance.createTime = now();
instance.state = ClusterDescription.STATE_LIVE;
- activeContainers.put(cid, instance);
+ putOwnedContainer(cid, instance);
//role history gets told
roleHistory.onContainerAssigned(container);
// pretend the container has just had its start actions submitted
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e2ce276/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
index a5b36d4..e7384c8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
@@ -164,18 +164,18 @@ public class ProviderAppState implements StateAccessForProviders {
}
@Override
- public List<RoleInstance> cloneActiveContainerList() {
- return appState.cloneActiveContainerList();
+ public List<RoleInstance> cloneOwnedContainerList() {
+ return appState.cloneOwnedContainerList();
}
@Override
- public int getNumActiveContainers() {
- return appState.getNumActiveContainers();
+ public int getNumOwnedContainers() {
+ return appState.getNumOwnedContainers();
}
@Override
- public RoleInstance getActiveContainer(ContainerId id) {
- return appState.getActiveContainer(id);
+ public RoleInstance getOwnedContainer(ContainerId id) {
+ return appState.getOwnedContainer(id);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e2ce276/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
index acba8cc..b236dfc 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
@@ -144,20 +144,20 @@ public interface StateAccessForProviders {
* @return the active containers at the time
* the call was made
*/
- List<RoleInstance> cloneActiveContainerList();
+ List<RoleInstance> cloneOwnedContainerList();
/**
* Get the number of active containers
* @return the number of active containers the time the call was made
*/
- int getNumActiveContainers();
+ int getNumOwnedContainers();
/**
* Get any active container with the given ID
* @param id container Id
* @return the active container or null if it is not found
*/
- RoleInstance getActiveContainer(ContainerId id);
+ RoleInstance getOwnedContainer(ContainerId id);
/**
* Create a clone of the list of live cluster nodes.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e2ce276/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
index 5645e0e..95f0417 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
@@ -96,7 +96,7 @@ public class ContainerStatsBlock extends HtmlBlock {
protected void render(Block html) {
// TODO Probably better to just get a copy of this list for us to avoid the repeated synchronization?
// does this change if we have 50 node, 100node, 500 node clusters?
- final Map<String,RoleInstance> containerInstances = getContainerInstances(slider.getAppState().cloneActiveContainerList());
+ final Map<String,RoleInstance> containerInstances = getContainerInstances(slider.getAppState().cloneOwnedContainerList());
for (Entry<String,RoleStatus> entry : slider.getRoleStatusByName().entrySet()) {
final String name = entry.getKey();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7e2ce276/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
index 90a3ee7..54bdb09 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
@@ -30,7 +30,6 @@ import org.apache.slider.server.appmaster.web.WebAppApi;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.net.URL;
import java.util.Map;
import java.util.Map.Entry;
@@ -63,7 +62,7 @@ public class IndexBlock extends HtmlBlock {
UL<DIV<Hamlet>> ul = div.ul();
- ul.li("Total number of containers for cluster: " + appState.getNumActiveContainers());
+ ul.li("Total number of containers for cluster: " + appState.getNumOwnedContainers());
ul.li("Cluster created: " + getInfoAvoidingNulls(StatusKeys.INFO_CREATE_TIME_HUMAN));
ul.li("Cluster last flexed: " + getInfoAvoidingNulls(StatusKeys.INFO_FLEX_TIME_HUMAN));
ul.li("Cluster running since: " + getInfoAvoidingNulls(StatusKeys.INFO_LIVE_TIME_HUMAN));
[29/50] [abbrv] git commit: SLIDER-326 regression: integration tests
failing in/after freeze
Posted by st...@apache.org.
SLIDER-326 regression: integration tests failing in/after freeze
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b8c27622
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b8c27622
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b8c27622
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: b8c27622820a688e02c197f66b5e3725c4f18919
Parents: 15da279
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 15 13:46:35 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 15 13:46:35 2014 +0100
----------------------------------------------------------------------
.../slider/funtest/framework/CommandTestBase.groovy | 16 +++++++++++++++-
.../lifecycle/AgentClusterLifecycleIT.groovy | 11 ++++++++---
.../hbase/funtest/HBaseClusterLifecycleIT.groovy | 16 ++++++++++++----
.../hbase/funtest/HBaseIntegrationIT.groovy | 5 ++++-
.../providers/hbase/funtest/HBaseLoadIT.groovy | 7 +++++++
src/test/clusters/sandbox/slider/slider-client.xml | 2 +-
6 files changed, 47 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b8c27622/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index ba7be53..278bd2b 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -211,6 +211,20 @@ abstract class CommandTestBase extends SliderTestUtils {
])
}
+
+ static SliderShell freeze(
+ int exitCode,
+ String name,
+ Collection<String> args) {
+ slider(exitCode, [ACTION_FREEZE, name] + args)
+ }
+
+ /**
+ * Freeze cluster: no exit code checking
+ * @param name
+ * @param args
+ * @return
+ */
static SliderShell freeze(String name, Collection<String> args) {
slider([ACTION_FREEZE, name] + args)
}
@@ -288,7 +302,7 @@ abstract class CommandTestBase extends SliderTestUtils {
}
static SliderShell thaw(String name, Collection<String> args) {
- slider([ACTION_THAW, name] + args)
+ slider(0, [ACTION_THAW, name] + args)
}
static SliderShell registry(int result, Collection<String> commands) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b8c27622/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
index b30d670..6b0f2bd 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
@@ -126,7 +126,7 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
log.info("Connected via Client {}", sliderClient.toString())
//freeze
- freeze(CLUSTER, [
+ freeze(0, CLUSTER, [
ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
ARG_MESSAGE, "freeze-in-test-cluster-lifecycle"
])
@@ -145,7 +145,7 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
exists(0, CLUSTER)
describe " >>> Cluster is now thawed."
- freeze(CLUSTER,
+ freeze(0, CLUSTER,
[
ARG_FORCE,
ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
@@ -178,7 +178,12 @@ public class AgentClusterLifecycleIT extends AgentCommandTestBase
StatusKeys.INFO_CONTAINERS_AM_RESTART)
assert restarted != null
assert Integer.parseInt(restarted) == 0
- freeze(CLUSTER)
+ freeze(0, CLUSTER,
+ [
+ ARG_FORCE,
+ ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
+ ARG_MESSAGE, "final-shutdown"
+ ])
destroy(0, CLUSTER)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b8c27622/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy
index aca12ef..63b5fb6 100644
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy
@@ -118,11 +118,13 @@ public class HBaseClusterLifecycleIT extends HBaseCommandTestBase
log.info("Connected via Client {}", sliderClient.toString())
//freeze
- freeze(CLUSTER, [
+ def frozen = freeze(0, CLUSTER, [
ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
- ARG_MESSAGE, "freeze-in-test cluster lifecycle"
+ ARG_MESSAGE, "freeze-in-test-cluster-lifecycle"
])
+ frozen.assertExitCode(0)
+// sleep(FREEZE_WAIT_TIME)
//cluster exists if you don't want it to be live
exists(0, CLUSTER, false)
// condition returns false if it is required to be live
@@ -136,7 +138,7 @@ public class HBaseClusterLifecycleIT extends HBaseCommandTestBase
ARG_WAIT, Integer.toString(THAW_WAIT_TIME),
])
exists(0, CLUSTER)
- freeze(CLUSTER,
+ freeze(0, CLUSTER,
[
ARG_FORCE,
ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
@@ -165,7 +167,13 @@ public class HBaseClusterLifecycleIT extends HBaseCommandTestBase
StatusKeys.INFO_CONTAINERS_AM_RESTART)
assert restarted != null
assert Integer.parseInt(restarted) == 0
- freeze(CLUSTER)
+ freeze(0, CLUSTER,
+ [
+ ARG_FORCE,
+ ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
+ ARG_MESSAGE, "teardown-freeze"
+ ])
+
destroy(0, CLUSTER)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b8c27622/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseIntegrationIT.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseIntegrationIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseIntegrationIT.groovy
index 69da02a..21a7494 100644
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseIntegrationIT.groovy
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseIntegrationIT.groovy
@@ -24,13 +24,16 @@ import org.apache.hadoop.hbase.IntegrationTestingUtility
import org.apache.hadoop.util.ToolRunner
import org.apache.slider.api.ClusterDescription
import org.apache.slider.client.SliderClient
-import org.apache.slider.providers.hbase.HBaseConfigFileOptions;
+import org.apache.slider.providers.hbase.HBaseConfigFileOptions
+import org.junit.Ignore;
/* Runs IntegrationTestIngest on cluster
*
* Note: this test runs for about 20 minutes
* please set slider.test.timeout.seconds accordingly
*/
+
+@Ignore("appears localhost only")
class HBaseIntegrationIT extends FunctionalHBaseClusterIT {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b8c27622/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseLoadIT.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseLoadIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseLoadIT.groovy
index 7393186..61bcc70 100644
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseLoadIT.groovy
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseLoadIT.groovy
@@ -23,6 +23,8 @@ import org.apache.hadoop.hbase.util.LoadTestTool
import org.apache.slider.api.ClusterDescription
import org.apache.slider.client.SliderClient
import org.apache.slider.providers.hbase.HBaseConfigFileOptions
+import org.junit.Assume
+import org.junit.Ignore
class HBaseLoadIT extends FunctionalHBaseClusterIT {
@@ -32,6 +34,11 @@ class HBaseLoadIT extends FunctionalHBaseClusterIT {
}
@Override
+ void testHBaseCreateCluster() throws Throwable {
+ super.testHBaseCreateCluster()
+ }
+
+ @Override
void clusterOperations(
String clustername,
SliderClient sliderClient,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b8c27622/src/test/clusters/sandbox/slider/slider-client.xml
----------------------------------------------------------------------
diff --git a/src/test/clusters/sandbox/slider/slider-client.xml b/src/test/clusters/sandbox/slider/slider-client.xml
index 30937ec..5ac5d59 100644
--- a/src/test/clusters/sandbox/slider/slider-client.xml
+++ b/src/test/clusters/sandbox/slider/slider-client.xml
@@ -52,7 +52,7 @@
<property>
<name>slider.test.agent.enabled</name>
<description>Flag to enable/disable Agent tests</description>
- <value>false</value>
+ <value>true</value>
</property>
[44/50] [abbrv] git commit: SLIDER-332 fixed a couple of license
headers (Gour Saha via billie)
Posted by st...@apache.org.
SLIDER-332 fixed a couple of license headers (Gour Saha via billie)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/624cafe7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/624cafe7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/624cafe7
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 624cafe78b4bbacd4ccb769edc06a4319b82ef26
Parents: 80eec9b
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Mon Aug 18 18:25:47 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Mon Aug 18 18:25:47 2014 -0700
----------------------------------------------------------------------
slider-agent/src/main/python/kazoo/handlers/gevent.py | 7 +++++--
slider-agent/src/main/python/kazoo/handlers/threading.py | 5 +++--
slider-agent/src/main/python/kazoo/tests/__init__.py | 1 +
3 files changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/624cafe7/slider-agent/src/main/python/kazoo/handlers/gevent.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/handlers/gevent.py b/slider-agent/src/main/python/kazoo/handlers/gevent.py
index 5e07143..060320c 100644
--- a/slider-agent/src/main/python/kazoo/handlers/gevent.py
+++ b/slider-agent/src/main/python/kazoo/handlers/gevent.py
@@ -1,5 +1,8 @@
-"""license: Apache License 2.0, see LICENSE for more details."""
-"""A gevent based handler."""
+"""
+license: Apache License 2.0, see LICENSE for more details.
+A gevent based handler.
+"""
+
from __future__ import absolute_import
import atexit
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/624cafe7/slider-agent/src/main/python/kazoo/handlers/threading.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/handlers/threading.py b/slider-agent/src/main/python/kazoo/handlers/threading.py
index b2e355a..3ca9a8f 100644
--- a/slider-agent/src/main/python/kazoo/handlers/threading.py
+++ b/slider-agent/src/main/python/kazoo/handlers/threading.py
@@ -1,5 +1,6 @@
-"""license: Apache License 2.0, see LICENSE for more details."""
-"""A threading based handler.
+"""
+license: Apache License 2.0, see LICENSE for more details.
+A threading based handler.
The :class:`SequentialThreadingHandler` is intended for regular Python
environments that use threads.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/624cafe7/slider-agent/src/main/python/kazoo/tests/__init__.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/__init__.py b/slider-agent/src/main/python/kazoo/tests/__init__.py
index e69de29..901253b 100644
--- a/slider-agent/src/main/python/kazoo/tests/__init__.py
+++ b/slider-agent/src/main/python/kazoo/tests/__init__.py
@@ -0,0 +1 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
[05/50] [abbrv] git commit: SLIDER-314 factor out internal keys into
class InternalKeys
Posted by st...@apache.org.
SLIDER-314 factor out internal keys into class InternalKeys
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1f6490d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1f6490d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1f6490d5
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 1f6490d5f6ec8d73f5cdb6f04600aa180be25b81
Parents: b111268
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 13 15:03:34 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 13 15:03:34 2014 +0100
----------------------------------------------------------------------
.../api/ClusterDescriptionOperations.java | 12 +-
.../org/apache/slider/api/InternalKeys.java | 114 +++++++++++++++++++
.../java/org/apache/slider/api/OptionKeys.java | 69 +----------
.../org/apache/slider/client/SliderClient.java | 7 +-
.../slider/core/build/InstanceBuilder.java | 3 +-
.../apache/slider/providers/ProviderUtils.java | 9 +-
.../slideram/SliderAMClientProvider.java | 4 +-
.../server/appmaster/SliderAppMaster.java | 22 ++--
.../agent/TestAgentProviderService.java | 9 +-
.../accumulo/AccumuloClientProvider.java | 3 +-
.../accumulo/AccumuloProviderService.java | 9 +-
.../providers/hbase/HBaseClientProvider.java | 3 +-
.../providers/hbase/HBaseProviderService.java | 4 +-
...reezeReconfigureThawLiveRegionService.groovy | 4 +-
14 files changed, 166 insertions(+), 106 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionOperations.java b/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionOperations.java
index 7e73a92..21ece2b 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionOperations.java
@@ -64,15 +64,15 @@ public class ClusterDescriptionOperations {
MapOperations appOptions =
aggregateConf.getAppConfOperations().getGlobalOptions();
- cd.type = internalOptions.getOption(OptionKeys.INTERNAL_PROVIDER_NAME,
+ cd.type = internalOptions.getOption(InternalKeys.INTERNAL_PROVIDER_NAME,
SliderProviderFactory.DEFAULT_CLUSTER_TYPE);
- cd.dataPath = internalOptions.get(OptionKeys.INTERNAL_DATA_DIR_PATH);
+ cd.dataPath = internalOptions.get(InternalKeys.INTERNAL_DATA_DIR_PATH);
cd.name = internalOptions.get(OptionKeys.APPLICATION_NAME);
- cd.originConfigurationPath = internalOptions.get(OptionKeys.INTERNAL_SNAPSHOT_CONF_PATH);
- cd.generatedConfigurationPath = internalOptions.get(OptionKeys.INTERNAL_GENERATED_CONF_PATH);
- cd.setImagePath(internalOptions.get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH));
- cd.setApplicationHome(internalOptions.get(OptionKeys.INTERNAL_APPLICATION_HOME));
+ cd.originConfigurationPath = internalOptions.get(InternalKeys.INTERNAL_SNAPSHOT_CONF_PATH);
+ cd.generatedConfigurationPath = internalOptions.get(InternalKeys.INTERNAL_GENERATED_CONF_PATH);
+ cd.setImagePath(internalOptions.get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH));
+ cd.setApplicationHome(internalOptions.get(InternalKeys.INTERNAL_APPLICATION_HOME));
cd.setZkPath(appOptions.get(ZOOKEEPER_PATH));
cd.setZkHosts(appOptions.get(ZOOKEEPER_QUORUM));
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
new file mode 100644
index 0000000..4045c0e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api;
+
+/**
+ * Keys for internal use, go into `internal.json` and not intended for normal
+ * use except when tuning Slider AM operations
+ */
+public interface InternalKeys {
+
+
+ /**
+ * Home dir of the app: {@value}
+ * If set, implies there is a home dir to use
+ */
+ String INTERNAL_APPLICATION_HOME = "internal.application.home";
+ /**
+ * Path to an image file containing the app: {@value}
+ */
+ String INTERNAL_APPLICATION_IMAGE_PATH = "internal.application.image.path";
+ /**
+ * Time in milliseconds to wait after forking any in-AM
+ * process before attempting to start up the containers: {@value}
+ *
+ * A shorter value brings the cluster up faster, but means that if the
+ * in AM process fails (due to a bad configuration), then time
+ * is wasted starting containers on a cluster that isn't going to come
+ * up
+ */
+ String INTERNAL_CONTAINER_STARTUP_DELAY = "internal.container.startup.delay";
+ /**
+ * internal temp directory: {@value}
+ */
+ String INTERNAL_AM_TMP_DIR = "internal.tmp.dir";
+ /**
+ * where a snapshot of the original conf dir is: {@value}
+ */
+ String INTERNAL_SNAPSHOT_CONF_PATH = "internal.snapshot.conf.path";
+ /**
+ * where a snapshot of the original conf dir is: {@value}
+ */
+ String INTERNAL_GENERATED_CONF_PATH = "internal.generated.conf.path";
+ /**
+ * where a snapshot of the original conf dir is: {@value}
+ */
+ String INTERNAL_PROVIDER_NAME = "internal.provider.name";
+ /**
+ * where a snapshot of the original conf dir is: {@value}
+ */
+ String INTERNAL_DATA_DIR_PATH = "internal.data.dir.path";
+ /**
+ * Time in milliseconds to wait after forking any in-AM
+ * process before attempting to start up the containers: {@value}
+ *
+ * A shorter value brings the cluster up faster, but means that if the
+ * in AM process fails (due to a bad configuration), then time
+ * is wasted starting containers on a cluster that isn't going to come
+ * up
+ */
+ int DEFAULT_INTERNAL_CONTAINER_STARTUP_DELAY = 5000;
+ /**
+ * Version of the app: {@value}
+ */
+ String KEYTAB_LOCATION = "internal.keytab.location";
+
+
+ /**
+ * Flag to indicate whether or not the chaos monkey is enabled:
+ * {@value}
+ */
+ String INTERNAL_CHAOS_MONKEY_ENABLED = "internal.chaos.monkey.enabled";
+ boolean DEFAULT_INTERNAL_CHAOS_MONKEY_ENABLED = false;
+
+
+ /**
+ * Rate
+ */
+
+ String INTERNAL_CHAOS_MONKEY_RATE = "internal.chaos.monkey.rate";
+
+ int DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_DAYS = 0;
+ int DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_HOURS = 1;
+ int DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_MINUTES = 0;
+
+ String INTERNAL_CHAOS_MONKEY_PROBABILITY =
+ "internal.chaos.monkey.probability";
+ /**
+ * Probabilies are out of 10000 ; 100==1%
+ */
+
+ String INTERNAL_CHAOS_MONKEY_PROBABILITY_AM_FAILURE = INTERNAL_CHAOS_MONKEY_PROBABILITY +".amfailure";
+ int DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE = 10;
+ String INTERNAL_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE =
+ INTERNAL_CHAOS_MONKEY_PROBABILITY + ".containerfailure";
+ int DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE = 100;
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java b/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
index 006e280..320cc4f 100644
--- a/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
@@ -22,18 +22,7 @@ package org.apache.slider.api;
* Keys for entries in the <code>options</code> section
* of a cluster description.
*/
-public interface OptionKeys {
-
- /**
- * Home dir of the app: {@value}
- * If set, implies there is a home dir to use
- */
- String INTERNAL_APPLICATION_HOME = "internal.application.home";
-
- /**
- * Path to an image file containing the app: {@value}
- */
- String INTERNAL_APPLICATION_IMAGE_PATH = "internal.application.image.path";
+public interface OptionKeys extends InternalKeys {
/**
* Time in milliseconds to wait after forking any in-AM
@@ -44,72 +33,18 @@ public interface OptionKeys {
* is wasted starting containers on a cluster that isn't going to come
* up
*/
- String INTERNAL_CONTAINER_STARTUP_DELAY = "internal.container.startup.delay";
-
- /**
- * Time in milliseconds to wait after forking any in-AM
- * process before attempting to start up the containers: {@value}
- *
- * A shorter value brings the cluster up faster, but means that if the
- * in AM process fails (due to a bad configuration), then time
- * is wasted starting containers on a cluster that isn't going to come
- * up
- */
String APPLICATION_TYPE = "application.type";
- /**
- * Time in milliseconds to wait after forking any in-AM
- * process before attempting to start up the containers: {@value}
- *
- * A shorter value brings the cluster up faster, but means that if the
- * in AM process fails (due to a bad configuration), then time
- * is wasted starting containers on a cluster that isn't going to come
- * up
- */
- String APPLICATION_NAME = "application.name";
-
- /**
- * delay for container startup:{@value}
- */
- int DEFAULT_CONTAINER_STARTUP_DELAY = 5000;
- /**
- * Version of the app: {@value}
- */
- String KEYTAB_LOCATION = "internal.keytab.location";
+ String APPLICATION_NAME = "application.name";
/**
* Prefix for site.xml options: {@value}
*/
String SITE_XML_PREFIX = "site.";
- /**
- * internal temp directory: {@value}
- */
- String INTERNAL_AM_TMP_DIR = "internal.tmp.dir";
/**
- * where a snapshot of the original conf dir is: {@value}
- */
- String INTERNAL_SNAPSHOT_CONF_PATH = "internal.snapshot.conf.path";
-
- /**
- * where a snapshot of the original conf dir is: {@value}
- */
- String INTERNAL_GENERATED_CONF_PATH = "internal.generated.conf.path";
-
- /**
- * where a snapshot of the original conf dir is: {@value}
- */
- String INTERNAL_PROVIDER_NAME = "internal.provider.name";
-
-
- /**
- * where a snapshot of the original conf dir is: {@value}
- */
- String INTERNAL_DATA_DIR_PATH = "internal.data.dir.path";
-
- /**
* Zookeeper quorum host list: {@value}
*/
String ZOOKEEPER_QUORUM = "zookeeper.quorum";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 03f78be..1337570 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.OptionKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.SliderClusterProtocol;
@@ -793,16 +794,16 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
instanceDefinition.getAppConfOperations();
Path generatedConfDirPath =
createPathThatMustExist(internalOptions.getMandatoryOption(
- OptionKeys.INTERNAL_GENERATED_CONF_PATH));
+ InternalKeys.INTERNAL_GENERATED_CONF_PATH));
Path snapshotConfPath =
createPathThatMustExist(internalOptions.getMandatoryOption(
- OptionKeys.INTERNAL_SNAPSHOT_CONF_PATH));
+ InternalKeys.INTERNAL_SNAPSHOT_CONF_PATH));
// cluster Provider
AbstractClientProvider provider = createClientProvider(
internalOptions.getMandatoryOption(
- OptionKeys.INTERNAL_PROVIDER_NAME));
+ InternalKeys.INTERNAL_PROVIDER_NAME));
// make sure the conf dir is valid;
// now build up the image path
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
index 0a33c36..937b777 100644
--- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.OptionKeys;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.common.SliderXmlConfKeys;
@@ -139,7 +140,7 @@ public class InstanceBuilder {
instancePaths.dataPath.toUri());
- internalOps.set(OptionKeys.INTERNAL_PROVIDER_NAME, provider);
+ internalOps.set(InternalKeys.INTERNAL_PROVIDER_NAME, provider);
internalOps.set(OptionKeys.APPLICATION_NAME, clustername);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index cb7d27a..db9f494 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.OptionKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.RoleKeys;
@@ -282,9 +283,9 @@ public class ProviderUtils implements RoleKeys {
MapOperations globalOptions =
instanceDefinition.getInternalOperations().getGlobalOptions();
String applicationHome =
- globalOptions.get(OptionKeys.INTERNAL_APPLICATION_HOME);
+ globalOptions.get(InternalKeys.INTERNAL_APPLICATION_HOME);
String imagePath =
- globalOptions.get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ globalOptions.get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
return buildPathToHomeDir(imagePath, applicationHome, bindir, script);
}
@@ -350,8 +351,8 @@ public class ProviderUtils implements RoleKeys {
String script) throws FileNotFoundException {
String homedir = buildPathToHomeDir(
- internal.get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH),
- internal.get(OptionKeys.INTERNAL_APPLICATION_HOME),
+ internal.get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH),
+ internal.get(InternalKeys.INTERNAL_APPLICATION_HOME),
bindir,
script);
return buildScriptPath(bindir, script, homedir);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
index 6aeb801..c3870b8 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.RoleKeys;
import org.apache.slider.common.SliderKeys;
@@ -123,7 +123,7 @@ public class SliderAMClientProvider extends AbstractClientProvider implements
dataPath = instanceDefinition.getInternalOperations()
.getGlobalOptions()
.getMandatoryOption(
- OptionKeys.INTERNAL_DATA_DIR_PATH);
+ InternalKeys.INTERNAL_DATA_DIR_PATH);
Path path = new Path(dataPath);
sliderFileSystem.verifyDirectoryWriteAccess(path);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index ee0e59e..e6c7aba 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -56,7 +56,7 @@ import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.WebApps;
import org.apache.slider.api.ClusterDescription;
-import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.RoleKeys;
import org.apache.slider.api.SliderClusterProtocol;
@@ -529,10 +529,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
serviceConf.set(SliderAmIpFilter.WS_CONTEXT_ROOT, WS_CONTEXT_ROOT + "|" + WS_AGENT_CONTEXT_ROOT);
//get our provider
- MapOperations globalInternalOptions =
- instanceDefinition.getInternalOperations().getGlobalOptions();
+ MapOperations globalInternalOptions = getGlobalInternalOptions();
String providerType = globalInternalOptions.getMandatoryOption(
- OptionKeys.INTERNAL_PROVIDER_NAME);
+ InternalKeys.INTERNAL_PROVIDER_NAME);
log.info("Cluster provider type is {}", providerType);
SliderProviderFactory factory =
SliderProviderFactory.createSliderProviderFactory(
@@ -751,7 +750,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
String rolesTmpSubdir = appMasterContainerID.toString() + "/roles";
- String amTmpDir = globalInternalOptions.getMandatoryOption(OptionKeys.INTERNAL_AM_TMP_DIR);
+ String amTmpDir = globalInternalOptions.getMandatoryOption(InternalKeys.INTERNAL_AM_TMP_DIR);
Path tmpDirPath = new Path(amTmpDir);
Path launcherTmpDirPath = new Path(tmpDirPath, rolesTmpSubdir);
@@ -927,9 +926,18 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* @return the generated configuration dir
*/
public String getGeneratedConfDir() {
+ return getGlobalInternalOptions().get(
+ InternalKeys.INTERNAL_GENERATED_CONF_PATH);
+ }
+
+ /**
+ * Get the global internal options for the AM
+ * @return a map to access the internals
+ */
+ public MapOperations getGlobalInternalOptions() {
return getInstanceDefinition()
.getInternalOperations().
- getGlobalOptions().get(OptionKeys.INTERNAL_GENERATED_CONF_PATH);
+ getGlobalOptions();
}
/**
@@ -1179,7 +1187,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
/**
* Schedule the failure window
- * @param resources
+ * @param resources the resource tree
* @throws BadConfigException if the window is out of range
*/
private void scheduleFailureWindowResets(ConfTree resources) throws
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index a9b9e99..22de0d2 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -27,13 +27,10 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.Token;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterDescriptionKeys;
import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.OptionKeys;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.tools.SliderFileSystem;
@@ -218,7 +215,7 @@ public class TestAgentProviderService {
public void testRegistration() throws IOException {
ConfTree tree = new ConfTree();
- tree.global.put(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");
+ tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");
AgentProviderService aps = new AgentProviderService();
ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class);
@@ -585,7 +582,7 @@ public class TestAgentProviderService {
InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
ConfTree tree = new ConfTree();
- tree.global.put(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");
+ tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");
AgentProviderService aps = new AgentProviderService();
ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java
index db99360..7f99573 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.slider.api.InternalKeys;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.api.OptionKeys;
@@ -135,7 +136,7 @@ public class AccumuloClientProvider extends AbstractClientProvider implements
propagateClientFSBinding(sitexml);
setDatabasePath(sitexml,
- globalInstanceOptions.getMandatoryOption(OptionKeys.INTERNAL_DATA_DIR_PATH));
+ globalInstanceOptions.getMandatoryOption(InternalKeys.INTERNAL_DATA_DIR_PATH));
String quorum =
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
index c511efb..7dedd2c 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.slider.api.InternalKeys;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.OptionKeys;
@@ -161,7 +162,7 @@ public class AccumuloProviderService extends AbstractProviderService implements
//Add binaries
//now add the image if it was set
String imageURI = instanceDefinition.getInternalOperations()
- .get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ .get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
fileSystem.maybeAddImagePath(launcher.getLocalResources(), imageURI);
CommandLineBuilder commandLine = new CommandLineBuilder();
@@ -333,8 +334,8 @@ public class AccumuloProviderService extends AbstractProviderService implements
//callback to AM to trigger cluster review is set up to happen after
//the init/verify action has succeeded
int delay = internalOperations.getGlobalOptions().getOptionInt(
- OptionKeys.INTERNAL_CONTAINER_STARTUP_DELAY,
- OptionKeys.DEFAULT_CONTAINER_STARTUP_DELAY);
+ InternalKeys.INTERNAL_CONTAINER_STARTUP_DELAY,
+ InternalKeys.DEFAULT_INTERNAL_CONTAINER_STARTUP_DELAY);
ProviderCompletedCallable completedCallable =
new ProviderCompletedCallable(execInProgress, null);
Service notifier = new WorkflowCallbackService<>(
@@ -364,7 +365,7 @@ public class AccumuloProviderService extends AbstractProviderService implements
String dataDir = cd.getInternalOperations()
.getGlobalOptions()
.getMandatoryOption(
- OptionKeys.INTERNAL_DATA_DIR_PATH);
+ InternalKeys.INTERNAL_DATA_DIR_PATH);
Path accumuloInited = new Path(dataDir, INSTANCE_ID);
FileSystem fs2 = FileSystem.get(accumuloInited.toUri(), getConf());
return fs2.exists(accumuloInited);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
index cd104a2..76bf73a 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
@@ -21,6 +21,7 @@ package org.apache.slider.providers.hbase;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.slider.api.InternalKeys;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.api.OptionKeys;
@@ -122,7 +123,7 @@ public class HBaseClientProvider extends AbstractClientProvider implements
sitexml.put(KEY_HBASE_ROOTDIR,
globalInstanceOptions.getMandatoryOption(
- OptionKeys.INTERNAL_DATA_DIR_PATH)
+ InternalKeys.INTERNAL_DATA_DIR_PATH)
);
providerUtils.propagateOption(globalAppOptions, OptionKeys.ZOOKEEPER_PATH,
sitexml, KEY_ZNODE_PARENT);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index f72967f..feffb08 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -22,9 +22,9 @@ package org.apache.slider.providers.hbase;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.slider.api.InternalKeys;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.api.ClusterDescription;
-import org.apache.slider.api.OptionKeys;
import org.apache.slider.api.RoleKeys;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.core.conf.AggregateConf;
@@ -152,7 +152,7 @@ public class HBaseProviderService extends AbstractProviderService implements
//Add binaries
//now add the image if it was set
String imageURI = instanceDefinition.getInternalOperations()
- .get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ .get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
coreFS.maybeAddImagePath(launcher.getLocalResources(), imageURI);
CommandLineBuilder cli = new CommandLineBuilder();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1f6490d5/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeReconfigureThawLiveRegionService.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeReconfigureThawLiveRegionService.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeReconfigureThawLiveRegionService.groovy
index d50517c..f6748e0 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeReconfigureThawLiveRegionService.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/freezethaw/TestFreezeReconfigureThawLiveRegionService.groovy
@@ -25,9 +25,9 @@ import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.Path
import org.apache.hadoop.hbase.ClusterStatus
import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.api.InternalKeys
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.api.ClusterDescription
-import org.apache.slider.api.OptionKeys
import org.apache.slider.core.build.InstanceIO
import org.apache.slider.providers.hbase.HBaseKeys
import org.apache.slider.common.tools.ConfigHelper
@@ -93,7 +93,7 @@ class TestFreezeReconfigureThawLiveRegionService
clusterDir)
def snapshotPath = instanceDefinition.internalOperations.get(
- OptionKeys.INTERNAL_SNAPSHOT_CONF_PATH)
+ InternalKeys.INTERNAL_SNAPSHOT_CONF_PATH)
assert snapshotPath != null
Path confdir = new Path(snapshotPath);
[18/50] [abbrv] git commit: SLIDER-322 hbase and accumulo -> java6
language
Posted by st...@apache.org.
SLIDER-322 hbase and accumulo -> java6 language
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/280afd88
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/280afd88
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/280afd88
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 280afd88960457864e214da9254242c57a023fc6
Parents: 397585a
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 14 17:47:01 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 14 17:47:01 2014 +0100
----------------------------------------------------------------------
.../accumulo/AccumuloProviderService.java | 19 ++++++++++--
.../providers/hbase/HBaseClientProvider.java | 6 ++--
.../providers/hbase/HBaseProviderService.java | 31 +++++++++++++++++++-
3 files changed, 49 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/280afd88/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
index 7dedd2c..b8f4c00 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
@@ -170,6 +170,8 @@ public class AccumuloProviderService extends AbstractProviderService implements
String heap = "-Xmx" + appComponent.getOption(RoleKeys.JVM_HEAP, DEFAULT_JVM_HEAP);
String opt = "ACCUMULO_OTHER_OPTS";
if (SliderUtils.isSet(heap)) {
+/* JDK7
+
switch (role) {
case AccumuloKeys.ROLE_MASTER:
opt = "ACCUMULO_MASTER_OPTS";
@@ -184,6 +186,16 @@ public class AccumuloProviderService extends AbstractProviderService implements
opt = "ACCUMULO_GC_OPTS";
break;
}
+*/
+ if (AccumuloKeys.ROLE_MASTER.equals(role)) {
+ opt = "ACCUMULO_MASTER_OPTS";
+ } else if (AccumuloKeys.ROLE_TABLET.equals(role)) {
+ opt = "ACCUMULO_TSERVER_OPTS";
+ } else if (AccumuloKeys.ROLE_MONITOR.equals(role)) {
+ opt = "ACCUMULO_MONITOR_OPTS";
+ } else if (AccumuloKeys.ROLE_GARBAGE_COLLECTOR.equals(role)) {
+ opt = "ACCUMULO_GC_OPTS";
+ }
launcher.setEnv(opt, heap);
}
@@ -239,7 +251,7 @@ public class AccumuloProviderService extends AbstractProviderService implements
String accumuloScript = AccumuloClientProvider.buildScriptBinPath(instance);
- List<String> launchSequence = new ArrayList<>(8);
+ List<String> launchSequence = new ArrayList<String>(8);
launchSequence.add(0, accumuloScript);
Collections.addAll(launchSequence, commands);
return launchSequence;
@@ -338,7 +350,8 @@ public class AccumuloProviderService extends AbstractProviderService implements
InternalKeys.DEFAULT_INTERNAL_CONTAINER_STARTUP_DELAY);
ProviderCompletedCallable completedCallable =
new ProviderCompletedCallable(execInProgress, null);
- Service notifier = new WorkflowCallbackService<>(
+ // JDK7
+ Service notifier = new WorkflowCallbackService(
"accumulo notifier",
completedCallable,
delay,
@@ -389,7 +402,7 @@ public class AccumuloProviderService extends AbstractProviderService implements
@Override
public Map<String, String> buildProviderStatus() {
- Map<String,String> status = new HashMap<>();
+ Map<String,String> status = new HashMap<String, String>();
return status;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/280afd88/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
index 76bf73a..9ad872f 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
@@ -108,7 +108,7 @@ public class HBaseClientProvider extends AbstractClientProvider implements
MapOperations worker = appconf.getMandatoryComponent(HBaseKeys.ROLE_WORKER);
- Map<String, String> sitexml = new HashMap<>();
+ Map<String, String> sitexml = new HashMap<String, String>();
//map all cluster-wide site. options
providerUtils.propagateSiteOptions(globalAppOptions, sitexml);
@@ -210,7 +210,7 @@ public class HBaseClientProvider extends AbstractClientProvider implements
}
}
- private static Set<String> knownRoleNames = new HashSet<>();
+ private static Set<String> knownRoleNames = new HashSet<String>();
static {
List<ProviderRole> roles = HBaseRoles.getRoles();
knownRoleNames.add(SliderKeys.COMPONENT_AM);
@@ -263,7 +263,7 @@ public class HBaseClientProvider extends AbstractClientProvider implements
// add any and all dependency files
Map<String, LocalResource> providerResources =
- new HashMap<>();
+ new HashMap<String, LocalResource>();
ProviderUtils.addProviderJar(providerResources,
this,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/280afd88/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index feffb08..1fb392c 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -180,6 +180,8 @@ public class HBaseProviderService extends AbstractProviderService implements
String roleCommand;
String logfile;
//now look at the role
+
+/* JDK7
switch (role) {
case ROLE_WORKER:
//role is region server
@@ -210,6 +212,33 @@ public class HBaseProviderService extends AbstractProviderService implements
throw new SliderInternalStateException("Cannot start role %s", role);
}
+*/
+ if (ROLE_WORKER.equals(role)) {
+ //role is region server
+ roleCommand = REGION_SERVER;
+ logfile = "/region-server.txt";
+
+ } else if (ROLE_MASTER.equals(role)) {
+ roleCommand = MASTER;
+ logfile = "/master.txt";
+
+ } else if (ROLE_REST_GATEWAY.equals(role)) {
+ roleCommand = REST_GATEWAY;
+ logfile = "/rest-gateway.txt";
+
+ } else if (ROLE_THRIFT_GATEWAY.equals(role)) {
+ roleCommand = THRIFT_GATEWAY;
+ logfile = "/thrift-gateway.txt";
+
+ } else if (ROLE_THRIFT2_GATEWAY.equals(role)) {
+ roleCommand = THRIFT2_GATEWAY;
+ logfile = "/thrift2-gateway.txt";
+ }
+
+ else {
+ throw new SliderInternalStateException("Cannot start role %s", role);
+ }
+
cli.add(roleCommand);
cli.add(ACTION_START);
//log details
@@ -319,7 +348,7 @@ public class HBaseProviderService extends AbstractProviderService implements
* @return the provider status - map of entries to add to the info section
*/
public Map<String, String> buildProviderStatus() {
- Map<String, String> stats = new HashMap<>();
+ Map<String, String> stats = new HashMap<String, String>();
return stats;
}
[06/50] [abbrv] git commit: SLIDER-202: tweak mock tests to
automatically get name from test method unless overridden.
Posted by st...@apache.org.
SLIDER-202: tweak mock tests to automatically get name from test method unless overridden.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8ed52a90
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8ed52a90
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8ed52a90
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 8ed52a90ef413931bf48e23a6d4565694a95bdb2
Parents: 1f6490d
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 13 15:04:36 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 13 15:04:36 2014 +0100
----------------------------------------------------------------------
.../model/appstate/TestMockAppStateContainerFailure.groovy | 2 +-
.../model/appstate/TestMockAppStateDynamicRoles.groovy | 2 +-
.../model/appstate/TestMockAppStateFlexDynamicRoles.groovy | 2 +-
.../appmaster/model/appstate/TestMockAppStateFlexing.groovy | 2 +-
.../model/appstate/TestMockAppStateRMOperations.groovy | 2 +-
.../model/appstate/TestMockAppStateRebuildOnAMRestart.groovy | 2 +-
.../model/appstate/TestMockAppStateRolePlacement.groovy | 2 +-
.../model/appstate/TestMockAppStateRoleRelease.groovy | 2 +-
.../appstate/TestMockContainerResourceAllocations.groovy | 2 +-
.../server/appmaster/model/mock/BaseMockAppStateTest.groovy | 8 +++++++-
10 files changed, 16 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ed52a90/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
index c0f41f4..068b876 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
@@ -43,7 +43,7 @@ class TestMockAppStateContainerFailure extends BaseMockAppStateTest
@Override
String getTestName() {
- return "TestAppStateContainerFailure"
+ return "TestMockAppStateContainerFailure"
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ed52a90/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
index 1b01f76..136e1ea 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
@@ -42,7 +42,7 @@ class TestMockAppStateDynamicRoles extends BaseMockAppStateTest
@Override
String getTestName() {
- return "TestAppStateDynamicRoles"
+ return "TestMockAppStateDynamicRoles"
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ed52a90/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
index aa330f8..5c9dce9 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
@@ -44,7 +44,7 @@ class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
@Override
String getTestName() {
- return "TestAppStateDynamicRoles"
+ return "TestMockAppStateFlexDynamicRoles"
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ed52a90/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
index 80937dc..a7bf068 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
@@ -33,7 +33,7 @@ class TestMockAppStateFlexing extends BaseMockAppStateTest implements MockRoles
@Override
String getTestName() {
- return "TestMockFlexing"
+ return "TestMockAppStateFlexing"
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ed52a90/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
index 6a5df5c..f8e852e 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
@@ -40,7 +40,7 @@ class TestMockAppStateRMOperations extends BaseMockAppStateTest implements MockR
@Override
String getTestName() {
- return "TestMockRMOperations"
+ return "TestMockAppStateRMOperations"
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ed52a90/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
index c142c77..c2783f3 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
@@ -40,7 +40,7 @@ class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest
@Override
String getTestName() {
- return "TestAppStateRebuildOnAMRestart"
+ return "TestMockAppStateRebuildOnAMRestart"
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ed52a90/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
index e7ddf79..17ebc31 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
@@ -43,7 +43,7 @@ class TestMockAppStateRolePlacement extends BaseMockAppStateTest
@Override
String getTestName() {
- return "TestAppStateRolePlacement"
+ return "TestMockAppStateRolePlacement"
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ed52a90/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.groovy
index de4d493..addfaa5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.groovy
@@ -38,7 +38,7 @@ class TestMockAppStateRoleRelease extends BaseMockAppStateTest
@Override
String getTestName() {
- return "TestAppStateRolePlacement"
+ return "TestMockAppStateRoleRelease"
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ed52a90/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
index 0bf9a90..73d40ee 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
@@ -39,7 +39,7 @@ class TestMockContainerResourceAllocations extends BaseMockAppStateTest {
@Override
String getTestName() {
- "TestContainerResourceAllocations"
+ "TestMockContainerResourceAllocations"
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ed52a90/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
index 289b7c5..f96a238 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -101,7 +101,13 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
factory.newInstanceDefinition(0, 0, 0)
}
- abstract String getTestName();
+ /**
+ * Get the test name ... defaults to method name
+ * @return
+ */
+ String getTestName() {
+ methodName.methodName;
+ }
public RoleStatus getRole0Status() {
return appState.lookupRoleStatus(ROLE0)
[38/50] [abbrv] git commit: SLIDER-324. Exports should be attributed
to individual master components rather than only one master component
Posted by st...@apache.org.
SLIDER-324. Exports should be attributed to individual master components rather than only one master component
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bc9561d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bc9561d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bc9561d5
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: bc9561d59d3c0a68a6abaa47e8a91b87a36a1287
Parents: 599e972
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Fri Aug 15 13:32:12 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Fri Aug 15 13:32:12 2014 -0700
----------------------------------------------------------------------
app-packages/storm/metainfo.xml | 28 ++++----
.../agent/application/metadata/Component.java | 21 ++++--
.../application/metadata/ComponentExport.java | 54 ++++++++++++++
.../application/metadata/MetainfoParser.java | 9 +--
.../agent/TestAgentProviderService.java | 23 +++---
.../agent/application/metadata/metainfo.xml | 74 ++++++++++++++++++--
6 files changed, 171 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc9561d5/app-packages/storm/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index e94bc62..d9004ad 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -29,21 +29,25 @@
<name>QuickLinks</name>
<exports>
<export>
- <name>org.apache.slider.jmx</name>
+ <name>app.jmx</name>
<value>http://${STORM_REST_API_HOST}:${site.global.rest_api_port}/api/cluster/summary</value>
</export>
<export>
- <name>org.apache.slider.monitor</name>
+ <name>app.monitor</name>
<value>http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}</value>
</export>
<export>
- <name>org.apache.slider.metrics</name>
+ <name>app.metrics</name>
<value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value>
</export>
<export>
- <name>org.apache.slider.ganglia</name>
+ <name>ganglia.ui</name>
<value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value>
</export>
+ <export>
+ <name>nimbus.url</name>
+ <value>http://${NIMBUS_HOST}:${site.storm-site.nimbus.thrift.port}</value>
+ </export>
</exports>
</exportGroup>
</exportGroups>
@@ -78,6 +82,7 @@
<name>NIMBUS</name>
<category>MASTER</category>
<autoStartOnFailure>true</autoStartOnFailure>
+ <appExports>QuickLinks-nimbus.url,QuickLinks-ganglia.ui,QuickLinks-app.metrics</appExports>
<commandScript>
<script>scripts/nimbus.py</script>
<scriptType>PYTHON</scriptType>
@@ -89,6 +94,7 @@
<name>STORM_REST_API</name>
<category>MASTER</category>
<autoStartOnFailure>true</autoStartOnFailure>
+ <appExports>QuickLinks-app.jmx</appExports>
<commandScript>
<script>scripts/rest_api.py</script>
<scriptType>PYTHON</scriptType>
@@ -100,12 +106,12 @@
<name>SUPERVISOR</name>
<category>SLAVE</category>
<autoStartOnFailure>true</autoStartOnFailure>
- <exports>
- <export>
+ <componentExports>
+ <componentExport>
<name>log_viewer_port</name>
<value>${THIS_HOST}:${site.storm-site.logviewer.port}</value>
- </export>
- </exports>
+ </componentExport>
+ </componentExports>
<commandScript>
<script>scripts/supervisor.py</script>
<scriptType>PYTHON</scriptType>
@@ -117,6 +123,7 @@
<name>STORM_UI_SERVER</name>
<category>MASTER</category>
<publishConfig>true</publishConfig>
+ <appExports>QuickLinks-app.monitor</appExports>
<autoStartOnFailure>true</autoStartOnFailure>
<commandScript>
<script>scripts/ui_server.py</script>
@@ -148,10 +155,5 @@
</packages>
</osSpecific>
</osSpecifics>
-
- <configuration-dependencies>
- <config-type>storm-site</config-type>
- <config-type>global</config-type>
- </configuration-dependencies>
</application>
</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc9561d5/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
index acf6ecb..03c64d4 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
@@ -29,12 +29,13 @@ public class Component {
String minInstanceCount;
String maxInstanceCount;
String autoStartOnFailure;
+ String appExports;
CommandScript commandScript;
- List<Export> exports;
+ List<ComponentExport> componentExports;
public Component() {
publishConfig = Boolean.FALSE.toString();
- exports = new ArrayList<Export>();
+ componentExports = new ArrayList<ComponentExport>();
}
public String getName() {
@@ -69,6 +70,14 @@ public class Component {
this.autoStartOnFailure = autoStartOnFailure;
}
+ public String getAppExports() {
+ return appExports;
+ }
+
+ public void setAppExports(String appExports) {
+ this.appExports = appExports;
+ }
+
public String getMinInstanceCount() {
return minInstanceCount;
}
@@ -93,12 +102,12 @@ public class Component {
this.commandScript = commandScript;
}
- public void addExport(Export export) {
- exports.add(export);
+ public void addComponentExport(ComponentExport export) {
+ componentExports.add(export);
}
- public List<Export> getExports() {
- return exports;
+ public List<ComponentExport> getComponentExports() {
+ return componentExports;
}
public Boolean getRequiresAutoRestart() {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc9561d5/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java
new file mode 100644
index 0000000..a18854c
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+/**
+ *
+ */
+public class ComponentExport {
+ String name;
+ String value;
+
+ public ComponentExport() {
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public void setValue(String value) {
+ this.value = value;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("{");
+ sb.append(",\n\"name\": ").append(name);
+ sb.append(",\n\"value\": ").append(value);
+ sb.append('}');
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc9561d5/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
index ada5b83..4b91ff7 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
@@ -60,10 +60,11 @@ public class MetainfoParser {
digester.addBeanPropertySetter("*/component/minInstanceCount");
digester.addBeanPropertySetter("*/component/maxInstanceCount");
digester.addBeanPropertySetter("*/component/autoStartOnFailure");
- digester.addObjectCreate("*/component/export", Export.class);
- digester.addBeanPropertySetter("*/component/export/name");
- digester.addBeanPropertySetter("*/component/export/value");
- digester.addSetNext("*/component/export", "addExport");
+ digester.addBeanPropertySetter("*/component/appExports");
+ digester.addObjectCreate("*/component/componentExport", ComponentExport.class);
+ digester.addBeanPropertySetter("*/component/componentExport/name");
+ digester.addBeanPropertySetter("*/component/componentExport/value");
+ digester.addSetNext("*/component/componentExport", "addComponentExport");
digester.addSetNext("*/component", "addComponent");
digester.addObjectCreate("*/commandScript", CommandScript.class);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc9561d5/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 20c16ff..1ea11a6 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -43,6 +43,7 @@ import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.providers.agent.application.metadata.Application;
import org.apache.slider.providers.agent.application.metadata.CommandOrder;
import org.apache.slider.providers.agent.application.metadata.Component;
+import org.apache.slider.providers.agent.application.metadata.ComponentExport;
import org.apache.slider.providers.agent.application.metadata.Export;
import org.apache.slider.providers.agent.application.metadata.ExportGroup;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
@@ -138,6 +139,7 @@ public class TestAgentProviderService {
+ " <name>HBASE_MASTER</name>\n"
+ " <category>MASTER</category>\n"
+ " <autoStartOnFailure>true</autoStartOnFailure>\n"
+ + " <appExports>QuickLinks-JMX_Endpoint,QuickLinks-Master_Status</appExports>\n"
+ " <minInstanceCount>1</minInstanceCount>\n"
+ " <maxInstanceCount>2</maxInstanceCount>\n"
+ " <commandScript>\n"
@@ -155,16 +157,16 @@ public class TestAgentProviderService {
+ " <script>scripts/hbase_regionserver.py</script>\n"
+ " <scriptType>PYTHON</scriptType>\n"
+ " </commandScript>\n"
- + " <exports>\n"
- + " <export>\n"
+ + " <componentExports>\n"
+ + " <componentExport>\n"
+ " <name>PropertyA</name>\n"
+ " <value>${THIS_HOST}:${site.global.listen_port}</value>\n"
- + " </export>\n"
- + " <export>\n"
+ + " </componentExport>\n"
+ + " <componentExport>\n"
+ " <name>PropertyB</name>\n"
+ " <value>AConstant</value>\n"
- + " </export>\n"
- + " </exports>\n"
+ + " </componentExport>\n"
+ + " </componentExport>\n"
+ " </component>\n"
+ " </components>\n"
+ " <osSpecifics>\n"
@@ -496,7 +498,8 @@ public class TestAgentProviderService {
Assert.assertEquals(component.getMaxInstanceCount(), "2");
Assert.assertEquals(component.getCommandScript().getScript(), "scripts/hbase_master.py");
Assert.assertEquals(component.getCategory(), "MASTER");
- Assert.assertEquals(component.getExports().size(), 0);
+ Assert.assertEquals(component.getComponentExports().size(), 0);
+ Assert.assertEquals(component.getAppExports(), "QuickLinks-JMX_Endpoint,QuickLinks-Master_Status");
found++;
}
if (component.getName().equals("HBASE_REGIONSERVER")) {
@@ -506,9 +509,9 @@ public class TestAgentProviderService {
Assert.assertNull(component.getMaxInstanceCount());
Assert.assertEquals(component.getCommandScript().getScript(), "scripts/hbase_regionserver.py");
Assert.assertEquals(component.getCategory(), "SLAVE");
- Assert.assertEquals(component.getExports().size(), 2);
- List<Export> es = component.getExports();
- Export e = es.get(0);
+ Assert.assertEquals(component.getComponentExports().size(), 2);
+ List<ComponentExport> es = component.getComponentExports();
+ ComponentExport e = es.get(0);
Assert.assertEquals(e.getName(), "PropertyA");
Assert.assertEquals(e.getValue(), "${THIS_HOST}:${site.global.listen_port}");
e = es.get(1);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bc9561d5/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml b/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml
index 3d24f96..d9004ad 100644
--- a/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml
+++ b/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml
@@ -22,11 +22,67 @@
<name>STORM</name>
<comment>Apache Hadoop Stream processing framework</comment>
<version>0.9.1.2.1</version>
+ <exportedConfigs>storm-site</exportedConfigs>
+
+ <exportGroups>
+ <exportGroup>
+ <name>QuickLinks</name>
+ <exports>
+ <export>
+ <name>app.jmx</name>
+ <value>http://${STORM_REST_API_HOST}:${site.global.rest_api_port}/api/cluster/summary</value>
+ </export>
+ <export>
+ <name>app.monitor</name>
+ <value>http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}</value>
+ </export>
+ <export>
+ <name>app.metrics</name>
+ <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value>
+ </export>
+ <export>
+ <name>ganglia.ui</name>
+ <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value>
+ </export>
+ <export>
+ <name>nimbus.url</name>
+ <value>http://${NIMBUS_HOST}:${site.storm-site.nimbus.thrift.port}</value>
+ </export>
+ </exports>
+ </exportGroup>
+ </exportGroups>
+
+ <commandOrders>
+ <commandOrder>
+ <command>NIMBUS-START</command>
+ <requires>SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED,STORM_REST_API-INSTALLED
+ </requires>
+ </commandOrder>
+ <commandOrder>
+ <command>SUPERVISOR-START</command>
+ <requires>NIMBUS-STARTED</requires>
+ </commandOrder>
+ <commandOrder>
+ <command>DRPC_SERVER-START</command>
+ <requires>NIMBUS-STARTED</requires>
+ </commandOrder>
+ <commandOrder>
+ <command>STORM_REST_API-START</command>
+ <requires>NIMBUS-STARTED,DRPC_SERVER-STARTED,STORM_UI_SERVER-STARTED</requires>
+ </commandOrder>
+ <commandOrder>
+ <command>STORM_UI_SERVER-START</command>
+ <requires>NIMBUS-STARTED</requires>
+ </commandOrder>
+ </commandOrders>
+
<components>
<component>
<name>NIMBUS</name>
<category>MASTER</category>
+ <autoStartOnFailure>true</autoStartOnFailure>
+ <appExports>QuickLinks-nimbus.url,QuickLinks-ganglia.ui,QuickLinks-app.metrics</appExports>
<commandScript>
<script>scripts/nimbus.py</script>
<scriptType>PYTHON</scriptType>
@@ -37,6 +93,8 @@
<component>
<name>STORM_REST_API</name>
<category>MASTER</category>
+ <autoStartOnFailure>true</autoStartOnFailure>
+ <appExports>QuickLinks-app.jmx</appExports>
<commandScript>
<script>scripts/rest_api.py</script>
<scriptType>PYTHON</scriptType>
@@ -47,6 +105,13 @@
<component>
<name>SUPERVISOR</name>
<category>SLAVE</category>
+ <autoStartOnFailure>true</autoStartOnFailure>
+ <componentExports>
+ <componentExport>
+ <name>log_viewer_port</name>
+ <value>${THIS_HOST}:${site.storm-site.logviewer.port}</value>
+ </componentExport>
+ </componentExports>
<commandScript>
<script>scripts/supervisor.py</script>
<scriptType>PYTHON</scriptType>
@@ -57,6 +122,9 @@
<component>
<name>STORM_UI_SERVER</name>
<category>MASTER</category>
+ <publishConfig>true</publishConfig>
+ <appExports>QuickLinks-app.monitor</appExports>
+ <autoStartOnFailure>true</autoStartOnFailure>
<commandScript>
<script>scripts/ui_server.py</script>
<scriptType>PYTHON</scriptType>
@@ -67,6 +135,7 @@
<component>
<name>DRPC_SERVER</name>
<category>MASTER</category>
+ <autoStartOnFailure>true</autoStartOnFailure>
<commandScript>
<script>scripts/drpc_server.py</script>
<scriptType>PYTHON</scriptType>
@@ -86,10 +155,5 @@
</packages>
</osSpecific>
</osSpecifics>
-
- <configuration-dependencies>
- <config-type>storm-site</config-type>
- <config-type>global</config-type>
- </configuration-dependencies>
</application>
</metainfo>
[30/50] [abbrv] git commit: Merge branch
'feature/SLIDER-325_codahale_metrics_not_being_uploaded_to_AM_classpath' into
develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-325_codahale_metrics_not_being_uploaded_to_AM_classpath' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4ea66fad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4ea66fad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4ea66fad
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 4ea66faddb24564d7efc65367d6834db18b4d1fa
Parents: b8c2762 388805a
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 15 13:47:02 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 15 13:47:02 2014 +0100
----------------------------------------------------------------------
.../slideram/SliderAMClientProvider.java | 28 ++++++++++----------
1 file changed, 14 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
[45/50] [abbrv] git commit: SLIDER-302: add -live and --history flags
to the list action
Posted by st...@apache.org.
SLIDER-302: add -live and --history flags to the list action
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5d59454c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5d59454c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5d59454c
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 5d59454cf726ea8d626c9906220a43379c442a42
Parents: 624cafe
Author: Steve Loughran <st...@apache.org>
Authored: Mon Aug 11 11:06:52 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 19 10:53:45 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 50 +++++++++++++++-----
.../slider/common/params/ActionListArgs.java | 9 ++++
.../apache/slider/common/params/Arguments.java | 1 +
.../apache/slider/common/tools/SliderUtils.java | 49 +++++++++++++++++++
4 files changed, 98 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d59454c/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 93f6207..fb06454 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -51,6 +51,7 @@ import org.apache.slider.common.params.ActionFlexArgs;
import org.apache.slider.common.params.ActionFreezeArgs;
import org.apache.slider.common.params.ActionGetConfArgs;
import org.apache.slider.common.params.ActionKillContainerArgs;
+import org.apache.slider.common.params.ActionListArgs;
import org.apache.slider.common.params.ActionRegistryArgs;
import org.apache.slider.common.params.ActionStatusArgs;
import org.apache.slider.common.params.ActionThawArgs;
@@ -259,7 +260,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
serviceArgs.getActionAMSuicideArgs());
break;
case ACTION_LIST:
- exitCode = actionList(clusterName);
+ exitCode = actionList(clusterName, serviceArgs.getActionListArgs());
break;
case ACTION_REGISTRY:
exitCode = actionRegistry(
@@ -1400,30 +1401,48 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
/**
* Implement the list action: list all nodes
+ *
+ * live: List out only live instances
+ * history: List out only history instances
+ *
+ * If arguments are not given then list out both finished and
+ * running instances
+ *
+ * @param clustername List out specific cluster
+ * @param args Action list arguments
* @return exit code of 0 if a list was created
*/
@VisibleForTesting
- public int actionList(String clustername) throws IOException, YarnException {
+ public int actionList(String clustername, ActionListArgs args)
+ throws IOException, YarnException {
verifyBindingsDefined();
String user = UserGroupInformation.getCurrentUser().getUserName();
List<ApplicationReport> instances = listSliderInstances(user);
-
+ SliderUtils.sortApplicationReport(instances);
if (isUnset(clustername)) {
log.info("Instances for {}: {}",
- (user != null ? user : "all users"),
- instances.size());
+ (user != null ? user : "all users"),
+ instances.size());
for (ApplicationReport report : instances) {
- logAppReport(report);
+ logAppReport(report, args.live, args.history);
}
return EXIT_SUCCESS;
} else {
SliderUtils.validateClusterName(clustername);
log.debug("Listing cluster named {}", clustername);
- ApplicationReport report =
- findClusterInInstanceList(instances, clustername);
+ if (args.history) {
+ for (ApplicationReport report : instances) {
+ if (report.getName().equals(clustername)) {
+ logAppReport(report, args.live, args.history);
+ }
+ }
+ return EXIT_SUCCESS;
+ }
+ ApplicationReport report = findClusterInInstanceList(instances,
+ clustername);
if (report != null) {
- logAppReport(report);
+ logAppReport(report, true, false);
return EXIT_SUCCESS;
} else {
throw unknownClusterException(clustername);
@@ -1435,8 +1454,17 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* Log the application report at INFO
* @param report report to log
*/
- public void logAppReport(ApplicationReport report) {
- log.info(SliderUtils.appReportToString(report, "\n"));
+ public void logAppReport(ApplicationReport report, boolean live, boolean history) {
+ boolean active = report.getYarnApplicationState() == YarnApplicationState.RUNNING
+ || report.getYarnApplicationState() ==
+ YarnApplicationState.ACCEPTED;
+ if (active && live && !history) {
+ log.info(SliderUtils.appReportToString(report, "\n"));
+ } else if (!active && history && !live) {
+ log.info(SliderUtils.appReportToString(report, "\n"));
+ } else {
+ log.info(SliderUtils.appReportToString(report, "\n"));
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d59454c/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
index 8f42b69..586f0a1 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
@@ -18,6 +18,7 @@
package org.apache.slider.common.params;
+import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = {SliderActions.ACTION_LIST},
@@ -29,6 +30,14 @@ public class ActionListArgs extends AbstractActionArgs {
return SliderActions.ACTION_LIST;
}
+ @Parameter(names = {ARG_LIVE},
+ description = "List only live application instances")
+ public boolean live;
+
+ @Parameter(names = {ARG_HISTORY},
+ description = "List only historical application instances")
+ public boolean history;
+
/**
* Get the min #of params expected
* @return the min number of params in the {@link #parameters} field
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d59454c/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index 764ffca..a16bf98 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -46,6 +46,7 @@ public interface Arguments {
String ARG_GETCONF = "--getconf";
String ARG_GETFILES = "--getfiles";
String ARG_HELP = "--help";
+ String ARG_HISTORY = "--history";
String ARG_ID = "--id";
String ARG_IMAGE = "--image";
String ARG_INTERNAL = "--internal";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5d59454c/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 188b7d9..6aff38a 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -75,6 +75,8 @@ import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.Date;
import java.util.Enumeration;
import java.util.HashMap;
@@ -566,6 +568,53 @@ public final class SliderUtils {
}
/**
+ * Sorts the given list of application reports
+ * Finished instances are ordered by finished time and running/accepted instances are
+ * ordered by start time
+ * Finally Instance are order by finished instances and running instances
+ *
+ * @param instances
+ */
+ public static void sortApplicationReport(List<ApplicationReport> instances) {
+ if (instances.size() <= 1) {
+ return;
+ }
+ List<ApplicationReport> nonLiveInstance = new ArrayList<>(instances.size());
+ List<ApplicationReport> liveInstance = new ArrayList<>(instances.size());
+
+ for (ApplicationReport report : instances) {
+ if (report.getYarnApplicationState() == YarnApplicationState.RUNNING
+ || report.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
+ liveInstance.add(report);
+ } else {
+ nonLiveInstance.add(report);
+ }
+ }
+
+ if (liveInstance.size() > 1) {
+ Comparator<ApplicationReport> liveInstanceComparator = new Comparator<ApplicationReport>() {
+ @Override
+ public int compare(ApplicationReport r1, ApplicationReport r2) {
+ return Long.compare(r1.getStartTime(), r2.getStartTime());
+ }
+ };
+ Collections.sort(liveInstance, liveInstanceComparator);
+ }
+ if (nonLiveInstance.size() > 1) {
+ Comparator<ApplicationReport> nonLiveInstanceComparator = new Comparator<ApplicationReport>() {
+ @Override
+ public int compare(ApplicationReport r1, ApplicationReport r2) {
+ return Long.compare(r1.getFinishTime(), r2.getFinishTime());
+ }
+ };
+ Collections.sort(nonLiveInstance, nonLiveInstanceComparator);
+ }
+ instances.clear();
+ instances.addAll(nonLiveInstance);
+ instances.addAll(liveInstance);
+ }
+
+ /**
* Merge in one map to another -all entries in the second map are
* merged into the first -overwriting any duplicate keys.
* @param first first map -the updated one.
[21/50] [abbrv] git commit: SLIDER-181. Slider support for Storm
application package SLIDER-262. Slider agent should provide process
supervision such as auto-restart SLIDER-123. Status check for Storm on YARN
needs to be robust SLIDER-301. Add support fo
Posted by st...@apache.org.
SLIDER-181. Slider support for Storm application package
SLIDER-262. Slider agent should provide process supervision such as auto-restart
SLIDER-123. Status check for Storm on YARN needs to be robust
SLIDER-301. Add support for default status check through pid file
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/004fc8df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/004fc8df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/004fc8df
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 004fc8dfdfde58c04297d4001e861a0f8e93d69e
Parents: 966c8c8
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Aug 12 14:37:12 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Aug 14 17:54:57 2014 -0700
----------------------------------------------------------------------
app-packages/memcached-win/appConfig.json | 2 +-
.../memcached-win/package/scripts/memcached.py | 5 +++-
.../memcached-win/package/scripts/params.py | 1 +
app-packages/memcached/appConfig.json | 1 +
.../memcached/package/scripts/memcached.py | 5 +++-
.../memcached/package/scripts/params.py | 2 ++
app-packages/storm/appConfig.json | 6 ++--
app-packages/storm/metainfo.xml | 12 ++++++++
app-packages/storm/package/scripts/service.py | 24 ++++++++-------
.../storm/package/scripts/status_params.py | 1 +
.../src/main/python/agent/Controller.py | 3 +-
.../python/agent/CustomServiceOrchestrator.py | 31 +++++++++++++++++++-
.../core/providers/system.py | 3 +-
.../core/providers/windows/system.py | 11 +++++--
.../core/resources/system.py | 4 +++
.../python/resource_management/core/shell.py | 14 +++++----
.../agent/TestCustomServiceOrchestrator.py | 8 +++++
.../java/org/apache/slider/api/OptionKeys.java | 1 -
.../slider/common/tools/CoreFileSystem.java | 3 --
.../providers/agent/AgentClientProvider.java | 6 ----
.../providers/agent/AgentProviderService.java | 24 +++++++++------
.../agent/application/metadata/Component.java | 13 ++++++++
.../application/metadata/MetainfoParser.java | 1 +
.../agent/TestAgentProviderService.java | 6 ++++
24 files changed, 140 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/app-packages/memcached-win/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/appConfig.json b/app-packages/memcached-win/appConfig.json
index 63835ec..b76ecde 100644
--- a/app-packages/memcached-win/appConfig.json
+++ b/app-packages/memcached-win/appConfig.json
@@ -8,7 +8,7 @@
"site.global.app_user": "hadoop",
"site.global.app_root": "${AGENT_WORK_ROOT}\\app\\install",
-
+ "site.global.pid_file": "${AGENT_WORK_ROOT}\\app\\run\\component.pid",
"site.global.additional_cp": "C:\\hdp\\hadoop-2.4.0.2.1.3.0-1990\\share\\hadoop\\common\\lib\\*",
"site.global.xmx_val": "256m",
"site.global.xms_val": "128m",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/app-packages/memcached-win/package/scripts/memcached.py
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/package/scripts/memcached.py b/app-packages/memcached-win/package/scripts/memcached.py
index 1bb05dc..bc9905d 100644
--- a/app-packages/memcached-win/package/scripts/memcached.py
+++ b/app-packages/memcached-win/package/scripts/memcached.py
@@ -39,7 +39,8 @@ class Memcached(Script):
Execute(process_cmd,
user=params.app_user,
logoutput=False,
- wait_for_finish=False
+ wait_for_finish=False,
+ pid_file=params.pid_file
)
def stop(self, env):
@@ -49,6 +50,8 @@ class Memcached(Script):
def status(self, env):
import params
env.set_params(params)
+ #Check process status need to be changed for Windows
+ #check_process_status(params.pid_file)
if __name__ == "__main__":
Memcached().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/app-packages/memcached-win/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/package/scripts/params.py b/app-packages/memcached-win/package/scripts/params.py
index f8e64da..fab3714 100644
--- a/app-packages/memcached-win/package/scripts/params.py
+++ b/app-packages/memcached-win/package/scripts/params.py
@@ -26,6 +26,7 @@ config = Script.get_config()
app_root = config['configurations']['global']['app_root']
java64_home = config['hostLevelParams']['java_home']
app_user = config['configurations']['global']['app_user']
+pid_file = config['configurations']['global']['pid_file']
additional_cp = config['configurations']['global']['additional_cp']
xmx_val = config['configurations']['global']['xmx_val']
xms_val = config['configurations']['global']['xms_val']
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/app-packages/memcached/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached/appConfig.json b/app-packages/memcached/appConfig.json
index b179723..5f32030 100644
--- a/app-packages/memcached/appConfig.json
+++ b/app-packages/memcached/appConfig.json
@@ -8,6 +8,7 @@
"site.global.app_user": "yarn",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/jmemcached-1.0.0",
+ "site.global.pid_file": "${AGENT_WORK_ROOT}/app/run/component.pid",
"site.global.additional_cp": "/usr/lib/hadoop/lib/*",
"site.global.xmx_val": "256m",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/app-packages/memcached/package/scripts/memcached.py
----------------------------------------------------------------------
diff --git a/app-packages/memcached/package/scripts/memcached.py b/app-packages/memcached/package/scripts/memcached.py
index cf04cf1..6e14e86 100644
--- a/app-packages/memcached/package/scripts/memcached.py
+++ b/app-packages/memcached/package/scripts/memcached.py
@@ -24,6 +24,7 @@ from resource_management import *
class Memcached(Script):
def install(self, env):
self.install_packages(env)
+ pass
def configure(self, env):
import params
@@ -38,7 +39,8 @@ class Memcached(Script):
Execute(process_cmd,
user=params.app_user,
logoutput=False,
- wait_for_finish=False
+ wait_for_finish=False,
+ pid_file=params.pid_file
)
def stop(self, env):
@@ -48,6 +50,7 @@ class Memcached(Script):
def status(self, env):
import params
env.set_params(params)
+ check_process_status(params.pid_file)
if __name__ == "__main__":
Memcached().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/app-packages/memcached/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/memcached/package/scripts/params.py b/app-packages/memcached/package/scripts/params.py
index f8e64da..25b4055 100644
--- a/app-packages/memcached/package/scripts/params.py
+++ b/app-packages/memcached/package/scripts/params.py
@@ -26,6 +26,8 @@ config = Script.get_config()
app_root = config['configurations']['global']['app_root']
java64_home = config['hostLevelParams']['java_home']
app_user = config['configurations']['global']['app_user']
+pid_file = config['configurations']['global']['pid_file']
+
additional_cp = config['configurations']['global']['additional_cp']
xmx_val = config['configurations']['global']['xmx_val']
xms_val = config['configurations']['global']['xms_val']
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/app-packages/storm/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json
index 6d6aa3a..24078cf 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -3,7 +3,7 @@
"metadata": {
},
"global": {
- "application.def": "/slider/storm_v091.zip",
+ "application.def": "package/storm_v091.zip",
"config_types": "storm-site",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"package_list": "files/apache-storm-0.9.1.2.1.1.0-237.tar.gz",
@@ -68,7 +68,7 @@
"site.storm-site.logviewer.appender.name": "A1",
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
- "site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}]",
+ "site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
"site.storm-site.nimbus.file.copy.expiration.secs": "600",
"site.storm-site.supervisor.monitor.frequency.secs": "3",
"site.storm-site.transactional.zookeeper.servers": "null",
@@ -105,7 +105,7 @@
"site.storm-site.topology.trident.batch.emit.interval.millis": "500",
"site.storm-site.topology.builtin.metrics.bucket.size.secs": "60",
"site.storm-site.storm.thrift.transport": "backtype.storm.security.auth.SimpleTransportPlugin",
- "site.storm-site.logviewer.port": "0",
+ "site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}",
"site.storm-site.topology.debug": "false"
},
"components": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/app-packages/storm/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index 7edd794..e94bc62 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -22,6 +22,7 @@
<name>STORM</name>
<comment>Apache Hadoop Stream processing framework</comment>
<version>0.9.1.2.1</version>
+ <exportedConfigs>storm-site</exportedConfigs>
<exportGroups>
<exportGroup>
@@ -76,6 +77,7 @@
<component>
<name>NIMBUS</name>
<category>MASTER</category>
+ <autoStartOnFailure>true</autoStartOnFailure>
<commandScript>
<script>scripts/nimbus.py</script>
<scriptType>PYTHON</scriptType>
@@ -86,6 +88,7 @@
<component>
<name>STORM_REST_API</name>
<category>MASTER</category>
+ <autoStartOnFailure>true</autoStartOnFailure>
<commandScript>
<script>scripts/rest_api.py</script>
<scriptType>PYTHON</scriptType>
@@ -96,6 +99,13 @@
<component>
<name>SUPERVISOR</name>
<category>SLAVE</category>
+ <autoStartOnFailure>true</autoStartOnFailure>
+ <exports>
+ <export>
+ <name>log_viewer_port</name>
+ <value>${THIS_HOST}:${site.storm-site.logviewer.port}</value>
+ </export>
+ </exports>
<commandScript>
<script>scripts/supervisor.py</script>
<scriptType>PYTHON</scriptType>
@@ -107,6 +117,7 @@
<name>STORM_UI_SERVER</name>
<category>MASTER</category>
<publishConfig>true</publishConfig>
+ <autoStartOnFailure>true</autoStartOnFailure>
<commandScript>
<script>scripts/ui_server.py</script>
<scriptType>PYTHON</scriptType>
@@ -117,6 +128,7 @@
<component>
<name>DRPC_SERVER</name>
<category>MASTER</category>
+ <autoStartOnFailure>true</autoStartOnFailure>
<commandScript>
<script>scripts/drpc_server.py</script>
<scriptType>PYTHON</scriptType>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/app-packages/storm/package/scripts/service.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/scripts/service.py b/app-packages/storm/package/scripts/service.py
index 10fa5b9..13fcef2 100644
--- a/app-packages/storm/package/scripts/service.py
+++ b/app-packages/storm/package/scripts/service.py
@@ -22,7 +22,9 @@ limitations under the License.
from resource_management import *
import time
-
+"""
+Slider package uses jps as pgrep does not list the whole process start command
+"""
def service(
name,
action='start'):
@@ -30,25 +32,25 @@ def service(
import status_params
pid_file = status_params.pid_files[name]
+ container_id = status_params.container_id
no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
+
jps_path = format("{java64_home}/bin/jps")
- grep_and_awk = "| grep -v grep | awk '{print $1}'"
+ grep_and_awk = format("| grep {container_id}") + " | awk '{print $1}'"
if name == 'ui':
- #process_cmd = "^java.+backtype.storm.ui.core$"
- pid_chk_cmd = format("{jps_path} -vl | grep \"^[0-9 ]*backtype.storm.ui.core\" {grep_and_awk} > {pid_file}")
+ crt_pid_cmd = format("{jps_path} -vl | grep \"^[0-9 ]*backtype.storm.ui.core\" {grep_and_awk} > {pid_file}")
elif name == "rest_api":
- process_cmd = format("{java64_home}/bin/java -jar {rest_lib_dir}/`ls {rest_lib_dir} | grep -wE storm-rest-[0-9.-]+\.jar` server")
- crt_pid_cmd = format("pgrep -f \"{process_cmd}\" && pgrep -f \"{process_cmd}\" > {pid_file}")
+ rest_process_cmd = format("{java64_home}/bin/java -jar {rest_lib_dir}/`ls {rest_lib_dir} | grep -wE storm-rest-[0-9.-]+\.jar` server")
+ crt_pid_cmd = format("pgrep -f \"{rest_process_cmd}\" > {pid_file}")
else:
- #process_cmd = format("^java.+backtype.storm.daemon.{name}$")
- pid_chk_cmd = format("{jps_path} -vl | grep \"^[0-9 ]*backtype.storm.daemon.{name}\" {grep_and_awk} > {pid_file}")
+ crt_pid_cmd = format("{jps_path} -vl | grep \"^[0-9 ]*backtype.storm.daemon.{name}\" {grep_and_awk} > {pid_file}")
if action == "start":
if name == "rest_api":
- cmd = format("{process_cmd} {rest_api_conf_file} > {log_dir}/restapi.log")
+ cmd = format("{rest_process_cmd} {rest_api_conf_file} > {log_dir}/restapi.log")
else:
- cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {storm_bin} {name}")
+ cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {storm_bin} {name} > {log_dir}/{name}.out 2>&1")
Execute(cmd,
not_if=no_op_test,
@@ -67,7 +69,7 @@ def service(
else:
content = None
for i in xrange(12):
- Execute(pid_chk_cmd,
+ Execute(crt_pid_cmd,
user=params.storm_user,
logoutput=True
)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/app-packages/storm/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/scripts/status_params.py b/app-packages/storm/package/scripts/status_params.py
index eab83cf..5907446 100644
--- a/app-packages/storm/package/scripts/status_params.py
+++ b/app-packages/storm/package/scripts/status_params.py
@@ -21,6 +21,7 @@ from resource_management import *
config = Script.get_config()
+container_id = config['configurations']['global']['app_container_id']
pid_dir = config['configurations']['global']['app_pid_dir']
pid_nimbus = format("{pid_dir}/nimbus.pid")
pid_supervisor = format("{pid_dir}/supervisor.pid")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-agent/src/main/python/agent/Controller.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Controller.py b/slider-agent/src/main/python/agent/Controller.py
index 4a103cb..1a01bf8 100644
--- a/slider-agent/src/main/python/agent/Controller.py
+++ b/slider-agent/src/main/python/agent/Controller.py
@@ -237,7 +237,8 @@ class Controller(threading.Thread):
return
if serverId != self.responseId + 1:
- logger.error("Error in responseId sequence - restarting")
+ logger.error("Error in responseId sequence expected " + str(self.responseId + 1)
+ + " but got " + str(serverId) + " - restarting")
self.restartAgent()
else:
self.responseId = serverId
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 0f4acc6..0078c9c 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -259,7 +259,7 @@ class CustomServiceOrchestrator():
elif allocated_for_any in value:
## All unallocated ports should be set to 0
logger.info("Assigning port 0 " + "for " + value)
- value = "0"
+ value = self.set_all_unallocated_ports(value)
command['configurations'][key][k] = value
pass
pass
@@ -274,6 +274,35 @@ class CustomServiceOrchestrator():
pass
"""
+ All unallocated ports should be set to 0
+ Look for "${SOME_COMPONENT_NAME.ALLOCATED_PORT}"
+ or "${component.ALLOCATED_PORT}{DEFAULT_port}"
+ or "${component.ALLOCATED_PORT}{DEFAULT_port}{DO_NOT_PROPAGATE}"
+ """
+
+ def set_all_unallocated_ports(self, value):
+ pattern_start = "${"
+ sub_section_start = "}{"
+ pattern_end = "}"
+ index = value.find(pattern_start)
+ while index != -1:
+ replace_index_start = index
+ replace_index_end = value.find(pattern_end, replace_index_start)
+ next_pattern_start = value.find(sub_section_start, replace_index_start)
+ while next_pattern_start == replace_index_end:
+ replace_index_end = value.find(pattern_end, replace_index_end + 1)
+ next_pattern_start = value.find(sub_section_start, next_pattern_start + 1)
+ pass
+
+ value = value[:replace_index_start] + "0" + value[replace_index_end + 1:]
+
+ # look for the next
+ index = value.find(pattern_start)
+
+ return value
+ pass
+
+ """
Port allocation can asks for multiple dynamic ports
port_req_pattern is of type ${component_name.ALLOCATED_PORT}
append {DEFAULT_ and find the default value
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-agent/src/main/python/resource_management/core/providers/system.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/providers/system.py b/slider-agent/src/main/python/resource_management/core/providers/system.py
index e273498..6969c62 100644
--- a/slider-agent/src/main/python/resource_management/core/providers/system.py
+++ b/slider-agent/src/main/python/resource_management/core/providers/system.py
@@ -242,7 +242,8 @@ class ExecuteProvider(Provider):
shell.checked_call(self.resource.command, logoutput=self.resource.logoutput,
cwd=self.resource.cwd, env=self.resource.environment,
preexec_fn=_preexec_fn(self.resource), user=self.resource.user,
- wait_for_finish=self.resource.wait_for_finish, timeout=self.resource.timeout)
+ wait_for_finish=self.resource.wait_for_finish, timeout=self.resource.timeout,
+ pid_file=self.resource.pid_file)
break
except Fail as ex:
if i == self.resource.tries-1: # last try
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
index fa29d7d..f0d4825 100644
--- a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
+++ b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
@@ -30,12 +30,16 @@ import subprocess
import shutil
-def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish=True, timeout=None):
- # TODO implement timeout, logoutput, wait_for_finish
+def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish=True, timeout=None, pid_file_name=None):
+ # TODO implement logoutput
Logger.info("Executing %s" % (command))
proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
cwd=cwd, env=env, shell=False)
if not wait_for_finish:
+ if pid_file_name:
+ pidfile = open(pid_file_name, 'w')
+ pidfile.write(str(proc.pid))
+ pidfile.close()
return None, None
if timeout:
@@ -135,7 +139,8 @@ class ExecuteProvider(Provider):
try:
_call_command(self.resource.command, logoutput=self.resource.logoutput,
cwd=self.resource.cwd, env=self.resource.environment,
- wait_for_finish=self.resource.wait_for_finish, timeout=self.resource.timeout)
+ wait_for_finish=self.resource.wait_for_finish, timeout=self.resource.timeout,
+ pid_file_name=self.resource.pid_file)
break
except Fail as ex:
if i == self.resource.tries - 1: # last try
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-agent/src/main/python/resource_management/core/resources/system.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/resources/system.py b/slider-agent/src/main/python/resource_management/core/resources/system.py
index 2c832a4..a63d993 100644
--- a/slider-agent/src/main/python/resource_management/core/resources/system.py
+++ b/slider-agent/src/main/python/resource_management/core/resources/system.py
@@ -101,6 +101,10 @@ class Execute(Resource):
- try_sleep
"""
wait_for_finish = BooleanArgument(default=True)
+ """
+ if wait_for_finish is True then optionally the caller can ask for the pid to be written
+ """
+ pid_file = ResourceArgument()
class ExecuteScript(Resource):
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-agent/src/main/python/resource_management/core/shell.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/shell.py b/slider-agent/src/main/python/resource_management/core/shell.py
index 92312d5..fb2c946 100644
--- a/slider-agent/src/main/python/resource_management/core/shell.py
+++ b/slider-agent/src/main/python/resource_management/core/shell.py
@@ -31,15 +31,15 @@ from exceptions import ExecuteTimeoutException
from resource_management.core.logger import Logger
def checked_call(command, logoutput=False,
- cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None):
- return _call(command, logoutput, True, cwd, env, preexec_fn, user, wait_for_finish, timeout)
+ cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, pid_file=None):
+ return _call(command, logoutput, True, cwd, env, preexec_fn, user, wait_for_finish, timeout, pid_file)
def call(command, logoutput=False,
- cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None):
- return _call(command, logoutput, False, cwd, env, preexec_fn, user, wait_for_finish, timeout)
+ cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, pid_file=None):
+ return _call(command, logoutput, False, cwd, env, preexec_fn, user, wait_for_finish, timeout, pid_file)
def _call(command, logoutput=False, throw_on_failure=True,
- cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None):
+ cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, pid_file_name=None):
"""
Execute shell command
@@ -67,6 +67,10 @@ def _call(command, logoutput=False, throw_on_failure=True,
preexec_fn=preexec_fn)
if not wait_for_finish:
+ if pid_file_name:
+ pidfile = open(pid_file_name, 'w')
+ pidfile.write(str(proc.pid))
+ pidfile.close()
return None, None
if timeout:
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index c07f05f..48e4df4 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -484,12 +484,20 @@ class TestCustomServiceOrchestrator(TestCase):
command['configurations']['oozie-site'] = {}
command['configurations']['oozie-site']['log_root'] = "${AGENT_LOG_ROOT}"
command['configurations']['oozie-site']['a_port'] = "${HBASE_MASTER.ALLOCATED_PORT}"
+ command['configurations']['oozie-site']['ignore_port1'] = "[${HBASE_RS.ALLOCATED_PORT}]"
+ command['configurations']['oozie-site']['ignore_port2'] = "[${HBASE_RS.ALLOCATED_PORT},${HBASE_REST.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]"
+ command['configurations']['oozie-site']['ignore_port3'] = "[${HBASE_RS.ALLOCATED_PORT}{a}{b}{c},${A.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${A.ALLOCATED_PORT}{DEFAULT_3}{DO_NOT_PROPAGATE}]"
+ command['configurations']['oozie-site']['ignore_port4'] = "${HBASE_RS}{a}{b}{c}"
allocated_ports = {}
orchestrator.finalize_command(command, False, allocated_ports)
self.assertEqual(command['configurations']['hbase-site']['work_root'], tempWorkDir)
self.assertEqual(command['configurations']['oozie-site']['log_root'], tempdir)
self.assertEqual(command['configurations']['oozie-site']['a_port'], "10023")
+ self.assertEqual(command['configurations']['oozie-site']['ignore_port1'], "[0]")
+ self.assertEqual(command['configurations']['oozie-site']['ignore_port2'], "[0,0]")
+ self.assertEqual(command['configurations']['oozie-site']['ignore_port3'], "[0,0,0]")
+ self.assertEqual(command['configurations']['oozie-site']['ignore_port4'], "${HBASE_RS}{a}{b}{c}")
self.assertEqual(orchestrator.applied_configs, {})
self.assertEqual(len(allocated_ports), 1)
self.assertTrue('oozie-site.a_port' in allocated_ports)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java b/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
index 320cc4f..a035a99 100644
--- a/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
@@ -35,7 +35,6 @@ public interface OptionKeys extends InternalKeys {
*/
String APPLICATION_TYPE = "application.type";
-
String APPLICATION_NAME = "application.name";
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index 2381a82..a145b8f 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -48,11 +48,8 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.charset.Charset;
-import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipFile;
import static org.apache.slider.common.SliderXmlConfKeys.CLUSTER_DIRECTORY_PERMISSIONS;
import static org.apache.slider.common.SliderXmlConfKeys.DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 8f92ffa..3a1ee76 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -92,12 +92,6 @@ public class AgentClientProvider extends AbstractClientProvider
getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
Path appDefPath = new Path(appDef);
sliderFileSystem.verifyFileExists(appDefPath);
-
- String pkgList = instanceDefinition.getAppConfOperations().
- getGlobalOptions().getOption(AgentKeys.PACKAGE_LIST, null);
- if (pkgList != null) {
- sliderFileSystem.verifyFileExistsInZip(appDefPath, pkgList);
- }
String agentConf = instanceDefinition.getAppConfOperations().
getGlobalOptions().getOption(AgentKeys.AGENT_CONF, "");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 3225066..559c4e7 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -131,6 +131,7 @@ public class AgentProviderService extends AbstractProviderService implements
private HeartbeatMonitor monitor;
private Boolean canAnyMasterPublish = null;
private AgentLaunchParameter agentLaunchParameter = null;
+ private String clusterName = null;
private final Map<String, ComponentInstanceState> componentStatuses =
new ConcurrentHashMap<String, ComponentInstanceState>();
@@ -629,6 +630,13 @@ public class AgentProviderService extends AbstractProviderService implements
return this.heartbeatMonitorInterval;
}
+ private String getClusterName() {
+ if (clusterName == null || clusterName.length() == 0) {
+ clusterName = getAmState().getInternalsSnapshot().get(OptionKeys.APPLICATION_NAME);
+ }
+ return clusterName;
+ }
+
/**
* Publish a named property bag that may contain name-value pairs for app configurations such as hbase-site
* @param name
@@ -973,11 +981,10 @@ public class AgentProviderService extends AbstractProviderService implements
throws SliderException {
assert getAmState().isApplicationLive();
ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot();
ExecutionCommand cmd = new ExecutionCommand(AgentCommandType.EXECUTION_COMMAND);
prepareExecutionCommand(cmd);
- String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME);
+ String clusterName = getClusterName();
cmd.setClusterName(clusterName);
cmd.setRoleCommand(Command.INSTALL.toString());
cmd.setServiceName(clusterName);
@@ -1050,10 +1057,9 @@ public class AgentProviderService extends AbstractProviderService implements
throws SliderException {
assert getAmState().isApplicationLive();
ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot();
StatusCommand cmd = new StatusCommand();
- String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME);
+ String clusterName = getClusterName();
cmd.setCommandType(AgentCommandType.STATUS_COMMAND);
cmd.setComponentName(roleName);
@@ -1079,10 +1085,9 @@ public class AgentProviderService extends AbstractProviderService implements
protected void addGetConfigCommand(String roleName, String containerId, HeartBeatResponse response)
throws SliderException {
assert getAmState().isApplicationLive();
- ConfTreeOperations internalsConf = getAmState().getInternalsSnapshot();
StatusCommand cmd = new StatusCommand();
- String clusterName = internalsConf.get(OptionKeys.APPLICATION_NAME);
+ String clusterName = getClusterName();
cmd.setCommandType(AgentCommandType.STATUS_COMMAND);
cmd.setComponentName(roleName);
@@ -1203,7 +1208,7 @@ public class AgentProviderService extends AbstractProviderService implements
Map<String, String> tokens, String containerId) {
Map<String, String> config = new HashMap<String, String>();
if (configName.equals(GLOBAL_CONFIG_TAG)) {
- addDefaultGlobalConfig(config);
+ addDefaultGlobalConfig(config, containerId);
}
// add role hosts to tokens
addRoleRelatedTokens(tokens);
@@ -1248,10 +1253,11 @@ public class AgentProviderService extends AbstractProviderService implements
return hosts;
}
- private void addDefaultGlobalConfig(Map<String, String> config) {
- config.put("app_log_dir", "${AGENT_LOG_ROOT}/app/log");
+ private void addDefaultGlobalConfig(Map<String, String> config, String containerId) {
+ config.put("app_log_dir", "${AGENT_LOG_ROOT}");
config.put("app_pid_dir", "${AGENT_WORK_ROOT}/app/run");
config.put("app_install_dir", "${AGENT_WORK_ROOT}/app/install");
+ config.put("app_container_id", containerId);
}
private void buildRoleHostDetails(Map<String, String> details) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
index f7b77ff..acf6ecb 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
@@ -28,6 +28,7 @@ public class Component {
String publishConfig;
String minInstanceCount;
String maxInstanceCount;
+ String autoStartOnFailure;
CommandScript commandScript;
List<Export> exports;
@@ -60,6 +61,14 @@ public class Component {
this.publishConfig = publishConfig;
}
+ public String getAutoStartOnFailure() {
+ return autoStartOnFailure;
+ }
+
+ public void setAutoStartOnFailure(String autoStartOnFailure) {
+ this.autoStartOnFailure = autoStartOnFailure;
+ }
+
public String getMinInstanceCount() {
return minInstanceCount;
}
@@ -92,6 +101,10 @@ public class Component {
return exports;
}
+ public Boolean getRequiresAutoRestart() {
+ return Boolean.parseBoolean(this.autoStartOnFailure);
+ }
+
@Override
public String toString() {
final StringBuilder sb =
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
index deb7a6a..23134b6 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
@@ -39,6 +39,7 @@ public class MetainfoParser {
digester.addBeanPropertySetter("*/application/comment");
digester.addBeanPropertySetter("*/application/version");
digester.addBeanPropertySetter("*/application/exportedConfigs");
+ digester.addBeanPropertySetter("*/application/autoStartOnFailure");
digester.addObjectCreate("*/commandOrder", CommandOrder.class);
digester.addBeanPropertySetter("*/commandOrder/command");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/004fc8df/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index ee61129..a68e770 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -134,6 +134,7 @@ public class TestAgentProviderService {
+ " <component>\n"
+ " <name>HBASE_MASTER</name>\n"
+ " <category>MASTER</category>\n"
+ + " <autoStartOnFailure>true</autoStartOnFailure>\n"
+ " <minInstanceCount>1</minInstanceCount>\n"
+ " <maxInstanceCount>2</maxInstanceCount>\n"
+ " <commandScript>\n"
@@ -146,6 +147,7 @@ public class TestAgentProviderService {
+ " <name>HBASE_REGIONSERVER</name>\n"
+ " <category>SLAVE</category>\n"
+ " <minInstanceCount>1</minInstanceCount>\n"
+ + " <autoStartOnFailure>Falsee</autoStartOnFailure>\n"
+ " <commandScript>\n"
+ " <script>scripts/hbase_regionserver.py</script>\n"
+ " <scriptType>PYTHON</scriptType>\n"
@@ -469,6 +471,8 @@ public class TestAgentProviderService {
int found = 0;
for (Component component : components) {
if (component.getName().equals("HBASE_MASTER")) {
+ Assert.assertEquals(component.getAutoStartOnFailure(), "true");
+ Assert.assertEquals(component.getRequiresAutoRestart(), Boolean.TRUE);
Assert.assertEquals(component.getMinInstanceCount(), "1");
Assert.assertEquals(component.getMaxInstanceCount(), "2");
Assert.assertEquals(component.getCommandScript().getScript(), "scripts/hbase_master.py");
@@ -477,6 +481,8 @@ public class TestAgentProviderService {
found++;
}
if (component.getName().equals("HBASE_REGIONSERVER")) {
+ Assert.assertEquals(component.getAutoStartOnFailure(), "Falsee");
+ Assert.assertEquals(component.getRequiresAutoRestart(), Boolean.FALSE);
Assert.assertEquals(component.getMinInstanceCount(), "1");
Assert.assertNull(component.getMaxInstanceCount());
Assert.assertEquals(component.getCommandScript().getScript(), "scripts/hbase_regionserver.py");
[42/50] [abbrv] git commit: SLIDER-332. Add LICENSE file and license
headers to kazoo scripts (Gour Saha via smohanty)
Posted by st...@apache.org.
SLIDER-332. Add LICENSE file and license headers to kazoo scripts (Gour Saha via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a34cca8d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a34cca8d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a34cca8d
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: a34cca8dcdf8cf4918a5a54f5c472befc69ed922
Parents: 72daf37
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Mon Aug 18 15:46:49 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Mon Aug 18 15:46:49 2014 -0700
----------------------------------------------------------------------
slider-agent/src/main/python/kazoo/LICENSE | 176 +++++++++++++++++++
slider-agent/src/main/python/kazoo/__init__.py | 1 +
slider-agent/src/main/python/kazoo/client.py | 1 +
.../src/main/python/kazoo/exceptions.py | 1 +
.../src/main/python/kazoo/handlers/__init__.py | 1 +
.../src/main/python/kazoo/handlers/gevent.py | 1 +
.../src/main/python/kazoo/handlers/threading.py | 1 +
.../src/main/python/kazoo/handlers/utils.py | 1 +
slider-agent/src/main/python/kazoo/hosts.py | 1 +
.../src/main/python/kazoo/interfaces.py | 1 +
.../src/main/python/kazoo/loggingsupport.py | 1 +
.../src/main/python/kazoo/protocol/__init__.py | 1 +
.../main/python/kazoo/protocol/connection.py | 1 +
.../src/main/python/kazoo/protocol/paths.py | 1 +
.../main/python/kazoo/protocol/serialization.py | 1 +
.../src/main/python/kazoo/protocol/states.py | 1 +
.../src/main/python/kazoo/recipe/__init__.py | 1 +
.../src/main/python/kazoo/recipe/barrier.py | 1 +
.../src/main/python/kazoo/recipe/counter.py | 1 +
.../src/main/python/kazoo/recipe/election.py | 1 +
.../src/main/python/kazoo/recipe/lock.py | 1 +
.../src/main/python/kazoo/recipe/partitioner.py | 1 +
.../src/main/python/kazoo/recipe/party.py | 1 +
.../src/main/python/kazoo/recipe/queue.py | 1 +
.../src/main/python/kazoo/recipe/watchers.py | 1 +
slider-agent/src/main/python/kazoo/retry.py | 1 +
slider-agent/src/main/python/kazoo/security.py | 1 +
.../src/main/python/kazoo/testing/__init__.py | 1 +
.../src/main/python/kazoo/testing/common.py | 1 +
.../src/main/python/kazoo/testing/harness.py | 1 +
.../src/main/python/kazoo/tests/test_barrier.py | 1 +
.../src/main/python/kazoo/tests/test_build.py | 1 +
.../src/main/python/kazoo/tests/test_client.py | 1 +
.../main/python/kazoo/tests/test_connection.py | 1 +
.../src/main/python/kazoo/tests/test_counter.py | 1 +
.../main/python/kazoo/tests/test_election.py | 1 +
.../main/python/kazoo/tests/test_exceptions.py | 1 +
.../python/kazoo/tests/test_gevent_handler.py | 1 +
.../src/main/python/kazoo/tests/test_lock.py | 1 +
.../main/python/kazoo/tests/test_partitioner.py | 1 +
.../src/main/python/kazoo/tests/test_party.py | 1 +
.../src/main/python/kazoo/tests/test_paths.py | 1 +
.../src/main/python/kazoo/tests/test_queue.py | 1 +
.../src/main/python/kazoo/tests/test_retry.py | 1 +
.../main/python/kazoo/tests/test_security.py | 1 +
.../kazoo/tests/test_threading_handler.py | 1 +
.../main/python/kazoo/tests/test_watchers.py | 1 +
.../src/main/python/kazoo/tests/util.py | 1 +
48 files changed, 223 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/LICENSE
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/LICENSE b/slider-agent/src/main/python/kazoo/LICENSE
new file mode 100644
index 0000000..68c771a
--- /dev/null
+++ b/slider-agent/src/main/python/kazoo/LICENSE
@@ -0,0 +1,176 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/__init__.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/__init__.py b/slider-agent/src/main/python/kazoo/__init__.py
index 792d600..a7bacf3 100644
--- a/slider-agent/src/main/python/kazoo/__init__.py
+++ b/slider-agent/src/main/python/kazoo/__init__.py
@@ -1 +1,2 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
#
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/client.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/client.py b/slider-agent/src/main/python/kazoo/client.py
index 6e3a219..a315489 100644
--- a/slider-agent/src/main/python/kazoo/client.py
+++ b/slider-agent/src/main/python/kazoo/client.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
"""Kazoo Zookeeper Client"""
import inspect
import logging
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/exceptions.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/exceptions.py b/slider-agent/src/main/python/kazoo/exceptions.py
index 8d4f0f3..9c9e71d 100644
--- a/slider-agent/src/main/python/kazoo/exceptions.py
+++ b/slider-agent/src/main/python/kazoo/exceptions.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
"""Kazoo Exceptions"""
from collections import defaultdict
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/handlers/__init__.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/handlers/__init__.py b/slider-agent/src/main/python/kazoo/handlers/__init__.py
index 792d600..a7bacf3 100644
--- a/slider-agent/src/main/python/kazoo/handlers/__init__.py
+++ b/slider-agent/src/main/python/kazoo/handlers/__init__.py
@@ -1 +1,2 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
#
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/handlers/gevent.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/handlers/gevent.py b/slider-agent/src/main/python/kazoo/handlers/gevent.py
index 6e40cae..5e07143 100644
--- a/slider-agent/src/main/python/kazoo/handlers/gevent.py
+++ b/slider-agent/src/main/python/kazoo/handlers/gevent.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
"""A gevent based handler."""
from __future__ import absolute_import
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/handlers/threading.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/handlers/threading.py b/slider-agent/src/main/python/kazoo/handlers/threading.py
index 8e6648d..b2e355a 100644
--- a/slider-agent/src/main/python/kazoo/handlers/threading.py
+++ b/slider-agent/src/main/python/kazoo/handlers/threading.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
"""A threading based handler.
The :class:`SequentialThreadingHandler` is intended for regular Python
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/handlers/utils.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/handlers/utils.py b/slider-agent/src/main/python/kazoo/handlers/utils.py
index ad204b5..60d6404 100644
--- a/slider-agent/src/main/python/kazoo/handlers/utils.py
+++ b/slider-agent/src/main/python/kazoo/handlers/utils.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
"""Kazoo handler helpers"""
HAS_FNCTL = True
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/hosts.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/hosts.py b/slider-agent/src/main/python/kazoo/hosts.py
index 15ce447..ca0dd35 100644
--- a/slider-agent/src/main/python/kazoo/hosts.py
+++ b/slider-agent/src/main/python/kazoo/hosts.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
import random
try:
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/interfaces.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/interfaces.py b/slider-agent/src/main/python/kazoo/interfaces.py
index 351f1fd..7aff561 100644
--- a/slider-agent/src/main/python/kazoo/interfaces.py
+++ b/slider-agent/src/main/python/kazoo/interfaces.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
"""Kazoo Interfaces
.. versionchanged:: 1.4
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/loggingsupport.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/loggingsupport.py b/slider-agent/src/main/python/kazoo/loggingsupport.py
index 5ed2f8f..91ae632 100644
--- a/slider-agent/src/main/python/kazoo/loggingsupport.py
+++ b/slider-agent/src/main/python/kazoo/loggingsupport.py
@@ -1,2 +1,3 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
BLATHER = 5 # log level for low-level debugging
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/protocol/__init__.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/protocol/__init__.py b/slider-agent/src/main/python/kazoo/protocol/__init__.py
index 792d600..a7bacf3 100644
--- a/slider-agent/src/main/python/kazoo/protocol/__init__.py
+++ b/slider-agent/src/main/python/kazoo/protocol/__init__.py
@@ -1 +1,2 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
#
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/protocol/connection.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/protocol/connection.py b/slider-agent/src/main/python/kazoo/protocol/connection.py
index e6ad63d..3cbb87f 100644
--- a/slider-agent/src/main/python/kazoo/protocol/connection.py
+++ b/slider-agent/src/main/python/kazoo/protocol/connection.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
"""Zookeeper Protocol Connection Handler"""
import logging
import os
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/protocol/paths.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/protocol/paths.py b/slider-agent/src/main/python/kazoo/protocol/paths.py
index 52b2d6b..e37c1a7 100644
--- a/slider-agent/src/main/python/kazoo/protocol/paths.py
+++ b/slider-agent/src/main/python/kazoo/protocol/paths.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
def normpath(path, trailing=False):
"""Normalize path, eliminating double slashes, etc."""
comps = path.split('/')
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/protocol/serialization.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/protocol/serialization.py b/slider-agent/src/main/python/kazoo/protocol/serialization.py
index f44f49a..3b5df6c 100644
--- a/slider-agent/src/main/python/kazoo/protocol/serialization.py
+++ b/slider-agent/src/main/python/kazoo/protocol/serialization.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
"""Zookeeper Serializers, Deserializers, and NamedTuple objects"""
from collections import namedtuple
import struct
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/protocol/states.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/protocol/states.py b/slider-agent/src/main/python/kazoo/protocol/states.py
index 395c013..d09c2c6 100644
--- a/slider-agent/src/main/python/kazoo/protocol/states.py
+++ b/slider-agent/src/main/python/kazoo/protocol/states.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
"""Kazoo State and Event objects"""
from collections import namedtuple
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/recipe/__init__.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/recipe/__init__.py b/slider-agent/src/main/python/kazoo/recipe/__init__.py
index 792d600..a7bacf3 100644
--- a/slider-agent/src/main/python/kazoo/recipe/__init__.py
+++ b/slider-agent/src/main/python/kazoo/recipe/__init__.py
@@ -1 +1,2 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
#
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/recipe/barrier.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/recipe/barrier.py b/slider-agent/src/main/python/kazoo/recipe/barrier.py
index 05addb4..729ac07 100644
--- a/slider-agent/src/main/python/kazoo/recipe/barrier.py
+++ b/slider-agent/src/main/python/kazoo/recipe/barrier.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
"""Zookeeper Barriers
:Maintainer: None
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/recipe/counter.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/recipe/counter.py b/slider-agent/src/main/python/kazoo/recipe/counter.py
index ed80f51..026d724 100644
--- a/slider-agent/src/main/python/kazoo/recipe/counter.py
+++ b/slider-agent/src/main/python/kazoo/recipe/counter.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
"""Zookeeper Counter
:Maintainer: None
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/recipe/election.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/recipe/election.py b/slider-agent/src/main/python/kazoo/recipe/election.py
index 3089fa6..5349e26 100644
--- a/slider-agent/src/main/python/kazoo/recipe/election.py
+++ b/slider-agent/src/main/python/kazoo/recipe/election.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
"""ZooKeeper Leader Elections
:Maintainer: None
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/recipe/lock.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/recipe/lock.py b/slider-agent/src/main/python/kazoo/recipe/lock.py
index 8c944b7..8f14847 100644
--- a/slider-agent/src/main/python/kazoo/recipe/lock.py
+++ b/slider-agent/src/main/python/kazoo/recipe/lock.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
"""Zookeeper Locking Implementations
:Maintainer: None
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/recipe/partitioner.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/recipe/partitioner.py b/slider-agent/src/main/python/kazoo/recipe/partitioner.py
index 8de4231..0d92661 100644
--- a/slider-agent/src/main/python/kazoo/recipe/partitioner.py
+++ b/slider-agent/src/main/python/kazoo/recipe/partitioner.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
"""Zookeeper Partitioner Implementation
:Maintainer: None
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/recipe/party.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/recipe/party.py b/slider-agent/src/main/python/kazoo/recipe/party.py
index 7186c10..4fd873e 100644
--- a/slider-agent/src/main/python/kazoo/recipe/party.py
+++ b/slider-agent/src/main/python/kazoo/recipe/party.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
"""Party
:Maintainer: Ben Bangert <be...@groovie.org>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/recipe/queue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/recipe/queue.py b/slider-agent/src/main/python/kazoo/recipe/queue.py
index e2e30dd..81289b9 100644
--- a/slider-agent/src/main/python/kazoo/recipe/queue.py
+++ b/slider-agent/src/main/python/kazoo/recipe/queue.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
"""Zookeeper based queue implementations.
:Maintainer: None
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/recipe/watchers.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/recipe/watchers.py b/slider-agent/src/main/python/kazoo/recipe/watchers.py
index 0624153..f9667f8 100644
--- a/slider-agent/src/main/python/kazoo/recipe/watchers.py
+++ b/slider-agent/src/main/python/kazoo/recipe/watchers.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
"""Higher level child and data watching API's.
:Maintainer: Ben Bangert <be...@groovie.org>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/retry.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/retry.py b/slider-agent/src/main/python/kazoo/retry.py
index 152e9a6..229b99d 100644
--- a/slider-agent/src/main/python/kazoo/retry.py
+++ b/slider-agent/src/main/python/kazoo/retry.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
import logging
import random
import time
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/security.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/security.py b/slider-agent/src/main/python/kazoo/security.py
index 1bd989d..4532489 100644
--- a/slider-agent/src/main/python/kazoo/security.py
+++ b/slider-agent/src/main/python/kazoo/security.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
"""Kazoo Security"""
from base64 import b64encode
from collections import namedtuple
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/testing/__init__.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/testing/__init__.py b/slider-agent/src/main/python/kazoo/testing/__init__.py
index bf8f149..660546b 100644
--- a/slider-agent/src/main/python/kazoo/testing/__init__.py
+++ b/slider-agent/src/main/python/kazoo/testing/__init__.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
from kazoo.testing.harness import KazooTestCase
from kazoo.testing.harness import KazooTestHarness
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/testing/common.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/testing/common.py b/slider-agent/src/main/python/kazoo/testing/common.py
index 8e1e9b4..b497a8e 100644
--- a/slider-agent/src/main/python/kazoo/testing/common.py
+++ b/slider-agent/src/main/python/kazoo/testing/common.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
#
# Copyright (C) 2010-2011, 2011 Canonical Ltd. All Rights Reserved
#
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/testing/harness.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/testing/harness.py b/slider-agent/src/main/python/kazoo/testing/harness.py
index 0a9079a..93cc744 100644
--- a/slider-agent/src/main/python/kazoo/testing/harness.py
+++ b/slider-agent/src/main/python/kazoo/testing/harness.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
"""Kazoo testing harnesses"""
import atexit
import logging
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_barrier.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_barrier.py b/slider-agent/src/main/python/kazoo/tests/test_barrier.py
index 94b6e92..461664f 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_barrier.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_barrier.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
import threading
from nose.tools import eq_
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_build.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_build.py b/slider-agent/src/main/python/kazoo/tests/test_build.py
index 1f9316f..0f75d7c 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_build.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_build.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
import os
from nose import SkipTest
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_client.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_client.py b/slider-agent/src/main/python/kazoo/tests/test_client.py
index b99ee36..eb19ef5 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_client.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_client.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
import socket
import sys
import threading
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_connection.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_connection.py b/slider-agent/src/main/python/kazoo/tests/test_connection.py
index faa121e..c764b03 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_connection.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_connection.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
from collections import namedtuple
import os
import errno
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_counter.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_counter.py b/slider-agent/src/main/python/kazoo/tests/test_counter.py
index d582a45..b0361d0 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_counter.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_counter.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
import uuid
from nose.tools import eq_
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_election.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_election.py b/slider-agent/src/main/python/kazoo/tests/test_election.py
index 12a04a2..a9610bf 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_election.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_election.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
import uuid
import sys
import threading
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_exceptions.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_exceptions.py b/slider-agent/src/main/python/kazoo/tests/test_exceptions.py
index 154bcb7..e469089 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_exceptions.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_exceptions.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
from unittest import TestCase
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_gevent_handler.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_gevent_handler.py b/slider-agent/src/main/python/kazoo/tests/test_gevent_handler.py
index 67ccb11..71d9727 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_gevent_handler.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_gevent_handler.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
import unittest
from nose import SkipTest
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_lock.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_lock.py b/slider-agent/src/main/python/kazoo/tests/test_lock.py
index 36b6d42..6dd15b0 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_lock.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_lock.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
import uuid
import threading
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_partitioner.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_partitioner.py b/slider-agent/src/main/python/kazoo/tests/test_partitioner.py
index a323d07..1a4f205 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_partitioner.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_partitioner.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
import uuid
import time
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_party.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_party.py b/slider-agent/src/main/python/kazoo/tests/test_party.py
index d44eec7..61400ae 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_party.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_party.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
import uuid
from nose.tools import eq_
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_paths.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_paths.py b/slider-agent/src/main/python/kazoo/tests/test_paths.py
index e092196..c9064bb 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_paths.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_paths.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
import sys
from unittest import TestCase
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_queue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_queue.py b/slider-agent/src/main/python/kazoo/tests/test_queue.py
index 6a9ec68..4c13ca9 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_queue.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_queue.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
import uuid
from nose import SkipTest
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_retry.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_retry.py b/slider-agent/src/main/python/kazoo/tests/test_retry.py
index b85739b..84c8d41 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_retry.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_retry.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
import unittest
from nose.tools import eq_
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_security.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_security.py b/slider-agent/src/main/python/kazoo/tests/test_security.py
index 4a7e670..587c265 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_security.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_security.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
import unittest
from nose.tools import eq_
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_threading_handler.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_threading_handler.py b/slider-agent/src/main/python/kazoo/tests/test_threading_handler.py
index b4b93f1..4de5781 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_threading_handler.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_threading_handler.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
import threading
import unittest
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/test_watchers.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/test_watchers.py b/slider-agent/src/main/python/kazoo/tests/test_watchers.py
index 1c51d60..44795c4 100644
--- a/slider-agent/src/main/python/kazoo/tests/test_watchers.py
+++ b/slider-agent/src/main/python/kazoo/tests/test_watchers.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
import time
import threading
import uuid
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a34cca8d/slider-agent/src/main/python/kazoo/tests/util.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/kazoo/tests/util.py b/slider-agent/src/main/python/kazoo/tests/util.py
index 298be71..906cbc0 100644
--- a/slider-agent/src/main/python/kazoo/tests/util.py
+++ b/slider-agent/src/main/python/kazoo/tests/util.py
@@ -1,3 +1,4 @@
+"""license: Apache License 2.0, see LICENSE for more details."""
##############################################################################
#
# Copyright Zope Foundation and Contributors.
[11/50] [abbrv] git commit: SLIDER-310 failure threshold can be set
per component, which overrides the global value
Posted by st...@apache.org.
SLIDER-310 failure threshold can be set per component, which overrides the global value
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/564462e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/564462e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/564462e8
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 564462e89fcc80b6214585491ce712d65edd1bed
Parents: 2f8bda2
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 14 12:37:05 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 14 13:19:27 2014 +0100
----------------------------------------------------------------------
.../slider/server/appmaster/state/AppState.java | 17 +++++++++--
.../TestRegionServerFailureThreshold.groovy | 31 ++++++++++++++++----
2 files changed, 40 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/564462e8/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index d616678..d22c716 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1480,7 +1480,8 @@ public class AppState {
throw new TriggerClusterTeardownException(
SliderExitCodes.EXIT_DEPLOYMENT_FAILED,
ErrorStrings.E_UNSTABLE_CLUSTER +
- " - failed with role %s failing %d times (%d in startup); threshold is %d - last failure: %s",
+ " - failed with role %s failing %d times (%d in startup);" +
+ " threshold is %d - last failure: %s",
role.getName(),
role.getFailed(),
role.getStartFailed(),
@@ -1489,8 +1490,18 @@ public class AppState {
}
}
- private int getFailureThresholdForRole(RoleStatus status) {
- return failureThreshold;
+ /**
+ * Get the failure threshold for a specific role, falling back to
+ * the global one if not
+ * @param roleStatus
+ * @return the threshold for failures
+ */
+ private int getFailureThresholdForRole(RoleStatus roleStatus) {
+ ConfTreeOperations resources =
+ instanceDefinition.getResourceOperations();
+ return resources.getComponentOptInt(roleStatus.getName(),
+ ResourceKeys.CONTAINER_FAILURE_SHORTLIFE,
+ failureThreshold);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/564462e8/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy
index b3aaa48..eb44ae0 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy
@@ -34,6 +34,8 @@ import org.apache.slider.client.SliderClient
import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
import org.junit.Test
+import static org.apache.slider.providers.hbase.HBaseKeys.ROLE_WORKER
+
/**
* test that if a container is killed too many times,
* the AM stays down
@@ -44,10 +46,17 @@ import org.junit.Test
class TestRegionServerFailureThreshold extends HBaseMiniClusterTestBase {
@Test
- public void testFailedRegionService() throws Throwable {
+ public void testRegionServerFailureThreshold() throws Throwable {
failureThresholdTestRun("", true, 2, 5)
}
+ /**
+ * Sets the failure threshold then runs the #of kill attempts
+ * @param testName
+ * @param toKill
+ * @param threshold
+ * @param killAttempts
+ */
private void failureThresholdTestRun(
String testName,
boolean toKill,
@@ -57,16 +66,23 @@ class TestRegionServerFailureThreshold extends HBaseMiniClusterTestBase {
int regionServerCount = 1
String clustername = createMiniCluster(testName, configuration, 1, 1, 1, true, true)
describe(
- "Create a single region service HBase instance then " + action + " the RS");
+ "Create a single region service HBase instance" +
+ "then $action the RS $killAttempts times with a threshold of $threshold");
//now launch the cluster
+ def globalThreshold = threshold - 1
ServiceLauncher<SliderClient> launcher = createHBaseCluster(
clustername,
regionServerCount,
[
+ Arguments.ARG_RES_COMP_OPT,
+ ROLE_WORKER,
+ ResourceKeys.CONTAINER_FAILURE_THRESHOLD,
+ Integer.toString(threshold),
+
Arguments.ARG_RESOURCE_OPT,
ResourceKeys.CONTAINER_FAILURE_THRESHOLD,
- Integer.toString(threshold)
+ Integer.toString(globalThreshold)
],
true,
true)
@@ -74,9 +90,14 @@ class TestRegionServerFailureThreshold extends HBaseMiniClusterTestBase {
addToTeardown(client);
def aggregateConf = client.loadPersistedClusterDescription(clustername)
log.info aggregateConf.toString()
- def failureOptValue = aggregateConf.resourceOperations.globalOptions.getMandatoryOptionInt(
+
+ def resourceOperations = aggregateConf.resourceOperations
+ def failureOptValue = resourceOperations.globalOptions.getMandatoryOptionInt(
ResourceKeys.CONTAINER_FAILURE_THRESHOLD)
- assert threshold == failureOptValue
+ assert globalThreshold == failureOptValue
+ def workerThreshold = resourceOperations.getComponentOptInt(ROLE_WORKER,
+ ResourceKeys.CONTAINER_FAILURE_THRESHOLD, 0)
+ assert threshold == workerThreshold
ClusterDescription status = client.getClusterDescription(clustername)
ClusterStatus clustat = basicHBaseClusterStartupSequence(client)
[10/50] [abbrv] git commit: SLIDER-313. Slider AM fails to restart on
kill, post newly created application.
Posted by st...@apache.org.
SLIDER-313. Slider AM fails to restart on kill, post newly created application.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2f8bda2f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2f8bda2f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2f8bda2f
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 2f8bda2f3da4e8ddc37710e66d29d36652754a0e
Parents: 9b20e47
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Aug 13 14:23:49 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Aug 13 14:23:49 2014 -0700
----------------------------------------------------------------------
.../providers/agent/AgentProviderService.java | 27 ++++++++++++--------
1 file changed, 16 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2f8bda2f/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 0c57b79..43ee7a6 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterDescriptionKeys;
import org.apache.slider.api.ClusterNode;
@@ -63,6 +62,7 @@ import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.providers.agent.application.metadata.OSPackage;
import org.apache.slider.providers.agent.application.metadata.OSSpecific;
import org.apache.slider.server.appmaster.actions.ProviderReportedContainerLoss;
+import org.apache.slider.server.appmaster.state.ContainerPriority;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
@@ -339,16 +339,21 @@ public class AgentProviderService extends AbstractProviderService implements
String applicationId, Map<Integer, ProviderRole> providerRoleMap) {
for (Container container : liveContainers) {
// get the role name and label
- Priority priority = container.getPriority();
- String roleName = providerRoleMap.get(priority.getPriority()).name;
- String label = getContainerLabel(container, roleName);
- log.info("Rebuilding in-memory: container {} in role {} in cluster {}",
- container.getId().toString(), roleName, applicationId);
- getComponentStatuses()
- .put(
- label,
- new ComponentInstanceState(roleName, container.getId(),
- applicationId));
+ ProviderRole role = providerRoleMap.get(ContainerPriority
+ .extractRole(container));
+ if (role != null) {
+ String roleName = role.name;
+ String label = getContainerLabel(container, roleName);
+ log.info("Rebuilding in-memory: container {} in role {} in cluster {}",
+ container.getId(), roleName, applicationId);
+ getComponentStatuses().put(
+ label,
+ new ComponentInstanceState(roleName, container.getId(),
+ applicationId));
+ } else {
+ log.warn("Role not found for container {} in cluster {}",
+ container.getId(), applicationId);
+ }
}
}
[15/50] [abbrv] git commit: SLIDER-322 move all <> constructors to
typed
Posted by st...@apache.org.
SLIDER-322 move all <> constructors to typed
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/daba289e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/daba289e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/daba289e
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: daba289e9327150078f3a38724db2dcbaec2ab12
Parents: 52f1f6f
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 14 16:47:45 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 14 16:47:45 2014 +0100
----------------------------------------------------------------------
.../src/main/java/org/apache/slider/Slider.java | 2 +-
.../apache/slider/api/ClusterDescription.java | 14 ++---
.../org/apache/slider/client/SliderClient.java | 6 +--
.../slider/client/SliderYarnClientImpl.java | 8 +--
.../common/params/AbstractActionArgs.java | 6 +--
.../params/AppAndResouceOptionArgsDelegate.java | 8 +--
.../org/apache/slider/common/params/ArgOps.java | 8 +--
.../apache/slider/common/params/CommonArgs.java | 4 +-
.../common/params/DontSplitArguments.java | 2 +-
.../slider/common/tools/ConfigHelper.java | 4 +-
.../slider/common/tools/CoreFileSystem.java | 2 +-
.../apache/slider/common/tools/SliderUtils.java | 10 ++--
.../org/apache/slider/core/conf/ConfTree.java | 6 +--
.../slider/core/conf/ConfTreeOperations.java | 4 +-
.../apache/slider/core/conf/MapOperations.java | 2 +-
.../slider/core/launch/AbstractLauncher.java | 10 ++--
.../core/launch/ClasspathConstructor.java | 2 +-
.../slider/core/launch/CommandLineBuilder.java | 2 +-
.../slider/core/main/ServiceLauncher.java | 2 +-
.../registry/docstore/PublishedConfigSet.java | 2 +-
.../docstore/PublishedConfiguration.java | 4 +-
.../slider/core/registry/docstore/UriMap.java | 2 +-
.../slider/core/registry/info/RegistryView.java | 3 +-
.../core/registry/info/ServiceInstanceData.java | 4 +-
.../apache/slider/core/zk/ZKIntegration.java | 2 +-
.../apache/slider/core/zk/ZookeeperUtils.java | 6 +--
.../providers/AbstractProviderService.java | 4 +-
.../apache/slider/providers/ProviderUtils.java | 2 +-
.../providers/agent/AgentClientProvider.java | 2 +-
.../providers/agent/AgentLaunchParameter.java | 2 +-
.../providers/agent/AgentProviderService.java | 47 ++++++++--------
.../providers/agent/ComponentCommandOrder.java | 9 ++--
.../agent/application/metadata/Application.java | 8 +--
.../agent/application/metadata/Component.java | 2 +-
.../slideram/SliderAMClientProvider.java | 4 +-
.../slideram/SliderAMProviderService.java | 2 +-
.../server/appmaster/SliderAppMaster.java | 18 +++----
.../appmaster/actions/ActionKillContainer.java | 2 +-
.../server/appmaster/actions/QueueService.java | 6 +--
.../appmaster/monkey/ChaosMonkeyService.java | 2 +-
.../slider/server/appmaster/state/AppState.java | 56 ++++++++++----------
.../MostRecentContainerReleaseSelector.java | 2 +-
.../server/appmaster/state/NodeInstance.java | 2 +-
.../slider/server/appmaster/state/NodeMap.java | 2 +-
.../state/OutstandingRequestTracker.java | 6 +--
.../appmaster/state/ProviderAppState.java | 4 +-
.../server/appmaster/state/RoleHistory.java | 15 +++---
.../server/appmaster/state/RoleStatus.java | 2 +-
.../server/appmaster/web/SliderAMWebApp.java | 2 +-
.../web/SliderAmFilterInitializer.java | 2 +-
.../server/appmaster/web/SliderAmIpFilter.java | 2 +-
.../server/appmaster/web/WebAppApiImpl.java | 2 +-
.../web/rest/publisher/PublisherResource.java | 4 +-
.../slider/server/avro/RoleHistoryWriter.java | 6 +--
.../server/servicemonitor/MonitorUtils.java | 2 +-
.../services/curator/RegistryBinderService.java | 6 +--
.../registry/RegistryRestResources.java | 3 +-
.../registry/SliderRegistryService.java | 2 +-
58 files changed, 181 insertions(+), 174 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/Slider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/Slider.java b/slider-core/src/main/java/org/apache/slider/Slider.java
index 0d25f00..5fc8618 100644
--- a/slider-core/src/main/java/org/apache/slider/Slider.java
+++ b/slider-core/src/main/java/org/apache/slider/Slider.java
@@ -42,7 +42,7 @@ public class Slider extends SliderClient {
//turn the args to a list
List<String> argsList = Arrays.asList(args);
//create a new list, as the ArrayList type doesn't push() on an insert
- List<String> extendedArgs = new ArrayList<>(argsList);
+ List<String> extendedArgs = new ArrayList<String>(argsList);
//insert the service name
extendedArgs.add(0, SERVICE_CLASSNAME);
//now have the service launcher do its work
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java b/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
index d5869a6..d875d66 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
@@ -165,40 +165,40 @@ public class ClusterDescription implements Cloneable {
* the Slider AM and the application that it deploys
*/
public Map<String, String> options =
- new HashMap<>();
+ new HashMap<String, String>();
/**
* cluster information
* This is only valid when querying the cluster status.
*/
public Map<String, String> info =
- new HashMap<>();
+ new HashMap<String, String>();
/**
* Statistics. This is only relevant when querying the cluster status
*/
public Map<String, Map<String, Integer>> statistics =
- new HashMap<>();
+ new HashMap<String, Map<String, Integer>>();
/**
* Instances: role->count
*/
public Map<String, List<String>> instances =
- new HashMap<>();
+ new HashMap<String, List<String>>();
/**
* Role options,
* role -> option -> value
*/
public Map<String, Map<String, String>> roles =
- new HashMap<>();
+ new HashMap<String, Map<String, String>>();
/**
* List of key-value pairs to add to a client config to set up the client
*/
public Map<String, String> clientProperties =
- new HashMap<>();
+ new HashMap<String, String>();
/**
* Status information
@@ -569,7 +569,7 @@ public class ClusterDescription implements Cloneable {
*/
@JsonIgnore
public Set<String> getRoleNames() {
- return new HashSet<>(roles.keySet());
+ return new HashSet<String>(roles.keySet());
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 1337570..356f99f 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1381,7 +1381,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
verifyBindingsDefined();
SliderUtils.validateClusterName(name);
log.debug("actionFlex({})", name);
- Map<String, Integer> roleInstances = new HashMap<>();
+ Map<String, Integer> roleInstances = new HashMap<String, Integer>();
Map<String, String> roleMap = args.getComponentMap();
for (Map.Entry<String, String> roleEntry : roleMap.entrySet()) {
String key = roleEntry.getKey();
@@ -1934,7 +1934,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
if (uuids.length == 0) {
// short cut on an empty list
- return new LinkedList<>();
+ return new LinkedList<ClusterNode>();
}
return createClusterOperations().listClusterNodes(uuids);
}
@@ -2133,7 +2133,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
+ serviceType
+ " name " + name);
}
- List<ServiceInstanceData> sids = new ArrayList<>(size);
+ List<ServiceInstanceData> sids = new ArrayList<ServiceInstanceData>(size);
for (CuratorServiceInstance<ServiceInstanceData> instance : instances) {
ServiceInstanceData payload = instance.payload;
logInstance(payload, registryArgs.verbose);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index e7b492b..3151a09 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -70,10 +70,10 @@ public class SliderYarnClientImpl extends YarnClientImpl {
*/
public List<ApplicationReport> listInstances(String user)
throws YarnException, IOException {
- Set<String> types = new HashSet<>(1);
+ Set<String> types = new HashSet<String>(1);
types.add(SliderKeys.APP_TYPE);
List<ApplicationReport> allApps = getApplications(types);
- List<ApplicationReport> results = new ArrayList<>();
+ List<ApplicationReport> results = new ArrayList<ApplicationReport>();
for (ApplicationReport report : allApps) {
if (StringUtils.isEmpty(user) || user.equals(report.getUser())) {
results.add(report);
@@ -97,7 +97,7 @@ public class SliderYarnClientImpl extends YarnClientImpl {
YarnException {
List<ApplicationReport> instances = listInstances(user);
List<ApplicationReport> results =
- new ArrayList<>(instances.size());
+ new ArrayList<ApplicationReport>(instances.size());
for (ApplicationReport report : instances) {
if (report.getName().equals(appname)) {
results.add(report);
@@ -240,7 +240,7 @@ public class SliderYarnClientImpl extends YarnClientImpl {
IOException {
List<ApplicationReport> instances = listInstances(user);
List<ApplicationReport> results =
- new ArrayList<>(instances.size());
+ new ArrayList<ApplicationReport>(instances.size());
for (ApplicationReport app : instances) {
if (app.getName().equals(appname)
&& isApplicationLive(app)) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
index 3a8cea4..f4a4569 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
@@ -55,7 +55,7 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
* This is the default parameter
*/
@Parameter
- public final List<String> parameters = new ArrayList<>();
+ public final List<String> parameters = new ArrayList<String>();
/**
* get the name: relies on arg 1 being the cluster name in all operations
@@ -76,7 +76,7 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
*/
@Parameter(names = ARG_DEFINE, arity = 1, description = "Definitions")
- public final List<String> definitions = new ArrayList<>();
+ public final List<String> definitions = new ArrayList<String>();
/**
* System properties
@@ -84,7 +84,7 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
@Parameter(names = {ARG_SYSPROP}, arity = 1,
description = "system properties in the form name value" +
" These are set after the JVM is started.")
- public final List<String> sysprops = new ArrayList<>(0);
+ public final List<String> sysprops = new ArrayList<String>(0);
@Parameter(names = {ARG_MANAGER_SHORT, ARG_MANAGER},
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
index 248e4c2..1f07de3 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
@@ -37,7 +37,7 @@ public class AppAndResouceOptionArgsDelegate extends AbstractArgsDelegate {
@Parameter(names = {ARG_OPTION, ARG_OPTION_SHORT}, arity = 2,
description = ARG_OPTION + "<name> <value>",
splitter = DontSplitArguments.class)
- public List<String> optionTuples = new ArrayList<>(0);
+ public List<String> optionTuples = new ArrayList<String>(0);
/**
@@ -47,7 +47,7 @@ public class AppAndResouceOptionArgsDelegate extends AbstractArgsDelegate {
description = "Component option " + ARG_COMP_OPT +
" <component> <name> <option>",
splitter = DontSplitArguments.class)
- public List<String> compOptTriples = new ArrayList<>(0);
+ public List<String> compOptTriples = new ArrayList<String>(0);
/**
* Resource Options
@@ -55,7 +55,7 @@ public class AppAndResouceOptionArgsDelegate extends AbstractArgsDelegate {
@Parameter(names = {ARG_RESOURCE_OPT, ARG_RESOURCE_OPT_SHORT}, arity = 2,
description = "Resource option "+ ARG_RESOURCE_OPT + "<name> <value>",
splitter = DontSplitArguments.class)
- public List<String> resOptionTuples = new ArrayList<>(0);
+ public List<String> resOptionTuples = new ArrayList<String>(0);
/**
@@ -65,7 +65,7 @@ public class AppAndResouceOptionArgsDelegate extends AbstractArgsDelegate {
description = "Component resource option " + ARG_RES_COMP_OPT +
" <component> <name> <option>",
splitter = DontSplitArguments.class)
- public List<String> resCompOptTriples = new ArrayList<>(0);
+ public List<String> resCompOptTriples = new ArrayList<String>(0);
public Map<String, String> getOptionsMap() throws
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java b/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
index b53ebd3..83754b3 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
@@ -44,7 +44,7 @@ public class ArgOps {
* create a 3-tuple
*/
public static List<Object> triple(String msg, int min, int max) {
- List<Object> l = new ArrayList<>(3);
+ List<Object> l = new ArrayList<Object>(3);
l.add(msg);
l.add(min);
l.add(max);
@@ -100,7 +100,7 @@ public class ArgOps {
public static Map<String, String> convertTupleListToMap(String description,
List<String> list) throws
BadCommandArgumentsException {
- Map<String, String> results = new HashMap<>();
+ Map<String, String> results = new HashMap<String, String>();
if (list != null && !list.isEmpty()) {
int size = list.size();
if (size % 2 != 0) {
@@ -134,7 +134,7 @@ public class ArgOps {
List<String> list) throws
BadCommandArgumentsException {
Map<String, Map<String, String>> results =
- new HashMap<>();
+ new HashMap<String, Map<String, String>>();
if (list != null && !list.isEmpty()) {
int size = list.size();
if (size % 3 != 0) {
@@ -149,7 +149,7 @@ public class ArgOps {
Map<String, String> roleMap = results.get(role);
if (roleMap == null) {
//demand create new role map
- roleMap = new HashMap<>();
+ roleMap = new HashMap<String, String>();
results.put(role, roleMap);
}
if (roleMap.get(key) != null) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
index 83e658d..5d94182 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
@@ -62,11 +62,11 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
*/
- public Map<String, String> definitionMap = new HashMap<>();
+ public Map<String, String> definitionMap = new HashMap<String, String>();
/**
* System properties
*/
- public Map<String, String> syspropsMap = new HashMap<>();
+ public Map<String, String> syspropsMap = new HashMap<String, String>();
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java b/slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
index 0344305..3225133 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
@@ -27,7 +27,7 @@ public class DontSplitArguments implements IParameterSplitter {
@Override
public List<String> split(String value) {
- List<String> list = new ArrayList<>(1);
+ List<String> list = new ArrayList<String>(1);
list.add(value);
return list;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
index 2f0b9ac..f8fd226 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
@@ -81,7 +81,7 @@ public class ConfigHelper {
*/
public static TreeSet<String> sortedConfigKeys(Iterable<Map.Entry<String, String>> conf) {
- TreeSet<String> sorted = new TreeSet<>();
+ TreeSet<String> sorted = new TreeSet<String>();
for (Map.Entry<String, String> entry : conf) {
sorted.add(entry.getKey());
}
@@ -510,7 +510,7 @@ public class ConfigHelper {
* @return hash map
*/
public static Map<String, String> buildMapFromConfiguration(Configuration conf) {
- Map<String, String> map = new HashMap<>();
+ Map<String, String> map = new HashMap<String, String>();
return SliderUtils.mergeEntries(map, conf);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index 9718be9..2381a82 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -442,7 +442,7 @@ public class CoreFileSystem {
//copied to the destination
FileStatus[] fileset = fileSystem.listStatus(srcDir);
Map<String, LocalResource> localResources =
- new HashMap<>(fileset.length);
+ new HashMap<String, LocalResource>(fileset.length);
for (FileStatus entry : fileset) {
LocalResource resource = createAmResource(entry.getPath(),
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 7a80f24..188b7d9 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -440,7 +440,7 @@ public final class SliderUtils {
* @return a stringified list
*/
public static List<String> collectionToStringList(Collection c) {
- List<String> l = new ArrayList<>(c.size());
+ List<String> l = new ArrayList<String>(c.size());
for (Object o : c) {
l.add(o.toString());
}
@@ -839,7 +839,7 @@ public final class SliderUtils {
* @return a possibly empty map of environment variables.
*/
public static Map<String, String> buildEnvMap(Map<String, String> roleOpts) {
- Map<String, String> env = new HashMap<>();
+ Map<String, String> env = new HashMap<String, String>();
if (roleOpts != null) {
for (Map.Entry<String, String> entry: roleOpts.entrySet()) {
String key = entry.getKey();
@@ -866,7 +866,7 @@ public final class SliderUtils {
Map<String, String> optionMap = entry.getValue();
Map<String, String> existingMap = clusterRoleMap.get(key);
if (existingMap == null) {
- existingMap = new HashMap<>();
+ existingMap = new HashMap<String, String>();
}
log.debug("Overwriting role options with command line values {}",
stringifyMap(optionMap));
@@ -1031,7 +1031,7 @@ public final class SliderUtils {
}
public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) {
- Map<String, Map<String, String>> dest = new HashMap<>();
+ Map<String, Map<String, String>> dest = new HashMap<String, Map<String, String>>();
for (Map.Entry<String, Map<String, String>> entry : src.entrySet()) {
dest.put(entry.getKey(), stringMapClone(entry.getValue()));
}
@@ -1039,7 +1039,7 @@ public final class SliderUtils {
}
public static Map<String, String> stringMapClone(Map<String, String> src) {
- Map<String, String> dest = new HashMap<>();
+ Map<String, String> dest = new HashMap<String, String>();
return mergeEntries(dest, src.entrySet());
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
index 2cbfd54..5517771 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
@@ -55,14 +55,14 @@ public final class ConfTree {
/**
* Metadata
*/
- public Map<String, Object> metadata = new HashMap<>(INITAL_MAP_CAPACITY);
+ public Map<String, Object> metadata = new HashMap<String, Object >(INITAL_MAP_CAPACITY);
/**
* Global options
*/
public Map<String, String> global =
- new HashMap<>(INITAL_MAP_CAPACITY);
+ new HashMap<String, String>(INITAL_MAP_CAPACITY);
/**
@@ -70,7 +70,7 @@ public final class ConfTree {
* role -> option -> value
*/
public Map<String, Map<String, String>> components =
- new HashMap<>(INITAL_MAP_CAPACITY);
+ new HashMap<String, Map<String, String>>(INITAL_MAP_CAPACITY);
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
index 1cb537a..bb17547 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
@@ -148,7 +148,7 @@ public class ConfTreeOperations {
return operations;
}
//create a new instances
- Map<String, String> map = new HashMap<>();
+ Map<String, String> map = new HashMap<String, String>();
confTree.components.put(name, map);
return new MapOperations(name, map);
}
@@ -159,7 +159,7 @@ public class ConfTreeOperations {
*/
@JsonIgnore
public Set<String> getComponentNames() {
- return new HashSet<>(confTree.components.keySet());
+ return new HashSet<String>(confTree.components.keySet());
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
index 98a51bc..4b1b44f 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
@@ -47,7 +47,7 @@ public class MapOperations implements Map<String, String> {
public final String name;
public MapOperations() {
- options = new HashMap<>();
+ options = new HashMap<String, String>();
name = "";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
index 8652b4c..644f627 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -58,15 +58,15 @@ public abstract class AbstractLauncher extends Configured {
/**
* Env vars; set up at final launch stage
*/
- protected final Map<String, String> envVars = new HashMap<>();
+ protected final Map<String, String> envVars = new HashMap<String, String>();
protected final MapOperations env = new MapOperations("env", envVars);
protected final ContainerLaunchContext containerLaunchContext =
Records.newRecord(ContainerLaunchContext.class);
- protected final List<String> commands = new ArrayList<>(20);
+ protected final List<String> commands = new ArrayList<String>(20);
protected final Map<String, LocalResource> localResources =
- new HashMap<>();
+ new HashMap<String, LocalResource>();
private final Map<String, ByteBuffer> serviceData =
- new HashMap<>();
+ new HashMap<String, ByteBuffer>();
// security
Credentials credentials = new Credentials();
@@ -269,7 +269,7 @@ public abstract class AbstractLauncher extends Configured {
public String[] dumpEnvToString() {
- List<String> nodeEnv = new ArrayList<>();
+ List<String> nodeEnv = new ArrayList<String>();
for (Map.Entry<String, String> entry : env.entrySet()) {
String envElt = String.format("%s=\"%s\"",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
index 3527149..8e49435 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
@@ -39,7 +39,7 @@ public class ClasspathConstructor {
public static final String CLASS_PATH_SEPARATOR = ApplicationConstants.CLASS_PATH_SEPARATOR;
// public static final String CLASS_PATH_SEPARATOR = File.pathSeparator;
- private final List<String> pathElements = new ArrayList<>();
+ private final List<String> pathElements = new ArrayList<String>();
public ClasspathConstructor() {
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java b/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
index dbaa981..f50bb48 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
@@ -30,7 +30,7 @@ import java.util.List;
* Special support for JVM command buildup.
*/
public class CommandLineBuilder {
- protected final List<String> argumentList = new ArrayList<>(20);
+ protected final List<String> argumentList = new ArrayList<String>(20);
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java b/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
index 846aca1..df12849 100644
--- a/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
@@ -419,7 +419,7 @@ public class ServiceLauncher<S extends Service>
if (argCount <= 1 ) {
return new String[0];
}
- List<String> argsList = new ArrayList<>(argCount);
+ List<String> argsList = new ArrayList<String>(argCount);
ListIterator<String> arguments = args.listIterator();
//skip that first entry
arguments.next();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
index f498916..eac34c0 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
@@ -41,7 +41,7 @@ public class PublishedConfigSet {
RestPaths.PUBLISHED_CONFIGURATION_REGEXP);
public Map<String, PublishedConfiguration> configurations =
- new HashMap<>();
+ new HashMap<String, PublishedConfiguration>();
public PublishedConfigSet() {
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
index 93282cc..f76b93b 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
@@ -46,7 +46,7 @@ public class PublishedConfiguration {
public String updatedTime;
- public Map<String, String> entries = new HashMap<>();
+ public Map<String, String> entries = new HashMap<String, String>();
public PublishedConfiguration() {
}
@@ -112,7 +112,7 @@ public class PublishedConfiguration {
* @param entries entries to put
*/
public void putValues(Iterable<Map.Entry<String, String>> entries) {
- this.entries = new HashMap<>();
+ this.entries = new HashMap<String, String>();
for (Map.Entry<String, String> entry : entries) {
this.entries.put(entry.getKey(), entry.getValue());
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java
index 120966f..a76e28d 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java
@@ -29,7 +29,7 @@ import java.util.Map;
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class UriMap {
- public Map<String, String> uris = new HashMap<>();
+ public Map<String, String> uris = new HashMap<String, String>();
@JsonIgnore
public void put(String key, String value) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java
index 07e4981..bdf70a2 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/RegistryView.java
@@ -31,7 +31,8 @@ public class RegistryView {
/**
* Endpoints
*/
- public Map<String, RegisteredEndpoint> endpoints = new HashMap<>(2);
+ public Map<String, RegisteredEndpoint> endpoints =
+ new HashMap<String, RegisteredEndpoint>(2);
public String configurationsURL;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java b/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java
index 80f0b34..c3c7e63 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/info/ServiceInstanceData.java
@@ -124,13 +124,13 @@ public class ServiceInstanceData implements Serializable {
public Map<String, RegisteredEndpoint> listEndpoints(boolean external) {
RegistryView view = getRegistryView(external);
if (view == null) {
- return new HashMap<>(0);
+ return new HashMap<String, RegisteredEndpoint>(0);
}
Map<String, RegisteredEndpoint> endpoints = view.endpoints;
if (endpoints != null) {
return endpoints;
} else {
- return new HashMap<>(0);
+ return new HashMap<String, RegisteredEndpoint>(0);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java b/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
index 54aeb4f..0d96559 100644
--- a/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
+++ b/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
@@ -49,7 +49,7 @@ public class ZKIntegration implements Watcher {
public static String SVC_SLIDER = "/" + ZK_SERVICES + "/" + ZK_SLIDER;
public static String SVC_SLIDER_USERS = SVC_SLIDER + "/" + ZK_USERS;
- public static final List<String> ZK_USERS_PATH_LIST = new ArrayList<>();
+ public static final List<String> ZK_USERS_PATH_LIST = new ArrayList<String>();
static {
ZK_USERS_PATH_LIST.add(ZK_SERVICES);
ZK_USERS_PATH_LIST.add(ZK_SLIDER);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java b/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
index 8bf25f9..61b1ff0 100644
--- a/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/core/zk/ZookeeperUtils.java
@@ -49,7 +49,7 @@ public class ZookeeperUtils {
if (strings != null) {
len = strings.length;
}
- List<String> tuples = new ArrayList<>(len);
+ List<String> tuples = new ArrayList<String>(len);
if (strings != null) {
for (String s : strings) {
tuples.add(s.trim());
@@ -70,7 +70,7 @@ public class ZookeeperUtils {
if (strings != null) {
len = strings.length;
}
- List<HostAndPort> list = new ArrayList<>(len);
+ List<HostAndPort> list = new ArrayList<HostAndPort>(len);
if (strings != null) {
for (String s : strings) {
list.add(HostAndPort.fromString(s.trim()));
@@ -113,7 +113,7 @@ public class ZookeeperUtils {
* @return
*/
public static String buildQuorum(List<HostAndPort> hostAndPorts, int defaultPort) {
- List<String> entries = new ArrayList<>(hostAndPorts.size());
+ List<String> entries = new ArrayList<String>(hostAndPorts.size());
for (HostAndPort hostAndPort : hostAndPorts) {
entries.add(buildQuorumEntry(hostAndPort, defaultPort));
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index a4a71dd..3fbd3cf 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -291,7 +291,7 @@ public abstract class AbstractProviderService
*/
@Override
public Map<String, String> buildProviderStatus() {
- return new HashMap<>();
+ return new HashMap<String, String>();
}
/*
@@ -300,7 +300,7 @@ public abstract class AbstractProviderService
*/
@Override
public Map<String, String> buildMonitorDetails(ClusterDescription clusterDesc) {
- Map<String, String> details = new LinkedHashMap<>();
+ Map<String, String> details = new LinkedHashMap<String, String>();
// add in all the
buildEndpointDetails(details);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index db9f494..4b8724a 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -419,7 +419,7 @@ public class ProviderUtils implements RoleKeys {
}
log.debug("Found {} entries in {}", ls.length, base);
- List<File> directories = new LinkedList<>();
+ List<File> directories = new LinkedList<File>();
StringBuilder dirs = new StringBuilder();
for (File file : ls) {
log.debug("{}", false);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 72b0a42..8f92ffa 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -216,7 +216,7 @@ public class AgentClientProvider extends AbstractClientProvider
}
Application application = metainfo.getApplication();
- tags = new HashSet<>();
+ tags = new HashSet<String>();
tags.add("Name: " + application.getName());
tags.add("Version: " + application.getVersion());
tags.add("Description: " + SliderUtils.truncate(application.getComment(), 80));
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java
index b839e58..c8b0e1d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java
@@ -94,7 +94,7 @@ class AgentLaunchParameter {
}
if (trackers == null) {
- trackers = new HashMap<>(10);
+ trackers = new HashMap<String, CommandTracker>(10);
}
String componentName = parameters[0];
CommandTracker tracker = new CommandTracker(Arrays.copyOfRange(parameters, 1, parameters.length));
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 43ee7a6..3225066 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -132,9 +132,12 @@ public class AgentProviderService extends AbstractProviderService implements
private Boolean canAnyMasterPublish = null;
private AgentLaunchParameter agentLaunchParameter = null;
- private final Map<String, ComponentInstanceState> componentStatuses = new ConcurrentHashMap<>();
- private final Map<String, Map<String, String>> componentInstanceData = new ConcurrentHashMap<>();
- private final Map<String, Map<String, String>> allocatedPorts = new ConcurrentHashMap<>();
+ private final Map<String, ComponentInstanceState> componentStatuses =
+ new ConcurrentHashMap<String, ComponentInstanceState>();
+ private final Map<String, Map<String, String>> componentInstanceData =
+ new ConcurrentHashMap<String, Map<String, String>>();
+ private final Map<String, Map<String, String>> allocatedPorts =
+ new ConcurrentHashMap<String, Map<String, String>>();
private final Map<String, String> workFolders =
Collections.synchronizedMap(new LinkedHashMap<String, String>(MAX_LOG_ENTRIES, 0.75f, false) {
protected boolean removeEldestEntry(Map.Entry eldest) {
@@ -691,7 +694,7 @@ public class AgentProviderService extends AbstractProviderService implements
* @return the provider status - map of entries to add to the info section
*/
public Map<String, String> buildProviderStatus() {
- Map<String, String> stats = new HashMap<>();
+ Map<String, String> stats = new HashMap<String, String>();
return stats;
}
@@ -707,7 +710,8 @@ public class AgentProviderService extends AbstractProviderService implements
workFolders.put(String.format("%s-%s-%s", hostFqdn, containerId, key), folders.get(key));
}
- publishApplicationInstanceData(LOG_FOLDERS_TAG, LOG_FOLDERS_TAG, (new HashMap<>(this.workFolders)).entrySet());
+ publishApplicationInstanceData(LOG_FOLDERS_TAG, LOG_FOLDERS_TAG,
+ (new HashMap<String, String>(this.workFolders)).entrySet());
}
@@ -750,7 +754,7 @@ public class AgentProviderService extends AbstractProviderService implements
String hostKeyFormat = "${%s_HOST}";
// publish export groups if any
- Map<String, String> replaceTokens = new HashMap<>();
+ Map<String, String> replaceTokens = new HashMap<String, String>();
for (Map.Entry<String, Map<String, ClusterNode>> entry : getRoleClusterNodeMapping().entrySet()) {
String hostName = getHostsList(entry.getValue().values(), true).iterator().next();
replaceTokens.put(String.format(hostKeyFormat, entry.getKey().toUpperCase(Locale.ENGLISH)), hostName);
@@ -768,7 +772,7 @@ public class AgentProviderService extends AbstractProviderService implements
List<Export> exports = exportGroup.getExports();
if (exports != null && !exports.isEmpty()) {
String exportGroupName = exportGroup.getName();
- Map<String, String> map = new HashMap<>();
+ Map<String, String> map = new HashMap<String, String>();
for (Export export : exports) {
String value = export.getValue();
// replace host names
@@ -797,7 +801,7 @@ public class AgentProviderService extends AbstractProviderService implements
String roleName) {
String portVarFormat = "${site.%s}";
String hostNamePattern = "${THIS_HOST}";
- Map<String, String> toPublish = new HashMap<>();
+ Map<String, String> toPublish = new HashMap<String, String>();
Application application = getMetainfo().getApplication();
for (Component component : application.getComponents()) {
@@ -831,7 +835,7 @@ public class AgentProviderService extends AbstractProviderService implements
if (toPublish.size() > 0) {
Map<String, String> perContainerData = null;
if (!getComponentInstanceData().containsKey(containerId)) {
- perContainerData = new ConcurrentHashMap<>();
+ perContainerData = new ConcurrentHashMap<String, String>();
} else {
perContainerData = getComponentInstanceData().get(containerId);
}
@@ -842,7 +846,7 @@ public class AgentProviderService extends AbstractProviderService implements
}
private void publishComponentInstanceData() {
- Map<String, String> dataToPublish = new HashMap<>();
+ Map<String, String> dataToPublish = new HashMap<String, String>();
synchronized (this.componentInstanceData) {
for (String container : getComponentInstanceData().keySet()) {
for (String prop : getComponentInstanceData().get(container).keySet()) {
@@ -979,7 +983,7 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setServiceName(clusterName);
cmd.setComponentName(roleName);
cmd.setRole(roleName);
- Map<String, String> hostLevelParams = new TreeMap<>();
+ Map<String, String> hostLevelParams = new TreeMap<String, String>();
hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME));
hostLevelParams.put(PACKAGE_LIST, getPackageList());
hostLevelParams.put(CONTAINER_ID, containerId);
@@ -1024,7 +1028,7 @@ public class AgentProviderService extends AbstractProviderService implements
}
private Map<String, String> setCommandParameters(String scriptPath, boolean recordConfig) {
- Map<String, String> cmdParams = new TreeMap<>();
+ Map<String, String> cmdParams = new TreeMap<String, String>();
cmdParams.put("service_package_folder",
"${AGENT_WORK_ROOT}/work/app/definition/package");
cmdParams.put("script", scriptPath);
@@ -1057,7 +1061,7 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setClusterName(clusterName);
cmd.setRoleCommand(StatusCommand.STATUS_COMMAND);
- Map<String, String> hostLevelParams = new TreeMap<>();
+ Map<String, String> hostLevelParams = new TreeMap<String, String>();
hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME));
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
@@ -1085,7 +1089,7 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setServiceName(clusterName);
cmd.setClusterName(clusterName);
cmd.setRoleCommand(StatusCommand.GET_CONFIG_COMMAND);
- Map<String, String> hostLevelParams = new TreeMap<>();
+ Map<String, String> hostLevelParams = new TreeMap<String, String>();
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
@@ -1112,7 +1116,7 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setServiceName(clusterName);
cmd.setComponentName(roleName);
cmd.setRole(roleName);
- Map<String, String> hostLevelParams = new TreeMap<>();
+ Map<String, String> hostLevelParams = new TreeMap<String, String>();
hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME));
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
@@ -1149,7 +1153,8 @@ public class AgentProviderService extends AbstractProviderService implements
ConfTreeOperations appConf, String containerId)
throws SliderException {
- Map<String, Map<String, String>> configurations = new TreeMap<>();
+ Map<String, Map<String, String>> configurations =
+ new TreeMap<String, Map<String, String>>();
Map<String, String> tokens = getStandardTokenMap(appConf);
List<String> configs = getApplicationConfigurationTypes(appConf);
@@ -1164,7 +1169,7 @@ public class AgentProviderService extends AbstractProviderService implements
}
private Map<String, String> getStandardTokenMap(ConfTreeOperations appConf) throws SliderException {
- Map<String, String> tokens = new HashMap<>();
+ Map<String, String> tokens = new HashMap<String, String>();
String nnuri = appConf.get("site.fs.defaultFS");
tokens.put("${NN_URI}", nnuri);
tokens.put("${NN_HOST}", URI.create(nnuri).getHost());
@@ -1180,7 +1185,7 @@ public class AgentProviderService extends AbstractProviderService implements
private List<String> getApplicationConfigurationTypes(ConfTreeOperations appConf) {
// for now, reading this from appConf. In the future, modify this method to
// process metainfo.xml
- List<String> configList = new ArrayList<>();
+ List<String> configList = new ArrayList<String>();
configList.add(GLOBAL_CONFIG_TAG);
String configTypes = appConf.get("config_types");
@@ -1190,13 +1195,13 @@ public class AgentProviderService extends AbstractProviderService implements
}
// remove duplicates. mostly worried about 'global' being listed
- return new ArrayList<>(new HashSet<>(configList));
+ return new ArrayList<String>(new HashSet<String>(configList));
}
private void addNamedConfiguration(String configName, Map<String, String> sourceConfig,
Map<String, Map<String, String>> configurations,
Map<String, String> tokens, String containerId) {
- Map<String, String> config = new HashMap<>();
+ Map<String, String> config = new HashMap<String, String>();
if (configName.equals(GLOBAL_CONFIG_TAG)) {
addDefaultGlobalConfig(config);
}
@@ -1235,7 +1240,7 @@ public class AgentProviderService extends AbstractProviderService implements
private Iterable<String> getHostsList(Collection<ClusterNode> values,
boolean hostOnly) {
- List<String> hosts = new ArrayList<>();
+ List<String> hosts = new ArrayList<String>();
for (ClusterNode cn : values) {
hosts.add(hostOnly ? cn.host : cn.host + "/" + cn.name);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
index a23cc38..f4ace5f 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
@@ -37,7 +37,8 @@ public class ComponentCommandOrder {
public static final Logger log =
LoggerFactory.getLogger(ComponentCommandOrder.class);
private static char SPLIT_CHAR = '-';
- Map<Command, Map<String, List<ComponentState>>> dependencies = new HashMap<>();
+ Map<Command, Map<String, List<ComponentState>>> dependencies =
+ new HashMap<Command, Map<String, List<ComponentState>>>();
public ComponentCommandOrder(List<CommandOrder> commandOrders) {
if (commandOrders != null && commandOrders.size() > 0) {
@@ -48,13 +49,13 @@ public class ComponentCommandOrder {
if (requiredStates.size() > 0) {
Map<String, List<ComponentState>> compDep = dependencies.get(componentCmd.command);
if (compDep == null) {
- compDep = new HashMap<>();
+ compDep = new HashMap<String, List<ComponentState>>();
dependencies.put(componentCmd.command, compDep);
}
List<ComponentState> requirements = compDep.get(componentCmd.componentName);
if (requirements == null) {
- requirements = new ArrayList<>();
+ requirements = new ArrayList<ComponentState>();
compDep.put(componentCmd.componentName, requirements);
}
@@ -70,7 +71,7 @@ public class ComponentCommandOrder {
}
String[] componentStates = requires.split(",");
- List<ComponentState> retList = new ArrayList<>();
+ List<ComponentState> retList = new ArrayList<ComponentState>();
for (String componentStateStr : componentStates) {
retList.add(getComponentState(componentStateStr));
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
index 016b4d4..d994e33 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
@@ -34,10 +34,10 @@ public class Application {
ConfigurationDependencies configDependencies;
public Application() {
- exportGroups = new ArrayList<>();
- components = new ArrayList<>();
- osSpecifics = new ArrayList<>();
- commandOrders = new ArrayList<>();
+ exportGroups = new ArrayList<ExportGroup>();
+ components = new ArrayList<Component>();
+ osSpecifics = new ArrayList<OSSpecific>();
+ commandOrders = new ArrayList<CommandOrder>();
}
public String getName() {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
index 9dba87c..f7b77ff 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
@@ -33,7 +33,7 @@ public class Component {
public Component() {
publishConfig = Boolean.FALSE.toString();
- exports = new ArrayList<>();
+ exports = new ArrayList<Export>();
}
public String getName() {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
index c3870b8..ffff76c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
@@ -83,7 +83,7 @@ public class SliderAMClientProvider extends AbstractClientProvider implements
* List of roles
*/
public static final List<ProviderRole> ROLES =
- new ArrayList<>();
+ new ArrayList<ProviderRole>();
public static final int KEY_AM = ROLE_AM_PRIORITY_INDEX;
@@ -147,7 +147,7 @@ public class SliderAMClientProvider extends AbstractClientProvider implements
throws IOException, SliderException {
Map<String, LocalResource> providerResources =
- new HashMap<>();
+ new HashMap<String, LocalResource>();
ProviderUtils.addProviderJar(providerResources,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
index 184c25a..071fc19 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
@@ -100,7 +100,7 @@ public class SliderAMProviderService extends AbstractProviderService implements
@Override
public List<ProviderRole> getRoles() {
- return new ArrayList<>(0);
+ return new ArrayList<ProviderRole>(0);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 7825e24..4a5c9b9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -401,7 +401,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//look at settings of Hadoop Auth, to pick up a problem seen once
checkAndWarnForAuthTokenProblems();
- executorService = new WorkflowExecutorService<>("AmExecutor",
+ executorService = new WorkflowExecutorService<ExecutorService>("AmExecutor",
Executors.newCachedThreadPool(
new ServiceThreadFactory("AmExecutor", true)));
addService(executorService);
@@ -642,7 +642,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//build the role map
List<ProviderRole> providerRoles =
- new ArrayList<>(providerService.getRoles());
+ new ArrayList<ProviderRole>(providerService.getRoles());
providerRoles.addAll(SliderAMClientProvider.ROLES);
// Start up the WebApp and track the URL for it
@@ -664,7 +664,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
.start(webApp);
appMasterTrackingUrl = "http://" + appMasterHostname + ":" + webApp.port();
WebAppService<SliderAMWebApp> webAppService =
- new WebAppService<>("slider", webApp);
+ new WebAppService<SliderAMWebApp>("slider", webApp);
webAppService.init(serviceConf);
webAppService.start();
@@ -744,7 +744,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// build up environment variables that the AM wants set in every container
// irrespective of provider and role.
- envVars = new HashMap<>();
+ envVars = new HashMap<String, String>();
if (hadoop_user_name != null) {
envVars.put(HADOOP_USER_NAME, hadoop_user_name);
}
@@ -1106,8 +1106,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
@Override //AMRMClientAsync
public void onContainersAllocated(List<Container> allocatedContainers) {
LOG_YARN.info("onContainersAllocated({})", allocatedContainers.size());
- List<ContainerAssignment> assignments = new ArrayList<>();
- List<AbstractRMOperation> operations = new ArrayList<>();
+ List<ContainerAssignment> assignments = new ArrayList<ContainerAssignment>();
+ List<AbstractRMOperation> operations = new ArrayList<AbstractRMOperation>();
//app state makes all the decisions
appState.onContainersAllocated(allocatedContainers, assignments, operations);
@@ -1203,7 +1203,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
log.info(
"Scheduling the failure window reset interval to every {} seconds",
seconds);
- RenewingAction<ResetFailureWindow> renew = new RenewingAction<>(
+ RenewingAction<ResetFailureWindow> renew = new RenewingAction<ResetFailureWindow>(
reset, seconds, seconds, TimeUnit.SECONDS, 0);
actionQueues.renewing("failures", renew);
} else {
@@ -1504,7 +1504,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
protected synchronized void launchProviderService(AggregateConf instanceDefinition,
File confDir)
throws IOException, SliderException {
- Map<String, String> env = new HashMap<>();
+ Map<String, String> env = new HashMap<String, String>();
boolean execStarted = providerService.exec(instanceDefinition, confDir, env, this);
if (execStarted) {
providerService.registerServiceListener(this);
@@ -1793,7 +1793,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//turn the args to a list
List<String> argsList = Arrays.asList(args);
//create a new list, as the ArrayList type doesn't push() on an insert
- List<String> extendedArgs = new ArrayList<>(argsList);
+ List<String> extendedArgs = new ArrayList<String>(argsList);
//insert the service name
extendedArgs.add(0, SERVICE_CLASSNAME);
//now have the service launcher do its work
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
index 95edca4..c1e7e6e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
@@ -58,7 +58,7 @@ public class ActionKillContainer extends AsyncAction {
public void execute(SliderAppMaster appMaster,
QueueAccess queueService,
AppState appState) throws Exception {
- List<AbstractRMOperation> opsList = new LinkedList<>();
+ List<AbstractRMOperation> opsList = new LinkedList<AbstractRMOperation>();
ContainerReleaseOperation release = new ContainerReleaseOperation(containerId);
opsList.add(release);
//now apply the operations
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
index ac59096..6ad579d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
@@ -57,19 +57,19 @@ implements Runnable, QueueAccess {
* Immediate actions.
*/
public final BlockingDeque<AsyncAction> actionQueue =
- new LinkedBlockingDeque<>();
+ new LinkedBlockingDeque<AsyncAction>();
/**
* Actions to be scheduled in the future
*/
- public final DelayQueue<AsyncAction> scheduledActions = new DelayQueue<>();
+ public final DelayQueue<AsyncAction> scheduledActions = new DelayQueue<AsyncAction>();
/**
* Map of renewing actions by name ... this is to allow them to
* be cancelled by name
*/
private final Map<String, RenewingAction<? extends AsyncAction>> renewingActions
- = new ConcurrentHashMap<>();
+ = new ConcurrentHashMap<String, RenewingAction<? extends AsyncAction>>();
/**
* Create a queue instance with a single thread executor
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
index cd4b46b..592889c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
@@ -74,7 +74,7 @@ public class ChaosMonkeyService extends AbstractService {
}
public RenewingAction<MonkeyPlayAction> getChaosAction(long time, TimeUnit timeUnit) {
- RenewingAction<MonkeyPlayAction> action = new RenewingAction<>(
+ RenewingAction<MonkeyPlayAction> action = new RenewingAction<MonkeyPlayAction>(
new MonkeyPlayAction(this, 0, TimeUnit.MILLISECONDS),
time,
time,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index d22c716..0d2afcc 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -135,7 +135,7 @@ public class AppState {
* Client properties created via the provider -static for the life
* of the application
*/
- private Map<String, String> clientProperties = new HashMap<>();
+ private Map<String, String> clientProperties = new HashMap<String, String>();
/**
The cluster description published to callers
@@ -146,13 +146,13 @@ public class AppState {
private ClusterDescription clusterSpec = new ClusterDescription();
private final Map<Integer, RoleStatus> roleStatusMap =
- new ConcurrentHashMap<>();
+ new ConcurrentHashMap<Integer, RoleStatus>();
private final Map<String, ProviderRole> roles =
- new ConcurrentHashMap<>();
+ new ConcurrentHashMap<String, ProviderRole>();
private final Map<Integer, ProviderRole> rolePriorityMap =
- new ConcurrentHashMap<>();
+ new ConcurrentHashMap<Integer, ProviderRole>();
/**
* The master node.
@@ -164,7 +164,7 @@ public class AppState {
* been allocated but are not live; it is a superset of the live list
*/
private final ConcurrentMap<ContainerId, RoleInstance> activeContainers =
- new ConcurrentHashMap<>();
+ new ConcurrentHashMap<ContainerId, RoleInstance>();
/**
* Hash map of the containers we have released, but we
@@ -172,7 +172,7 @@ public class AppState {
* containers is treated as a successful outcome
*/
private final ConcurrentMap<ContainerId, Container> containersBeingReleased =
- new ConcurrentHashMap<>();
+ new ConcurrentHashMap<ContainerId, Container>();
/**
* Counter for completed containers ( complete denotes successful or failed )
@@ -207,34 +207,34 @@ public class AppState {
* the node is promoted from here to the containerMap
*/
private final Map<ContainerId, RoleInstance> startingNodes =
- new ConcurrentHashMap<>();
+ new ConcurrentHashMap<ContainerId, RoleInstance>();
/**
* List of completed nodes. This isn't kept in the CD as it gets too
* big for the RPC responses. Indeed, we should think about how deep to get this
*/
private final Map<ContainerId, RoleInstance> completedNodes
- = new ConcurrentHashMap<>();
+ = new ConcurrentHashMap<ContainerId, RoleInstance>();
/**
* Nodes that failed to start.
* Again, kept out of the CD
*/
private final Map<ContainerId, RoleInstance> failedNodes =
- new ConcurrentHashMap<>();
+ new ConcurrentHashMap<ContainerId, RoleInstance>();
/**
* Nodes that came assigned to a role above that
* which were asked for -this appears to happen
*/
- private final Set<ContainerId> surplusNodes = new HashSet<>();
+ private final Set<ContainerId> surplusNodes = new HashSet<ContainerId>();
/**
* Map of containerID -> cluster nodes, for status reports.
* Access to this should be synchronized on the clusterDescription
*/
private final Map<ContainerId, RoleInstance> liveNodes =
- new ConcurrentHashMap<>();
+ new ConcurrentHashMap<ContainerId, RoleInstance>();
private final AtomicInteger completionOfNodeNotInLiveListEvent =
new AtomicInteger();
private final AtomicInteger completionOfUnknownContainerEvent =
@@ -472,7 +472,7 @@ public class AppState {
this.applicationInfo = applicationInfo != null ? applicationInfo
: new HashMap<String, String>();
- clientProperties = new HashMap<>();
+ clientProperties = new HashMap<String, String>();
containerReleaseSelector = releaseSelector;
@@ -794,7 +794,7 @@ public class AppState {
public synchronized List<RoleInstance> cloneActiveContainerList() {
Collection<RoleInstance> values = activeContainers.values();
- return new ArrayList<>(values);
+ return new ArrayList<RoleInstance>(values);
}
@@ -818,7 +818,7 @@ public class AppState {
public synchronized List<RoleInstance> cloneLiveContainerInfoList() {
List<RoleInstance> allRoleInstances;
Collection<RoleInstance> values = getLiveNodes().values();
- allRoleInstances = new ArrayList<>(values);
+ allRoleInstances = new ArrayList<RoleInstance>(values);
return allRoleInstances;
}
@@ -845,7 +845,7 @@ public class AppState {
Collection<String> containerIDs) {
//first, a hashmap of those containerIDs is built up
Set<String> uuidSet = new HashSet<String>(containerIDs);
- List<RoleInstance> nodes = new ArrayList<>(uuidSet.size());
+ List<RoleInstance> nodes = new ArrayList<RoleInstance>(uuidSet.size());
Collection<RoleInstance> clusterNodes = getLiveNodes().values();
for (RoleInstance node : clusterNodes) {
@@ -863,7 +863,7 @@ public class AppState {
* @return a list of nodes, may be empty
*/
public synchronized List<RoleInstance> enumLiveNodesInRole(String role) {
- List<RoleInstance> nodes = new ArrayList<>();
+ List<RoleInstance> nodes = new ArrayList<RoleInstance>();
Collection<RoleInstance> allRoleInstances = getLiveNodes().values();
for (RoleInstance node : allRoleInstances) {
if (role.isEmpty() || role.equals(node.role)) {
@@ -883,7 +883,7 @@ public class AppState {
*/
public synchronized List<RoleInstance> enumNodesWithRoleId(int roleId,
boolean active) {
- List<RoleInstance> nodes = new ArrayList<>();
+ List<RoleInstance> nodes = new ArrayList<RoleInstance>();
Collection<RoleInstance> allRoleInstances;
allRoleInstances = active? activeContainers.values() : liveNodes.values();
for (RoleInstance node : allRoleInstances) {
@@ -900,11 +900,11 @@ public class AppState {
* @return the map of Role name to list of role instances
*/
private synchronized Map<String, List<String>> createRoleToInstanceMap() {
- Map<String, List<String>> map = new HashMap<>();
+ Map<String, List<String>> map = new HashMap<String, List<String>>();
for (RoleInstance node : getLiveNodes().values()) {
List<String> containers = map.get(node.role);
if (containers == null) {
- containers = new ArrayList<>();
+ containers = new ArrayList<String>();
map.put(node.role, containers);
}
containers.add(node.id);
@@ -916,12 +916,12 @@ public class AppState {
* @return the map of Role name to list of Cluster Nodes, ready
*/
private synchronized Map<String, Map<String, ClusterNode>> createRoleToClusterNodeMap() {
- Map<String, Map<String, ClusterNode>> map = new HashMap<>();
+ Map<String, Map<String, ClusterNode>> map = new HashMap<String, Map<String, ClusterNode>>();
for (RoleInstance node : getLiveNodes().values()) {
Map<String, ClusterNode> containers = map.get(node.role);
if (containers == null) {
- containers = new HashMap<>();
+ containers = new HashMap<String, ClusterNode>();
map.put(node.role, containers);
}
Messages.RoleInstanceState pbuf = node.toProtobuf();
@@ -1401,7 +1401,7 @@ public class AppState {
MapOperations infoOps = new MapOperations("info", cd.info);
infoOps.mergeWithoutOverwrite(applicationInfo);
SliderUtils.addBuildInfo(infoOps, "status");
- cd.statistics = new HashMap<>();
+ cd.statistics = new HashMap<String, Map<String, Integer>>();
// build the map of node -> container IDs
Map<String, List<String>> instanceMap = createRoleToInstanceMap();
@@ -1410,7 +1410,7 @@ public class AppState {
//build the map of node -> containers
Map<String, Map<String, ClusterNode>> clusterNodes =
createRoleToClusterNodeMap();
- cd.status = new HashMap<>();
+ cd.status = new HashMap<String, Object>();
cd.status.put(ClusterDescriptionKeys.KEY_CLUSTER_LIVE, clusterNodes);
@@ -1429,7 +1429,7 @@ public class AppState {
cd.statistics.put(rolename, stats);
}
- Map<String, Integer> sliderstats = new HashMap<>();
+ Map<String, Integer> sliderstats = new HashMap<String, Integer>();
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_COMPLETED,
completedContainerCount.get());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED,
@@ -1453,7 +1453,7 @@ public class AppState {
public synchronized List<AbstractRMOperation> reviewRequestAndReleaseNodes()
throws SliderInternalStateException, TriggerClusterTeardownException {
log.debug("in reviewRequestAndReleaseNodes()");
- List<AbstractRMOperation> allOperations = new ArrayList<>();
+ List<AbstractRMOperation> allOperations = new ArrayList<AbstractRMOperation>();
for (RoleStatus roleStatus : getRoleStatusMap().values()) {
if (!roleStatus.getExcludeFromFlexing()) {
List<AbstractRMOperation> operations = reviewOneRole(roleStatus);
@@ -1527,7 +1527,7 @@ public class AppState {
@SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
private List<AbstractRMOperation> reviewOneRole(RoleStatus role)
throws SliderInternalStateException, TriggerClusterTeardownException {
- List<AbstractRMOperation> operations = new ArrayList<>();
+ List<AbstractRMOperation> operations = new ArrayList<AbstractRMOperation>();
int delta;
String details;
int expected;
@@ -1622,7 +1622,7 @@ public class AppState {
*/
public List<AbstractRMOperation> releaseContainer(ContainerId containerId)
throws SliderInternalStateException {
- List<AbstractRMOperation> operations = new ArrayList<>();
+ List<AbstractRMOperation> operations = new ArrayList<AbstractRMOperation>();
List<RoleInstance> activeRoleInstances = cloneActiveContainerList();
for (RoleInstance role : activeRoleInstances) {
if (role.container.getId().equals(containerId)) {
@@ -1666,7 +1666,7 @@ public class AppState {
Collection<RoleInstance> targets = cloneActiveContainerList();
log.info("Releasing {} containers", targets.size());
List<AbstractRMOperation> operations =
- new ArrayList<>(targets.size());
+ new ArrayList<AbstractRMOperation>(targets.size());
for (RoleInstance instance : targets) {
Container possible = instance.container;
ContainerId id = possible.getId();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
index 9d936a1..841dda3 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
@@ -40,7 +40,7 @@ public class MostRecentContainerReleaseSelector implements ContainerReleaseSelec
private static class newerThan implements Comparator<RoleInstance>, Serializable {
private final Comparator<Long> innerComparator =
- new Comparators.ComparatorReverser<>(new Comparators.LongComparator());
+ new Comparators.ComparatorReverser<Long>(new Comparators.LongComparator());
public int compare(RoleInstance o1, RoleInstance o2) {
return innerComparator.compare(o1.createTime, o2.createTime);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
index fad60f2..1ba2282 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
@@ -41,7 +41,7 @@ public class NodeInstance {
*/
public NodeInstance(String hostname, int roles) {
this.hostname = hostname;
- nodeEntries = new ArrayList<>(roles);
+ nodeEntries = new ArrayList<NodeEntry>(roles);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
index 372a0a5..570c194 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
@@ -73,7 +73,7 @@ public class NodeMap extends HashMap<String, NodeInstance> {
* in that role
*/
public List<NodeInstance> listActiveNodes(int role) {
- List<NodeInstance> nodes = new ArrayList<>();
+ List<NodeInstance> nodes = new ArrayList<NodeInstance>();
for (NodeInstance instance : values()) {
if (instance.getActiveRoleInstances(role) > 0) {
nodes.add(instance);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
index fa2c754..d847962 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
@@ -43,7 +43,7 @@ public class OutstandingRequestTracker {
LoggerFactory.getLogger(OutstandingRequestTracker.class);
private Map<OutstandingRequest, OutstandingRequest> requests =
- new HashMap<>();
+ new HashMap<OutstandingRequest, OutstandingRequest>();
/**
* Create a new request for the specific role. If a
@@ -161,7 +161,7 @@ public class OutstandingRequestTracker {
* @return possibly empty list of hostnames
*/
public synchronized List<NodeInstance> cancelOutstandingRequests(int role) {
- List<NodeInstance> hosts = new ArrayList<>();
+ List<NodeInstance> hosts = new ArrayList<NodeInstance>();
Iterator<Map.Entry<OutstandingRequest,OutstandingRequest>> iterator =
requests.entrySet().iterator();
while (iterator.hasNext()) {
@@ -178,6 +178,6 @@ public class OutstandingRequestTracker {
}
public synchronized List<OutstandingRequest> listOutstandingRequests() {
- return new ArrayList<>(requests.values());
+ return new ArrayList<OutstandingRequest>(requests.values());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
index 6caf1a9..a5b36d4 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
@@ -39,7 +39,7 @@ public class ProviderAppState implements StateAccessForProviders {
private final Map<String, PublishedConfigSet> publishedConfigSets =
- new ConcurrentHashMap<>(5);
+ new ConcurrentHashMap<String, PublishedConfigSet>(5);
private static final PatternValidator validator = new PatternValidator(
RestPaths.PUBLISHED_CONFIGURATION_SET_REGEXP);
private String applicationName;
@@ -92,7 +92,7 @@ public class ProviderAppState implements StateAccessForProviders {
public List<String> listConfigSets() {
synchronized (publishedConfigSets) {
- List<String> sets = new ArrayList<>(publishedConfigSets.keySet());
+ List<String> sets = new ArrayList<String>(publishedConfigSets.keySet());
return sets;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/daba289e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 2338103..edcf7ea 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -60,7 +60,7 @@ public class RoleHistory {
LoggerFactory.getLogger(RoleHistory.class);
private final List<ProviderRole> providerRoles;
private final Map<String, ProviderRole> providerRoleMap =
- new HashMap<>();
+ new HashMap<String, ProviderRole>();
private long startTime;
/**
* Time when saved
@@ -107,9 +107,8 @@ public class RoleHistory {
nodemap = new NodeMap(roleSize);
resetAvailableNodeLists();
- resetAvailableNodeLists();
outstandingRequests = new OutstandingRequestTracker();
- Map<Integer, RoleStatus> roleStats = new HashMap<>();
+ Map<Integer, RoleStatus> roleStats = new HashMap<Integer, RoleStatus>();
for (ProviderRole providerRole : providerRoles) {
@@ -141,7 +140,7 @@ public class RoleHistory {
*/
public void addNewProviderRole(ProviderRole providerRole)
throws BadConfigException {
- Map<Integer, RoleStatus> roleStats = new HashMap<>();
+ Map<Integer, RoleStatus> roleStats = new HashMap<Integer, RoleStatus>();
for (ProviderRole role : providerRoles) {
@@ -155,7 +154,7 @@ public class RoleHistory {
* Clear the lists of available nodes
*/
private synchronized void resetAvailableNodeLists() {
- availableNodes = new HashMap<>(roleSize);
+ availableNodes = new HashMap<Integer, LinkedList<NodeInstance>>(roleSize);
}
/**
@@ -433,7 +432,7 @@ public class RoleHistory {
private LinkedList<NodeInstance> getOrCreateNodesForRoleId(int id) {
LinkedList<NodeInstance> instances = availableNodes.get(id);
if (instances == null) {
- instances = new LinkedList<>();
+ instances = new LinkedList<NodeInstance>();
availableNodes.put(id, instances);
}
return instances;
@@ -569,7 +568,7 @@ public class RoleHistory {
List<Container> requested =
new ArrayList<Container>(allocatedContainers.size());
List<Container> unrequested =
- new ArrayList<>(allocatedContainers.size());
+ new ArrayList<Container>(allocatedContainers.size());
outstandingRequests.partitionRequests(this, allocatedContainers, requested, unrequested);
//give the unrequested ones lower priority
@@ -748,7 +747,7 @@ public class RoleHistory {
*/
@VisibleForTesting
public List<NodeInstance> cloneAvailableList(int role) {
- return new LinkedList<>(getOrCreateNodesForRoleId(role));
+ return new LinkedList<NodeInstance>(getOrCreateNodesForRoleId(role));
}
/**
[25/50] [abbrv] git commit: SLIDER-262. Slider agent should provide
process supervision such as auto-restart. Minor bug fix.
Posted by st...@apache.org.
SLIDER-262. Slider agent should provide process supervision such as auto-restart. Minor bug fix.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/18b05450
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/18b05450
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/18b05450
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 18b05450e886db00fdf38acb83d2b419d149493d
Parents: 806ebb2
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Aug 13 12:48:01 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Aug 14 17:57:43 2014 -0700
----------------------------------------------------------------------
.../providers/agent/AgentProviderService.java | 2 +-
.../application/metadata/MetainfoParser.java | 2 +-
.../agent/TestAgentProviderService.java | 19 ++++++++++---------
3 files changed, 12 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/18b05450/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index a40926a..b22ef39 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -1131,7 +1131,7 @@ public class AgentProviderService extends AbstractProviderService implements
@VisibleForTesting
protected void addStartCommand(String roleName, String containerId, HeartBeatResponse response,
- String scriptPath, Boolean isMarkedAutoRestart)
+ String scriptPath, boolean isMarkedAutoRestart)
throws
SliderException {
assert getAmState().isApplicationLive();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/18b05450/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
index 23134b6..ada5b83 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
@@ -39,7 +39,6 @@ public class MetainfoParser {
digester.addBeanPropertySetter("*/application/comment");
digester.addBeanPropertySetter("*/application/version");
digester.addBeanPropertySetter("*/application/exportedConfigs");
- digester.addBeanPropertySetter("*/application/autoStartOnFailure");
digester.addObjectCreate("*/commandOrder", CommandOrder.class);
digester.addBeanPropertySetter("*/commandOrder/command");
@@ -60,6 +59,7 @@ public class MetainfoParser {
digester.addBeanPropertySetter("*/component/publishConfig");
digester.addBeanPropertySetter("*/component/minInstanceCount");
digester.addBeanPropertySetter("*/component/maxInstanceCount");
+ digester.addBeanPropertySetter("*/component/autoStartOnFailure");
digester.addObjectCreate("*/component/export", Export.class);
digester.addBeanPropertySetter("*/component/export/name");
digester.addBeanPropertySetter("*/component/export/value");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/18b05450/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 463799c..1aeedbc 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -63,6 +63,7 @@ import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
+import org.mockito.Matchers;
import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -566,11 +567,11 @@ public class TestAgentProviderService {
String role_hm = "HBASE_MASTER";
String role_hrs = "HBASE_REGIONSERVER";
- AgentProviderService aps = new AgentProviderService();
- AgentProviderService mockAps = Mockito.spy(aps);
+ AgentProviderService aps1 = new AgentProviderService();
+ AgentProviderService mockAps = Mockito.spy(aps1);
doReturn(metainfo).when(mockAps).getMetainfo();
- AgentProviderService mockAps2 = Mockito.spy(aps);
+ AgentProviderService mockAps2 = Mockito.spy(aps1);
doReturn(metainfo2).when(mockAps2).getMetainfo();
Assert.assertTrue(mockAps.isMaster(role_hm));
@@ -644,7 +645,7 @@ public class TestAgentProviderService {
anyString(),
any(HeartBeatResponse.class),
anyString(),
- anyBoolean());
+ Matchers.anyBoolean());
doNothing().when(mockAps).addGetConfigCommand(
anyString(),
anyString(),
@@ -747,7 +748,7 @@ public class TestAgentProviderService {
anyString(),
any(HeartBeatResponse.class),
anyString(),
- anyBoolean());
+ Matchers.anyBoolean());
// RS still does not start
hb = new HeartBeat();
hb.setResponseId(3);
@@ -758,7 +759,7 @@ public class TestAgentProviderService {
anyString(),
any(HeartBeatResponse.class),
anyString(),
- anyBoolean());
+ Matchers.anyBoolean());
// MASTER succeeds install and issues start
hb = new HeartBeat();
@@ -778,7 +779,7 @@ public class TestAgentProviderService {
anyString(),
any(HeartBeatResponse.class),
anyString(),
- anyBoolean());
+ Matchers.anyBoolean());
Map<String, String> allocatedPorts = mockAps.getAllocatedPorts();
Assert.assertTrue(allocatedPorts != null);
Assert.assertTrue(allocatedPorts.size() == 1);
@@ -794,7 +795,7 @@ public class TestAgentProviderService {
anyString(),
any(HeartBeatResponse.class),
anyString(),
- anyBoolean());
+ Matchers.anyBoolean());
// MASTER succeeds start
hb = new HeartBeat();
hb.setResponseId(3);
@@ -819,7 +820,7 @@ public class TestAgentProviderService {
anyString(),
any(HeartBeatResponse.class),
anyString(),
- anyBoolean());
+ Matchers.anyBoolean());
// JDK7
} catch (SliderException he) {
log.warn(he.getMessage());
[23/50] [abbrv] git commit: SLIDER-262. Slider agent should provide
process supervision such as auto-restart
Posted by st...@apache.org.
SLIDER-262. Slider agent should provide process supervision such as auto-restart
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/506eb79e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/506eb79e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/506eb79e
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 506eb79e35ff6187648f4d874c22bb3fe3b18e58
Parents: 004fc8d
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Aug 12 18:38:15 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Aug 14 17:56:52 2014 -0700
----------------------------------------------------------------------
.../src/main/python/agent/ActionQueue.py | 22 +++++++---
slider-agent/src/main/python/agent/Constants.py | 1 +
.../src/main/python/agent/Controller.py | 25 ++++++++++-
.../python/agent/CustomServiceOrchestrator.py | 44 +++++++++++---------
slider-agent/src/main/python/agent/Heartbeat.py | 6 ++-
.../src/test/python/agent/TestActionQueue.py | 5 ++-
.../agent/TestCustomServiceOrchestrator.py | 8 ++--
.../src/test/python/agent/TestHeartbeat.py | 25 +++++++++--
.../slider/common/tools/CoreFileSystem.java | 3 ++
.../providers/agent/AgentProviderService.java | 35 +++++++++++++++-
.../web/rest/agent/HeartBeatResponse.java | 11 +++++
.../agent/TestAgentProviderService.java | 21 ++++++----
12 files changed, 162 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-agent/src/main/python/agent/ActionQueue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/ActionQueue.py b/slider-agent/src/main/python/agent/ActionQueue.py
index d4d8bc2..774d1e6 100644
--- a/slider-agent/src/main/python/agent/ActionQueue.py
+++ b/slider-agent/src/main/python/agent/ActionQueue.py
@@ -49,6 +49,7 @@ class ActionQueue(threading.Thread):
FAILED_STATUS = 'FAILED'
STORE_APPLIED_CONFIG = 'record_config'
+ AUTO_RESTART = 'auto_restart'
def __init__(self, config, controller):
super(ActionQueue, self).__init__()
@@ -120,6 +121,9 @@ class ActionQueue(threading.Thread):
logger.debug(pprint.pformat(command))
taskId = command['taskId']
+
+ reportResult = not command[Constants.AUTO_GENERATED]
+
# Preparing 'IN_PROGRESS' report
in_progress_status = self.commandStatuses.generate_report_template(command)
in_progress_status.update({
@@ -127,12 +131,17 @@ class ActionQueue(threading.Thread):
'tmperr': self.tmpdir + os.sep + 'errors-' + str(taskId) + '.txt',
'structuredOut': self.tmpdir + os.sep + 'structured-out-' + str(
taskId) + '.json',
- 'status': self.IN_PROGRESS_STATUS
+ 'status': self.IN_PROGRESS_STATUS,
+ 'reportResult': reportResult
})
- self.commandStatuses.put_command_status(command, in_progress_status)
+ self.commandStatuses.put_command_status(command, in_progress_status, reportResult)
store_config = False
if ActionQueue.STORE_APPLIED_CONFIG in command['commandParams']:
store_config = 'true' == command['commandParams'][ActionQueue.STORE_APPLIED_CONFIG]
+ store_command = False
+ if ActionQueue.AUTO_RESTART in command['roleParams']:
+ store_command = 'true' == command['roleParams'][ActionQueue.AUTO_RESTART]
+
# running command
commandresult = self.customServiceOrchestrator.runCommand(command,
@@ -141,7 +150,7 @@ class ActionQueue(threading.Thread):
in_progress_status[
'tmperr'],
True,
- store_config)
+ store_config or store_command)
# dumping results
status = self.COMPLETED_STATUS
if commandresult[Constants.EXIT_CODE] != 0:
@@ -152,6 +161,7 @@ class ActionQueue(threading.Thread):
'stderr': commandresult['stderr'],
Constants.EXIT_CODE: commandresult[Constants.EXIT_CODE],
'status': status,
+ 'reportResult': reportResult
})
if roleResult['stdout'] == '':
roleResult['stdout'] = 'None'
@@ -170,7 +180,7 @@ class ActionQueue(threading.Thread):
roleResult['allocatedPorts'] = commandresult[Constants.ALLOCATED_PORTS]
if Constants.FOLDERS in commandresult:
roleResult['folders'] = commandresult[Constants.FOLDERS]
- self.commandStatuses.put_command_status(command, roleResult)
+ self.commandStatuses.put_command_status(command, roleResult, reportResult)
# Store action result to agent response queue
def result(self):
@@ -185,8 +195,8 @@ class ActionQueue(threading.Thread):
service = command['serviceName']
component = command['componentName']
reportResult = True
- if 'auto_generated' in command:
- reportResult = not command['auto_generated']
+ if Constants.AUTO_GENERATED in command:
+ reportResult = not command[Constants.AUTO_GENERATED]
component_status = self.customServiceOrchestrator.requestComponentStatus(command)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-agent/src/main/python/agent/Constants.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Constants.py b/slider-agent/src/main/python/agent/Constants.py
index a120999..2975266 100644
--- a/slider-agent/src/main/python/agent/Constants.py
+++ b/slider-agent/src/main/python/agent/Constants.py
@@ -32,3 +32,4 @@ DO_NOT_HEARTBEAT = "DO_NOT_HEARTBEAT"
DO_NOT_HEARTBEAT_AFTER_ = "DO_NOT_HEARTBEAT_AFTER_"
ZK_QUORUM="zk_quorum"
ZK_REG_PATH="zk_reg_path"
+AUTO_GENERATED="auto_generated"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-agent/src/main/python/agent/Controller.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Controller.py b/slider-agent/src/main/python/agent/Controller.py
index 1a01bf8..77dcafb 100644
--- a/slider-agent/src/main/python/agent/Controller.py
+++ b/slider-agent/src/main/python/agent/Controller.py
@@ -86,6 +86,7 @@ class Controller(threading.Thread):
self.statusCommand = None
self.failureCount = 0
self.heartBeatRetryCount = 0
+ self.autoRestart = False
def __del__(self):
@@ -224,6 +225,9 @@ class Controller(threading.Thread):
serverId = int(response['responseId'])
+ if 'restartEnabled' in response:
+ restartEnabled = 'true' == response['restartEnabled']
+
if 'hasMappedComponents' in response.keys():
self.hasMappedComponents = response['hasMappedComponents'] != False
@@ -257,6 +261,17 @@ class Controller(threading.Thread):
logger.info("No commands sent from the Server.")
pass
+ # Add a start command
+ if self.componentActualState == State.INSTALLED and \
+ self.componentExpectedState == State.STARTED and restartEnabled:
+ stored_command = self.actionQueue.customServiceOrchestrator.stored_command
+ if len(stored_command) > 0:
+ auto_start_command = self.create_start_command(stored_command)
+ if auto_start_command:
+ self.updateStateBasedOnCommand([auto_start_command])
+ self.addToQueue([auto_start_command])
+ pass
+
# Add a status command
if (self.componentActualState != State.STARTING and \
self.componentExpectedState == State.STARTED) and \
@@ -328,6 +343,14 @@ class Controller(threading.Thread):
pass
logger.info("Controller stopped heart-beating.")
+ def create_start_command(self, stored_command):
+ taskId = int(stored_command['taskId'])
+ taskId = taskId + 1
+ stored_command['taskId'] = taskId
+ stored_command['commandId'] = "{0}-1".format(taskId)
+ stored_command[Constants.AUTO_GENERATED] = True
+ pass
+
def updateStateBasedOnCommand(self, commands):
for command in commands:
if command["roleCommand"] == "START":
@@ -388,7 +411,7 @@ class Controller(threading.Thread):
statusCommand["hostLevelParams"] = command["hostLevelParams"]
statusCommand["serviceName"] = command["serviceName"]
statusCommand["taskId"] = "status"
- statusCommand['auto_generated'] = True
+ statusCommand[Constants.AUTO_GENERATED] = True
logger.info("Status command: " + pprint.pformat(statusCommand))
return statusCommand
pass
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 0078c9c..6b2ace5 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -56,7 +56,7 @@ class CustomServiceOrchestrator():
self.status_commands_stderr = os.path.realpath(posixpath.join(self.tmp_dir,
'status_command_stderr.txt'))
self.public_fqdn = hostname.public_hostname()
- self.applied_configs = {}
+ self.stored_command = {}
# Clean up old status command files if any
try:
os.unlink(self.status_commands_stdout)
@@ -68,7 +68,7 @@ class CustomServiceOrchestrator():
def runCommand(self, command, tmpoutfile, tmperrfile,
- override_output_files=True, store_config=False):
+ override_output_files=True, store_command=False):
allocated_port = {}
try:
script_type = command['commandParams']['script_type']
@@ -86,7 +86,7 @@ class CustomServiceOrchestrator():
# We don't support anything else yet
message = "Unknown script type {0}".format(script_type)
raise AgentException(message)
- json_path = self.dump_command_to_json(command, allocated_port, store_config)
+ json_path = self.dump_command_to_json(command, allocated_port, store_command)
py_file_list = [script_tuple]
# filter None values
filtered_py_file_list = [i for i in py_file_list if i]
@@ -154,22 +154,28 @@ class CustomServiceOrchestrator():
return path
def getConfig(self, command):
- if 'commandParams' in command and 'config_type' in command['commandParams']:
- config_type = command['commandParams']['config_type']
- logger.info("Requesting applied config for type {0}".format(config_type))
- if config_type in self.applied_configs:
- return {
- 'configurations': {config_type: self.applied_configs[config_type]}
- }
+ if 'configurations' in self.stored_command:
+ if 'commandParams' in command and 'config_type' in command['commandParams']:
+ config_type = command['commandParams']['config_type']
+ logger.info("Requesting applied config for type {0}".format(config_type))
+ if config_type in self.stored_command['configurations']:
+ return {
+ 'configurations': {config_type: self.stored_command['configurations'][config_type]}
+ }
+ else:
+ return {
+ 'configurations': {}
+ }
+ pass
else:
+ logger.info("Requesting all applied config.")
return {
- 'configurations': {}
+ 'configurations': self.stored_command['configurations']
}
pass
else:
- logger.info("Requesting all applied config.")
return {
- 'configurations': self.applied_configs
+ 'configurations': {}
}
pass
@@ -184,7 +190,7 @@ class CustomServiceOrchestrator():
override_output_files = False
if command['roleCommand'] == "GET_CONFIG":
- return self.getConfig(command)
+ return self.getConfig(command)
else:
res = self.runCommand(command, self.status_commands_stdout,
@@ -198,7 +204,7 @@ class CustomServiceOrchestrator():
return res
pass
- def dump_command_to_json(self, command, allocated_ports, store_config=False):
+ def dump_command_to_json(self, command, allocated_ports, store_command=False):
"""
Converts command to json file and returns file path
"""
@@ -221,7 +227,7 @@ class CustomServiceOrchestrator():
if os.path.isfile(file_path):
os.unlink(file_path)
- self.finalize_command(command, store_config, allocated_ports)
+ self.finalize_command(command, store_command, allocated_ports)
with os.fdopen(os.open(file_path, os.O_WRONLY | os.O_CREAT,
0600), 'w') as f:
@@ -238,7 +244,7 @@ class CustomServiceOrchestrator():
Either a port gets allocated or if not then just set the value to "0"
"""
- def finalize_command(self, command, store_config, allocated_ports):
+ def finalize_command(self, command, store_command, allocated_ports):
component = command['componentName']
allocated_for_this_component_format = "${{{0}.ALLOCATED_PORT}}"
allocated_for_any = ".ALLOCATED_PORT}"
@@ -267,9 +273,9 @@ class CustomServiceOrchestrator():
pass
pass
- if store_config:
+ if store_command:
logger.info("Storing applied config: " + pprint.pformat(command['configurations']))
- self.applied_configs = command['configurations']
+ self.stored_command = command
pass
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-agent/src/main/python/agent/Heartbeat.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Heartbeat.py b/slider-agent/src/main/python/agent/Heartbeat.py
index 4f2207c..aa403d4 100644
--- a/slider-agent/src/main/python/agent/Heartbeat.py
+++ b/slider-agent/src/main/python/agent/Heartbeat.py
@@ -57,7 +57,11 @@ class Heartbeat:
if not self.actionQueue.commandQueue.empty():
commandsInProgress = True
if len(queueResult) != 0:
- heartbeat['reports'] = queueResult['reports']
+ heartbeat['reports'] = []
+ for report in queueResult['reports']:
+ if report['reportResult']:
+ del report['reportResult']
+ heartbeat['reports'].append(report)
if len(heartbeat['reports']) > 0:
# There may be IN_PROGRESS tasks
commandsInProgress = True
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-agent/src/test/python/agent/TestActionQueue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestActionQueue.py b/slider-agent/src/test/python/agent/TestActionQueue.py
index 21cc5dd..f0935a9 100644
--- a/slider-agent/src/test/python/agent/TestActionQueue.py
+++ b/slider-agent/src/test/python/agent/TestActionQueue.py
@@ -61,6 +61,8 @@ class TestActionQueue(TestCase):
'serviceName': u'HBASE',
'configurations': {'global': {}},
'configurationTags': {'global': {'tag': 'v1'}},
+ 'auto_generated': False,
+ 'roleParams': {'auto_restart':'false'},
'commandParams': {'script_type': 'PYTHON',
'script': 'scripts/abc.py',
'command_timeout': '600'}
@@ -365,7 +367,8 @@ class TestActionQueue(TestCase):
'structuredOut': '',
'exitcode': 0,
'allocatedPorts': {},
- 'folders': {'AGENT_LOG_ROOT': tempdir, 'AGENT_WORK_ROOT': tempdir}}
+ 'folders': {'AGENT_LOG_ROOT': tempdir, 'AGENT_WORK_ROOT': tempdir},
+ 'reportResult': True}
self.assertEqual(len(report['reports']), 1)
self.assertEqual(report['reports'][0], expected)
self.assertTrue(os.path.isfile(configname))
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index 48e4df4..6408809 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -38,6 +38,7 @@ from socket import socket
class TestCustomServiceOrchestrator(TestCase):
+
def setUp(self):
# disable stdout
out = StringIO.StringIO()
@@ -415,9 +416,10 @@ class TestCustomServiceOrchestrator(TestCase):
}
ret = orchestrator.runCommand(command, "out.txt", "err.txt", True, True)
+ self.assertEqual.__self__.maxDiff = None
self.assertEqual(ret['exitcode'], 0)
self.assertTrue(run_file_mock.called)
- self.assertEqual(orchestrator.applied_configs, expected)
+ self.assertEqual(orchestrator.stored_command, command)
ret = orchestrator.requestComponentStatus(command_get)
self.assertEqual(ret['configurations'], expected)
@@ -498,7 +500,7 @@ class TestCustomServiceOrchestrator(TestCase):
self.assertEqual(command['configurations']['oozie-site']['ignore_port2'], "[0,0]")
self.assertEqual(command['configurations']['oozie-site']['ignore_port3'], "[0,0,0]")
self.assertEqual(command['configurations']['oozie-site']['ignore_port4'], "${HBASE_RS}{a}{b}{c}")
- self.assertEqual(orchestrator.applied_configs, {})
+ self.assertEqual(orchestrator.stored_command, {})
self.assertEqual(len(allocated_ports), 1)
self.assertTrue('oozie-site.a_port' in allocated_ports)
self.assertEqual(allocated_ports['oozie-site.a_port'], '10023')
@@ -513,7 +515,7 @@ class TestCustomServiceOrchestrator(TestCase):
self.assertEqual(command['configurations']['hbase-site']['log_root'], tempdir + "/log")
self.assertEqual(command['configurations']['hbase-site']['blog_root'], "/b/" + tempdir + "/log")
self.assertEqual(command['configurations']['oozie-site']['b_port'], "0")
- self.assertEqual(orchestrator.applied_configs, command['configurations'])
+ self.assertEqual(orchestrator.stored_command, command)
def test_port_allocation(self):
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-agent/src/test/python/agent/TestHeartbeat.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestHeartbeat.py b/slider-agent/src/test/python/agent/TestHeartbeat.py
index a9da65d..70870ec 100644
--- a/slider-agent/src/test/python/agent/TestHeartbeat.py
+++ b/slider-agent/src/test/python/agent/TestHeartbeat.py
@@ -86,7 +86,8 @@ class TestHeartbeat(TestCase):
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
- 'exitcode': 777},
+ 'exitcode': 777,
+ 'reportResult' : True},
{'status': 'COMPLETED',
'stderr': 'stderr',
@@ -97,7 +98,8 @@ class TestHeartbeat(TestCase):
'role': 'role',
'actionId': 17,
'taskId': 'taskId',
- 'exitcode': 0},
+ 'exitcode': 0,
+ 'reportResult' : True},
{'status': 'FAILED',
'stderr': 'stderr',
@@ -108,7 +110,8 @@ class TestHeartbeat(TestCase):
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
- 'exitcode': 13},
+ 'exitcode': 13,
+ 'reportResult' : True},
{'status': 'COMPLETED',
'stderr': 'stderr',
@@ -120,8 +123,21 @@ class TestHeartbeat(TestCase):
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
- 'exitcode': 0}
+ 'exitcode': 0,
+ 'reportResult' : True},
+ {'status': 'COMPLETED',
+ 'stderr': 'stderr',
+ 'stdout': 'out',
+ 'clusterName': u'cc',
+ 'configurationTags': {'global': {'tag': 'v1'}},
+ 'roleCommand': u'INSTALL',
+ 'serviceName': u'HDFS',
+ 'role': u'DATANODE',
+ 'actionId': '1-1',
+ 'taskId': 3,
+ 'exitcode': 0,
+ 'reportResult' : False}
],
'componentStatus': [
{'status': 'HEALTHY', 'componentName': 'DATANODE', 'reportResult' : True},
@@ -159,6 +175,7 @@ class TestHeartbeat(TestCase):
'stderr': 'stderr'}], 'componentStatus': [
{'status': 'HEALTHY', 'componentName': 'DATANODE'},
{'status': 'UNHEALTHY', 'componentName': 'NAMENODE'}]}
+ self.assertEqual.__self__.maxDiff = None
self.assertEquals(hb, expected)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index a145b8f..def252a 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -50,6 +50,9 @@ import java.io.IOException;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Map;
+import java.util.Enumeration;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
import static org.apache.slider.common.SliderXmlConfKeys.CLUSTER_DIRECTORY_PERMISSIONS;
import static org.apache.slider.common.SliderXmlConfKeys.DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 559c4e7..a40926a 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -495,7 +495,7 @@ public class AgentProviderService extends AbstractProviderService implements
boolean canExecute = commandOrder.canExecute(roleName, command, getComponentStatuses().values());
if (canExecute) {
log.info("Starting {} on {}.", roleName, containerId);
- addStartCommand(roleName, containerId, response, scriptPath);
+ addStartCommand(roleName, containerId, response, scriptPath, isMarkedAutoRestart(roleName));
componentStatus.commandIssued(command);
} else {
log.info("Start of {} on {} delayed as dependencies have not started.", roleName, containerId);
@@ -509,6 +509,13 @@ public class AgentProviderService extends AbstractProviderService implements
addGetConfigCommand(roleName, containerId, response);
}
}
+
+ // if restart is required then signal
+ response.setRestartEnabled(false);
+ if (componentStatus.getState() == State.STARTED
+ && command == Command.NOP && isMarkedAutoRestart(roleName)) {
+ response.setRestartEnabled(true);
+ }
} catch (SliderException e) {
componentStatus.applyCommandResult(CommandResult.FAILED, command);
log.warn("Component instance failed operation.", e);
@@ -936,6 +943,25 @@ public class AgentProviderService extends AbstractProviderService implements
}
/**
+ * Checks if the role is marked auto-restart
+ * @param roleName
+ * @return
+ */
+ protected boolean isMarkedAutoRestart(String roleName) {
+ Application application = getMetainfo().getApplication();
+ if (application == null) {
+ log.error("Malformed app definition: Expect application as the top level element for metainfo.xml");
+ } else {
+ for (Component component : application.getComponents()) {
+ if (component.getName().equals(roleName)) {
+ return component.getRequiresAutoRestart();
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
* Can any master publish config explicitly, if not a random master is used
* @return
*/
@@ -1104,7 +1130,8 @@ public class AgentProviderService extends AbstractProviderService implements
}
@VisibleForTesting
- protected void addStartCommand(String roleName, String containerId, HeartBeatResponse response, String scriptPath)
+ protected void addStartCommand(String roleName, String containerId, HeartBeatResponse response,
+ String scriptPath, Boolean isMarkedAutoRestart)
throws
SliderException {
assert getAmState().isApplicationLive();
@@ -1126,6 +1153,10 @@ public class AgentProviderService extends AbstractProviderService implements
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
+ Map<String, String> roleParams = new TreeMap<>();
+ cmd.setRoleParams(roleParams);
+ cmd.getRoleParams().put("auto_restart", Boolean.toString(isMarkedAutoRestart));
+
cmd.setCommandParams(setCommandParameters(scriptPath, true));
Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java
index ca2db32..0545499 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java
@@ -40,6 +40,7 @@ public class HeartBeatResponse {
RegistrationCommand registrationCommand;
boolean restartAgent = false;
+ boolean restartEnabled = true;
boolean hasMappedComponents = false;
@JsonProperty("responseId")
@@ -92,6 +93,16 @@ public class HeartBeatResponse {
this.restartAgent = restartAgent;
}
+ @JsonProperty("restartEnabled")
+ public boolean getRstartEnabled() {
+ return restartEnabled;
+ }
+
+ @JsonProperty("restartEnabled")
+ public void setRestartEnabled(boolean restartEnabled) {
+ this.restartEnabled = restartEnabled;
+ }
+
@JsonProperty("hasMappedComponents")
public boolean hasMappedComponents() {
return hasMappedComponents;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/506eb79e/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index a68e770..463799c 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -78,6 +78,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import static org.easymock.EasyMock.anyBoolean;
import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.createNiceMock;
import static org.easymock.EasyMock.expect;
@@ -642,7 +643,8 @@ public class TestAgentProviderService {
anyString(),
anyString(),
any(HeartBeatResponse.class),
- anyString());
+ anyString(),
+ anyBoolean());
doNothing().when(mockAps).addGetConfigCommand(
anyString(),
anyString(),
@@ -744,7 +746,8 @@ public class TestAgentProviderService {
Mockito.verify(mockAps, Mockito.times(0)).addStartCommand(anyString(),
anyString(),
any(HeartBeatResponse.class),
- anyString());
+ anyString(),
+ anyBoolean());
// RS still does not start
hb = new HeartBeat();
hb.setResponseId(3);
@@ -754,7 +757,8 @@ public class TestAgentProviderService {
Mockito.verify(mockAps, Mockito.times(0)).addStartCommand(anyString(),
anyString(),
any(HeartBeatResponse.class),
- anyString());
+ anyString(),
+ anyBoolean());
// MASTER succeeds install and issues start
hb = new HeartBeat();
@@ -773,7 +777,8 @@ public class TestAgentProviderService {
Mockito.verify(mockAps, Mockito.times(1)).addStartCommand(anyString(),
anyString(),
any(HeartBeatResponse.class),
- anyString());
+ anyString(),
+ anyBoolean());
Map<String, String> allocatedPorts = mockAps.getAllocatedPorts();
Assert.assertTrue(allocatedPorts != null);
Assert.assertTrue(allocatedPorts.size() == 1);
@@ -788,7 +793,8 @@ public class TestAgentProviderService {
Mockito.verify(mockAps, Mockito.times(1)).addStartCommand(anyString(),
anyString(),
any(HeartBeatResponse.class),
- anyString());
+ anyString(),
+ anyBoolean());
// MASTER succeeds start
hb = new HeartBeat();
hb.setResponseId(3);
@@ -812,7 +818,8 @@ public class TestAgentProviderService {
Mockito.verify(mockAps, Mockito.times(2)).addStartCommand(anyString(),
anyString(),
any(HeartBeatResponse.class),
- anyString());
+ anyString(),
+ anyBoolean());
// JDK7
} catch (SliderException he) {
log.warn(he.getMessage());
@@ -959,7 +966,7 @@ public class TestAgentProviderService {
replay(access);
- mockAps.addStartCommand("HBASE_MASTER", "cid1", hbr, "");
+ mockAps.addStartCommand("HBASE_MASTER", "cid1", hbr, "", Boolean.FALSE);
Assert.assertTrue(hbr.getExecutionCommands().get(0).getConfigurations().containsKey("hbase-site"));
Map<String, String> hbaseSiteConf = hbr.getExecutionCommands().get(0).getConfigurations().get("hbase-site");
Assert.assertTrue(hbaseSiteConf.containsKey("a.port"));
[17/50] [abbrv] git commit: SLIDER-322 revert code constructs to
java6, retaining java7 code commented out as JDK7 for fast switch-back
Posted by st...@apache.org.
SLIDER-322 revert code constructs to java6, retaining java7 code commented out as JDK7 for fast switch-back
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/397585a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/397585a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/397585a0
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 397585a052ca9dacc358003f50fa53b934832b50
Parents: daba289
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 14 17:16:18 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 14 17:16:18 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 94 +++++++++++++++++++-
.../slider/common/tools/ConfigHelper.java | 56 ++++++++----
.../slider/core/persist/JsonSerDeser.java | 27 +++++-
.../PublishedConfigurationOutputter.java | 12 +++
.../server/appmaster/SliderAppMaster.java | 19 +++-
.../server/appmaster/web/SliderAmIpFilter.java | 6 +-
.../agent/TestAgentProviderService.java | 45 ++++++----
.../agent/TestComponentInstanceState.java | 11 +--
.../providers/agent/TestHeartbeatMonitor.java | 4 +-
.../management/TestAMManagementWebServices.java | 9 +-
.../publisher/TestAgentProviderService.java | 2 +-
.../workflow/ProcessCommandFactory.java | 6 +-
.../slider/test/MiniZooKeeperCluster.java | 6 +-
13 files changed, 242 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 356f99f..93f6207 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -206,6 +206,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @return the exit code
* @throws Throwable anything that went wrong
*/
+/* JDK7
+
@Override
public int runService() throws Throwable {
@@ -278,6 +280,66 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return exitCode;
}
+*/
+ @Override
+ public int runService() throws Throwable {
+
+ // choose the action
+ String action = serviceArgs.getAction();
+ int exitCode = EXIT_SUCCESS;
+ String clusterName = serviceArgs.getClusterName();
+ // actions
+ if (ACTION_BUILD.equals(action)) {
+ exitCode = actionBuild(clusterName, serviceArgs.getActionBuildArgs());
+ } else if (ACTION_CREATE.equals(action)) {
+ exitCode = actionCreate(clusterName, serviceArgs.getActionCreateArgs());
+ } else if (ACTION_FREEZE.equals(action)) {
+ exitCode = actionFreeze(clusterName,
+ serviceArgs.getActionFreezeArgs());
+ } else if (ACTION_THAW.equals(action)) {
+ exitCode = actionThaw(clusterName, serviceArgs.getActionThawArgs());
+ } else if (ACTION_DESTROY.equals(action)) {
+ exitCode = actionDestroy(clusterName);
+ } else if (ACTION_EXISTS.equals(action)) {
+ exitCode = actionExists(clusterName,
+ serviceArgs.getActionExistsArgs().live);
+ } else if (ACTION_FLEX.equals(action)) {
+ exitCode = actionFlex(clusterName, serviceArgs.getActionFlexArgs());
+ } else if (ACTION_GETCONF.equals(action)) {
+ exitCode = actionGetConf(clusterName, serviceArgs.getActionGetConfArgs());
+ } else if (ACTION_HELP.equals(action) ||
+ ACTION_USAGE.equals(action)) {
+ log.info(serviceArgs.usage());
+
+ } else if (ACTION_KILL_CONTAINER.equals(action)) {
+ exitCode = actionKillContainer(clusterName,
+ serviceArgs.getActionKillContainerArgs());
+
+ } else if (ACTION_AM_SUICIDE.equals(action)) {
+ exitCode = actionAmSuicide(clusterName,
+ serviceArgs.getActionAMSuicideArgs());
+
+ } else if (ACTION_LIST.equals(action)) {
+ exitCode = actionList(clusterName);
+ } else if (ACTION_REGISTRY.equals(action)) {
+ exitCode = actionRegistry(
+ serviceArgs.getActionRegistryArgs());
+ } else if (ACTION_STATUS.equals(action)) {
+ exitCode = actionStatus(clusterName,
+ serviceArgs.getActionStatusArgs());
+ } else if (ACTION_UPDATE.equals(action)) {
+ exitCode = actionUpdate(clusterName, serviceArgs.getActionUpdateArgs());
+
+ } else if (ACTION_VERSION.equals(action)) {
+
+ exitCode = actionVersion();
+ } else {
+ throw new SliderException(EXIT_UNIMPLEMENTED,
+ "Unimplemented: " + action);
+ }
+
+ return exitCode;
+ }
/**
* Delete the zookeeper node associated with the calling user and the cluster
**/
@@ -331,7 +393,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
client.createPath(zkPath, "", ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
return zkPath;
- } catch (InterruptedException | KeeperException e) {
+
+ //JDK7
+// } catch (InterruptedException | KeeperException e) {
+ } catch (InterruptedException e) {
+ log.warn("Unable to create zk node {}", zkPath, e);
+ } catch ( KeeperException e) {
log.warn("Unable to create zk node {}", zkPath, e);
}
}
@@ -1676,7 +1743,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_FALSE;
}
}
- } catch (YarnException | IOException e) {
+
+// JDK7 } catch (YarnException | IOException e) {
+ } catch (YarnException e) {
+ log.warn("Exception while waiting for the cluster {} to shut down: {}",
+ clustername, e);
+ } catch ( IOException e) {
log.warn("Exception while waiting for the cluster {} to shut down: {}",
clustername, e);
}
@@ -1740,6 +1812,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
try {
String description = "Slider Application Instance " + clustername;
+// JDK7
+/*
switch (format) {
case Arguments.FORMAT_XML:
Configuration siteConf = getSiteConf(status, clustername);
@@ -1753,6 +1827,17 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
default:
throw new BadCommandArgumentsException("Unknown format: " + format);
}
+*/
+ if (Arguments.FORMAT_XML.equals(format)) {
+ Configuration siteConf = getSiteConf(status, clustername);
+ siteConf.writeXml(writer);
+ } else if (Arguments.FORMAT_PROPERTIES.equals(format)) {
+ Properties props = new Properties();
+ props.putAll(status.clientProperties);
+ props.store(writer, description);
+ } else {
+ throw new BadCommandArgumentsException("Unknown format: " + format);
+ }
} finally {
// data is written.
// close the file
@@ -2318,7 +2403,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
try {
maybeStartRegistry();
return registry.instanceIDs(SliderKeys.APP_TYPE);
- } catch (YarnException | IOException e) {
+/// JDK7 } catch (YarnException | IOException e) {
+ } catch (IOException e) {
+ throw e;
+ } catch (YarnException e) {
throw e;
} catch (Exception e) {
throw new IOException(e);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
index f8fd226..b7e1323 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
@@ -182,24 +182,53 @@ public class ConfigHelper {
public Document parseConfiguration(FileSystem fs,
Path path) throws
IOException {
- int len = (int) fs.getLength(path);
- byte[] data = new byte[len];
- try(FSDataInputStream in = fs.open(path)) {
- in.readFully(0, data);
- }
+
+ byte[] data = loadBytes(fs, path);
//this is here to track down a parse issue
//related to configurations
- String s = new String(data, 0, len);
+ String s = new String(data, 0, data.length);
log.debug("XML resource {} is \"{}\"", path, s);
+/* JDK7
try (ByteArrayInputStream in = new ByteArrayInputStream(data)) {
Document document = parseConfigXML(in);
return document;
} catch (ParserConfigurationException | SAXException e) {
throw new IOException(e);
}
+*/
+ ByteArrayInputStream in= null;
+ try {
+ in = new ByteArrayInputStream(data);
+ Document document = parseConfigXML(in);
+ return document;
+ } catch (ParserConfigurationException e) {
+ throw new IOException(e);
+ } catch (SAXException e) {
+ throw new IOException(e);
+ } finally {
+ IOUtils.closeStream(in);
+ }
}
-
+
+ public static byte[] loadBytes(FileSystem fs, Path path) throws IOException {
+ int len = (int) fs.getLength(path);
+ byte[] data = new byte[len];
+ /* JDK7
+ try(FSDataInputStream in = fs.open(path)) {
+ in.readFully(0, data);
+ }
+*/
+ FSDataInputStream in = null;
+ in = fs.open(path);
+ try {
+ in.readFully(0, data);
+ } finally {
+ IOUtils.closeStream(in);
+ }
+ return data;
+ }
+
/**
* Load a configuration from ANY FS path. The normal Configuration
* loader only works with file:// URIs
@@ -209,13 +238,9 @@ public class ConfigHelper {
* @throws IOException
*/
public static Configuration loadConfiguration(FileSystem fs,
- Path path) throws
- IOException {
- int len = (int) fs.getLength(path);
- byte[] data = new byte[len];
- try (FSDataInputStream in = fs.open(path)) {
- in.readFully(0, data);
- }
+ Path path) throws IOException {
+ byte[] data = loadBytes(fs, path);
+
ByteArrayInputStream in2;
in2 = new ByteArrayInputStream(data);
@@ -523,7 +548,8 @@ public class ConfigHelper {
* @param valuesource the source of values
* @return a new configuration where <code>foreach key in keysource, get(key)==valuesource.get(key)</code>
*/
- public static Configuration resolveConfiguration(Iterable<Map.Entry<String, String>> keysource,
+ public static Configuration resolveConfiguration(
+ Iterable<Map.Entry<String, String>> keysource,
Configuration valuesource) {
Configuration result = new Configuration(false);
for (Map.Entry<String, String> entry : keysource) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
index 3512168..ab71683 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
@@ -105,7 +105,8 @@ public class JsonSerDeser<T> {
* @throws IOException IO problems
* @throws JsonMappingException failure to map from the JSON to this class
*/
- public T fromResource(String resource)
+/* JDK7
+ public T fromResource(String resource)
throws IOException, JsonParseException, JsonMappingException {
try(InputStream resStream = this.getClass().getResourceAsStream(resource)) {
if (resStream == null) {
@@ -116,6 +117,30 @@ public class JsonSerDeser<T> {
log.error("Exception while parsing json resource {}: {}", resource, e);
throw e;
}
+ }*/
+
+ /**
+ * Convert from a JSON file
+ * @param resource input file
+ * @return the parsed JSON
+ * @throws IOException IO problems
+ * @throws JsonMappingException failure to map from the JSON to this class
+ */
+ public synchronized T fromResource(String resource)
+ throws IOException, JsonParseException, JsonMappingException {
+ InputStream resStream = null;
+ try {
+ resStream = this.getClass().getResourceAsStream(resource);
+ if (resStream == null) {
+ throw new FileNotFoundException(resource);
+ }
+ return (T) (mapper.readValue(resStream, classType));
+ } catch (IOException e) {
+ log.error("Exception while parsing json resource {}: {}", resource, e);
+ throw e;
+ } finally {
+ IOUtils.closeStream(resStream);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
index 929b8ef..bf812dd 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
@@ -48,12 +48,24 @@ public abstract class PublishedConfigurationOutputter {
* @param dest destination file
* @throws IOException
*/
+/* JDK7
public void save(File dest) throws IOException {
try(FileOutputStream out = new FileOutputStream(dest)) {
save(out);
out.close();
}
}
+*/
+ public void save(File dest) throws IOException {
+ FileOutputStream out = null;
+ try {
+ out = new FileOutputStream(dest);
+ save(out);
+ out.close();
+ } finally {
+ org.apache.hadoop.io.IOUtils.closeStream(out);
+ }
+ }
/**
* Save the content. The default saves the asString() value
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 4a5c9b9..acd6870 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -459,7 +459,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
String action = serviceArgs.getAction();
List<String> actionArgs = serviceArgs.getActionArgs();
int exitCode;
- switch (action) {
+/* JDK7
+ switch (action) {
case SliderActions.ACTION_HELP:
log.info(getName() + serviceArgs.usage());
exitCode = LauncherExitCodes.EXIT_USAGE;
@@ -470,6 +471,15 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
default:
throw new SliderException("Unimplemented: " + action);
}
+ */
+ if (action.equals(SliderActions.ACTION_HELP)) {
+ log.info(getName() + serviceArgs.usage());
+ exitCode = SliderExitCodes.EXIT_USAGE;
+ } else if (action.equals(SliderActions.ACTION_CREATE)) {
+ exitCode = createAndRunCluster(actionArgs.get(0));
+ } else {
+ throw new SliderException("Unimplemented: " + action);
+ }
log.info("Exiting AM; final exit code = {}", exitCode);
return exitCode;
}
@@ -1048,9 +1058,16 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
try {
log.info("Unregistering AM status={} message={}", appStatus, appMessage);
asyncRMClient.unregisterApplicationMaster(appStatus, appMessage, null);
+/* JDK7
} catch (YarnException | IOException e) {
log.info("Failed to unregister application: " + e, e);
}
+*/
+ } catch (IOException e) {
+ log.info("Failed to unregister application: " + e, e);
+ } catch (YarnException e) {
+ log.info("Failed to unregister application: " + e, e);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
index aad443e..4c66876 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAmIpFilter.java
@@ -133,7 +133,11 @@ public class SliderAmIpFilter implements Filter {
principal);
chain.doFilter(requestWrapper, resp);
}
- } catch (IOException | ServletException e) {
+// JKD7 } catch (IOException | ServletException e) {
+ } catch (IOException e) {
+ log.warn("When fetching {}: {}", requestURI, e);
+ throw e;
+ } catch (ServletException e) {
log.warn("When fetching {}: {}", requestURI, e);
throw e;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 22de0d2..ee61129 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -291,7 +291,10 @@ public class TestAgentProviderService {
resourceComponent,
appComponent,
containerTmpDirPath);
- } catch (SliderException | IOException he) {
+ // JDK7
+ } catch (IOException he) {
+ log.warn("{}", he, he);
+ } catch (SliderException he) {
log.warn("{}", he, he);
}
@@ -317,14 +320,14 @@ public class TestAgentProviderService {
public ClusterDescription getClusterStatus() {
ClusterDescription cd = new ClusterDescription();
cd.status = new HashMap<String, Object>();
- Map<String, Map<String, ClusterNode>> roleMap = new HashMap<>();
+ Map<String, Map<String, ClusterNode>> roleMap = new HashMap<String, Map<String, ClusterNode>>();
ClusterNode cn1 = new ClusterNode(new MockContainerId(1));
cn1.host = "FIRST_HOST";
- Map<String, ClusterNode> map1 = new HashMap<>();
+ Map<String, ClusterNode> map1 = new HashMap<String, ClusterNode>();
map1.put("FIRST_CONTAINER", cn1);
ClusterNode cn2 = new ClusterNode(new MockContainerId(2));
cn2.host = "SECOND_HOST";
- Map<String, ClusterNode> map2 = new HashMap<>();
+ Map<String, ClusterNode> map2 = new HashMap<String, ClusterNode>();
map2.put("SECOND_CONTAINER", cn2);
ClusterNode cn3 = new ClusterNode(new MockContainerId(3));
cn3.host = "THIRD_HOST";
@@ -350,7 +353,7 @@ public class TestAgentProviderService {
};
aps.setAmState(appState);
- Map<String, String> tokens = new HashMap<>();
+ Map<String, String> tokens = new HashMap<String, String>();
aps.addRoleRelatedTokens(tokens);
Assert.assertEquals(2, tokens.size());
Assert.assertEquals("FIRST_HOST", tokens.get("${FIRST_ROLE_HOST}"));
@@ -367,7 +370,7 @@ public class TestAgentProviderService {
doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
doReturn(metainfo).when(mockAps).getMetainfo();
- Map<String, String> ports = new HashMap<>();
+ Map<String, String> ports = new HashMap<String, String>();
ports.put("global.listen_port", "10010");
mockAps.processAndPublishComponentSpecificData(ports,
"cid1",
@@ -408,17 +411,18 @@ public class TestAgentProviderService {
status.setClusterName("test");
status.setComponentName("HBASE_MASTER");
status.setRoleCommand("GET_CONFIG");
- Map<String, String> hbaseSite = new HashMap<>();
+ Map<String, String> hbaseSite = new HashMap<String, String>();
hbaseSite.put("hbase.master.info.port", "60012");
hbaseSite.put("c", "d");
- Map<String, Map<String, String>> configs = new HashMap<>();
+ Map<String, Map<String, String>> configs =
+ new HashMap<String, Map<String, String>>();
configs.put("hbase-site", hbaseSite);
configs.put("global", hbaseSite);
status.setConfigs(configs);
- hb.setComponentStatus(new ArrayList<>(Arrays.asList(status)));
+ hb.setComponentStatus(new ArrayList<ComponentStatus>(Arrays.asList(status)));
- Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<>();
- Map<String, ClusterNode> container = new HashMap<>();
+ Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<String, Map<String, ClusterNode>>();
+ Map<String, ClusterNode> container = new HashMap<String, ClusterNode>();
ClusterNode cn1 = new ClusterNode(new MockContainerId(1));
cn1.host = "HOST1";
container.put("cid1", cn1);
@@ -754,7 +758,7 @@ public class TestAgentProviderService {
cr.setRole("HBASE_MASTER");
cr.setRoleCommand("INSTALL");
cr.setStatus("COMPLETED");
- Map<String, String> ap = new HashMap<>();
+ Map<String, String> ap = new HashMap<String, String>();
ap.put("a.port", "10233");
cr.setAllocatedPorts(ap);
hb.setReports(Arrays.asList(cr));
@@ -803,7 +807,10 @@ public class TestAgentProviderService {
anyString(),
any(HeartBeatResponse.class),
anyString());
- } catch (SliderException | IOException he) {
+ // JDK7
+ } catch (SliderException he) {
+ log.warn(he.getMessage());
+ } catch (IOException he) {
log.warn(he.getMessage());
}
@@ -882,8 +889,8 @@ public class TestAgentProviderService {
doReturn("HOST1").when(mockAps).getClusterInfoPropertyValue(anyString());
doReturn(metainfo).when(mockAps).getMetainfo();
- Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<>();
- Map<String, ClusterNode> container = new HashMap<>();
+ Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<String, Map<String, ClusterNode>>();
+ Map<String, ClusterNode> container = new HashMap<String, ClusterNode>();
ClusterNode cn1 = new ClusterNode(new MockContainerId(1));
cn1.host = "HOST1";
container.put("cid1", cn1);
@@ -929,18 +936,18 @@ public class TestAgentProviderService {
doReturn("HOST1").when(mockAps).getClusterInfoPropertyValue(anyString());
- Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<>();
- Map<String, ClusterNode> container = new HashMap<>();
+ Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<String, Map<String, ClusterNode>>();
+ Map<String, ClusterNode> container = new HashMap<String, ClusterNode>();
ClusterNode cn1 = new ClusterNode(new MockContainerId(1));
cn1.host = "HOST1";
container.put("cid1", cn1);
roleClusterNodeMap.put("HBASE_MASTER", container);
doReturn(roleClusterNodeMap).when(mockAps).getRoleClusterNodeMapping();
- Map<String, String> allocatedPorts = new HashMap<>();
+ Map<String, String> allocatedPorts = new HashMap<String, String>();
allocatedPorts.put("hbase-site.a.port", "10023");
allocatedPorts.put("hbase-site.b.port", "10024");
doReturn(allocatedPorts).when(mockAps).getAllocatedPorts();
- Map<String, String> allocatedPorts2 = new HashMap<>();
+ Map<String, String> allocatedPorts2 = new HashMap<String, String>();
allocatedPorts2.put("hbase-site.random.port", "10025");
doReturn(allocatedPorts2).when(mockAps).getAllocatedPorts(anyString());
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentInstanceState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentInstanceState.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentInstanceState.java
index e1f3d02..a723394 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentInstanceState.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentInstanceState.java
@@ -44,7 +44,7 @@ public class TestComponentInstanceState {
}
@Test
- public void testGetNextStateBasedOnResult() {
+ public void testGetNextStateBasedOnResult() throws Exception {
TestCase.assertEquals(State.INSTALLING, State.INSTALLING.getNextState(CommandResult.IN_PROGRESS));
TestCase.assertEquals(State.STARTING, State.STARTING.getNextState(CommandResult.IN_PROGRESS));
expectExceptionOnGetNextForResult(IllegalArgumentException.class, State.INIT, CommandResult.IN_PROGRESS);
@@ -68,7 +68,7 @@ public class TestComponentInstanceState {
}
@Test
- public void testGetNextStateBasedOnCommand() {
+ public void testGetNextStateBasedOnCommand() throws Exception {
for (State state : states) {
TestCase.assertEquals(state, state.getNextState(Command.NOP));
}
@@ -87,7 +87,8 @@ public class TestComponentInstanceState {
expectIllegalArgumentException(State.STARTED, Command.START);
}
- protected void expectIllegalArgumentException(State state, Command command) {
+ protected void expectIllegalArgumentException(State state, Command command) throws
+ Exception {
expectExceptionOnGetNextForCommand(IllegalArgumentException.class,
state, command);
}
@@ -274,7 +275,7 @@ public class TestComponentInstanceState {
}
private <T extends Throwable> void expectExceptionOnGetNextForResult(
- Class<T> expected, State state, CommandResult result) {
+ Class<T> expected, State state, CommandResult result) throws Exception {
try {
state.getNextState(result);
TestCase.fail("Must fail");
@@ -286,7 +287,7 @@ public class TestComponentInstanceState {
}
private <T extends Throwable> void expectExceptionOnGetNextForCommand(
- Class<T> expected, State state, Command command) {
+ Class<T> expected, State state, Command command) throws Exception {
try {
state.getNextState(command);
TestCase.fail("Must fail");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
index 1ace44e..7314b72 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
@@ -56,7 +56,7 @@ public class TestHeartbeatMonitor {
AgentProviderService provider = createNiceMock(AgentProviderService.class);
HeartbeatMonitor hbm = new HeartbeatMonitor(provider, 500);
Assert.assertFalse(hbm.isAlive());
- Map<String, ComponentInstanceState> statuses = new HashMap<>();
+ Map<String, ComponentInstanceState> statuses = new HashMap<String, ComponentInstanceState>();
ContainerId container1 = new MockContainerId(1);
ComponentInstanceState state = new ComponentInstanceState("HBASE_MASTER",
container1, "Aid");
@@ -79,7 +79,7 @@ public class TestHeartbeatMonitor {
long now = 100000;
int wakeupInterval = 2 * 1000;
- Map<String, ComponentInstanceState> statuses = new HashMap<>();
+ Map<String, ComponentInstanceState> statuses = new HashMap<String, ComponentInstanceState>();
ContainerId masterContainer = new MockContainerId(1);
ContainerId slaveContainer = new MockContainerId(2);
ComponentInstanceState masterState = new ComponentInstanceState("HBASE_MASTER",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
index fd66bee..91aa2b0 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
@@ -177,7 +177,14 @@ public class TestAMManagementWebServices extends JerseyTest {
fs,
historyPath,
null, null, new SimpleReleaseSelector());
- } catch (IOException | BadClusterStateException | URISyntaxException | BadConfigException e) {
+// JDK7 } catch (IOException | BadClusterStateException | URISyntaxException | BadConfigException e) {
+ } catch (IOException e) {
+ log.error("{}", e, e);
+ } catch (BadClusterStateException e) {
+ log.error("{}", e, e);
+ } catch (URISyntaxException e) {
+ log.error("{}", e, e);
+ } catch (BadConfigException e) {
log.error("{}", e, e);
}
ProviderAppState providerAppState = new ProviderAppState("undefined",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
index a97e351..eb368e3 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
@@ -46,7 +46,7 @@ public class TestAgentProviderService extends AgentProviderService {
QueueAccess queueAccess,
List<Container> liveContainers) {
super.bind(stateAccessor, reg, queueAccess, liveContainers);
- Map<String,String> dummyProps = new HashMap<>();
+ Map<String,String> dummyProps = new HashMap<String, String>();
dummyProps.put("prop1", "val1");
dummyProps.put("prop2", "val2");
log.info("publishing dummy-site.xml with values {}", dummyProps);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java b/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
index e77eeb3..45fdc86 100644
--- a/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
+++ b/slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
@@ -37,7 +37,7 @@ public class ProcessCommandFactory {
* @return commands
*/
public List<String> ls(File dir) {
- List<String> commands = new ArrayList<>(5);
+ List<String> commands = new ArrayList<String>(5);
commands.add("ls");
commands.add("-1");
commands.add(dir.getAbsolutePath());
@@ -50,7 +50,7 @@ public class ProcessCommandFactory {
* @return commands
*/
public List<String> echo(String text) {
- List<String> commands = new ArrayList<>(5);
+ List<String> commands = new ArrayList<String>(5);
commands.add("echo");
commands.add(text);
return commands;
@@ -72,7 +72,7 @@ public class ProcessCommandFactory {
* @return commands
*/
public List<String> exitFalse() {
- List<String> commands = new ArrayList<>(2);
+ List<String> commands = new ArrayList<String>(2);
commands.add("false");
return commands;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/397585a0/slider-core/src/test/java/org/apache/slider/test/MiniZooKeeperCluster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/test/MiniZooKeeperCluster.java b/slider-core/src/test/java/org/apache/slider/test/MiniZooKeeperCluster.java
index cc2cc9b..d739324 100644
--- a/slider-core/src/test/java/org/apache/slider/test/MiniZooKeeperCluster.java
+++ b/slider-core/src/test/java/org/apache/slider/test/MiniZooKeeperCluster.java
@@ -75,9 +75,9 @@ public class MiniZooKeeperCluster {
this.started = false;
this.configuration = configuration;
activeZKServerIndex = -1;
- zooKeeperServers = new ArrayList<>();
- clientPortList = new ArrayList<>();
- standaloneServerFactoryList = new ArrayList<>();
+ zooKeeperServers = new ArrayList<ZooKeeperServer>();
+ clientPortList = new ArrayList<Integer>();
+ standaloneServerFactoryList = new ArrayList<NIOServerCnxnFactory>();
}
public void setDefaultClientPort(int clientPort) {
[08/50] [abbrv] git commit: SLIDER-202: Chaos Monkey adds a (slow)
minicluster test.
Posted by st...@apache.org.
SLIDER-202: Chaos Monkey adds a (slow) minicluster test.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/53028eaa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/53028eaa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/53028eaa
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 53028eaae347899cef9e695e9ee8b41594c6377b
Parents: 52bd11b
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 13 17:30:49 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 13 17:30:49 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/api/InternalKeys.java | 49 +++++++---
.../apache/slider/common/tools/SliderUtils.java | 2 +-
.../server/appmaster/SliderAppMaster.java | 39 ++++----
.../TestStandaloneAMMonkeyRestart.groovy | 94 ++++++++++++++++++++
.../TestRegionServerFailureThreshold.groovy | 5 +-
5 files changed, 154 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/53028eaa/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
index 4045c0e..ad384e2 100644
--- a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
@@ -84,31 +84,54 @@ public interface InternalKeys {
* Flag to indicate whether or not the chaos monkey is enabled:
* {@value}
*/
- String INTERNAL_CHAOS_MONKEY_ENABLED = "internal.chaos.monkey.enabled";
- boolean DEFAULT_INTERNAL_CHAOS_MONKEY_ENABLED = false;
+ String CHAOS_MONKEY_ENABLED = "internal.chaos.monkey.enabled";
+ boolean DEFAULT_CHAOS_MONKEY_ENABLED = false;
/**
* Rate
*/
- String INTERNAL_CHAOS_MONKEY_RATE = "internal.chaos.monkey.rate";
-
- int DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_DAYS = 0;
- int DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_HOURS = 1;
- int DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_MINUTES = 0;
+ String CHAOS_MONKEY_INTERVAL = "internal.chaos.monkey.interval";
+ String CHAOS_MONKEY_INTERVAL_DAYS = CHAOS_MONKEY_INTERVAL + ".days";
+ String CHAOS_MONKEY_INTERVAL_HOURS = CHAOS_MONKEY_INTERVAL + ".hours";
+ String CHAOS_MONKEY_INTERVAL_MINUTES = CHAOS_MONKEY_INTERVAL + ".minutes";
+ String CHAOS_MONKEY_INTERVAL_SECONDS = CHAOS_MONKEY_INTERVAL + ".seconds";
- String INTERNAL_CHAOS_MONKEY_PROBABILITY =
+ int DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS = 0;
+ int DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS = 0;
+ int DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES = 0;
+
+ /**
+ * Prefix for all chaos monkey probabilities
+ */
+ String CHAOS_MONKEY_PROBABILITY =
"internal.chaos.monkey.probability";
/**
* Probabilies are out of 10000 ; 100==1%
*/
- String INTERNAL_CHAOS_MONKEY_PROBABILITY_AM_FAILURE = INTERNAL_CHAOS_MONKEY_PROBABILITY +".amfailure";
- int DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE = 10;
- String INTERNAL_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE =
- INTERNAL_CHAOS_MONKEY_PROBABILITY + ".containerfailure";
- int DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE = 100;
+ /**
+ * Probability of a monkey check killing the AM: {@value}
+ */
+ String CHAOS_MONKEY_PROBABILITY_AM_FAILURE = CHAOS_MONKEY_PROBABILITY +".amfailure";
+
+ /**
+ * Default probability of a monkey check killing the AM: {@value}
+ */
+ int DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE = 0;
+
+ /**
+ * Probability of a monkey check killing a container: {@value}
+ */
+
+ String CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE =
+ CHAOS_MONKEY_PROBABILITY + ".containerfailure";
+
+ /**
+ * Default probability of a monkey check killing the a container: {@value}
+ */
+ int DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE = 0;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/53028eaa/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index d7f159c..7a80f24 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -1476,7 +1476,7 @@ public final class SliderUtils {
}
is = new ByteArrayInputStream(content);
} else {
- log.info("Size unknown. Reading {}", zipEntry.getName());
+ log.debug("Size unknown. Reading {}", zipEntry.getName());
ByteArrayOutputStream baos = new ByteArrayOutputStream();
while (true) {
int byteRead = zis.read();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/53028eaa/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 1642cc5..7825e24 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -779,17 +779,16 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// now do the registration
registerServiceInstance(clustername, appid);
- sliderAMProvider.start();
+ // chaos monkey
+ maybeStartMonkey();
+ // Start the Slider AM provider
+ sliderAMProvider.start();
-
- // launch the provider; this is expected to trigger a callback that
+ // launch the real provider; this is expected to trigger a callback that
// starts the node review process
launchProviderService(instanceDefinition, confDir);
- // chaos monkey
- maybeStartMonkey();
-
try {
//now block waiting to be told to exit the process
waitForAMCompletionSignal();
@@ -1459,7 +1458,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
rmOperationHandler.execute(operations);
}
-
/**
* Get the RM operations handler for direct scheduling of work.
*/
@@ -1739,42 +1737,45 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
signalAMComplete(exitCode, exception.toString());
}
-
- public boolean maybeStartMonkey() {
+
+ /**
+ * Start the chaos monkey
+ * @return true if it started
+ */
+ private boolean maybeStartMonkey() {
MapOperations internals = getGlobalInternalOptions();
Boolean enabled =
- internals.getOptionBool(InternalKeys.INTERNAL_CHAOS_MONKEY_ENABLED,
- InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_ENABLED);
+ internals.getOptionBool(InternalKeys.CHAOS_MONKEY_ENABLED,
+ InternalKeys.DEFAULT_CHAOS_MONKEY_ENABLED);
if (!enabled) {
log.info("Chaos monkey disabled");
}
long monkeyInterval = internals.getTimeRange(
- InternalKeys.INTERNAL_CHAOS_MONKEY_RATE,
- InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_DAYS,
- InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_HOURS,
- InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_MINUTES,
+ InternalKeys.CHAOS_MONKEY_INTERVAL,
+ InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS,
+ InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS,
+ InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES,
0);
log.info("Adding Chaos Monkey scheduled every {} seconds ({} hours)",
monkeyInterval, monkeyInterval/(60*60));
monkey = new ChaosMonkeyService(metrics, actionQueues);
int amKillProbability = internals.getOptionInt(
- InternalKeys.INTERNAL_CHAOS_MONKEY_PROBABILITY_AM_FAILURE,
+ InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE,
InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE);
if (amKillProbability > 0) {
- log.info("Adding AM killer with probability %f", amKillProbability/100.0);
monkey.addTarget("AM killer",
new ChaosKillAM(actionQueues, -1), amKillProbability
);
}
int containerKillProbability = internals.getOptionInt(
- InternalKeys.INTERNAL_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE,
+ InternalKeys.CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE,
InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE);
if (containerKillProbability > 0) {
monkey.addTarget("Container killer",
new ChaosKillContainer(appState, actionQueues, rmOperationHandler),
- amKillProbability
+ containerKillProbability
);
}
initAndAddService(monkey);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/53028eaa/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
new file mode 100644
index 0000000..162bab0
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMMonkeyRestart.groovy
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.SleepJob
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.api.InternalKeys
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.ActionAMSuicideArgs
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.exceptions.ErrorStrings
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * kill a masterless AM and verify it shuts down. This test
+ * also sets the retry count to 1 to stop recreation attempts
+ */
+@CompileStatic
+@Slf4j
+
+class TestStandaloneAMMonkeyRestart extends AgentMiniClusterTestBase {
+
+
+ @Test
+ public void testStandaloneAMMonkeyRestart() throws Throwable {
+ describe "Run a Standalone AM with the Chaos monkey set to kill it"
+ // patch the configuration for AM restart
+ int threshold = 2;
+ YarnConfiguration conf = getRestartableConfiguration(threshold)
+
+ String clustername = createMiniCluster("", conf, 1, true)
+ ServiceLauncher<SliderClient> launcher =
+ createStandaloneAMWithArgs(clustername,
+ [
+ Arguments.ARG_OPTION, InternalKeys.CHAOS_MONKEY_ENABLED, "true",
+ Arguments.ARG_OPTION, InternalKeys.CHAOS_MONKEY_INTERVAL_SECONDS, "8",
+ Arguments.ARG_OPTION, InternalKeys.CHAOS_MONKEY_PROBABILITY_AM_FAILURE, "75000",
+ ],
+ true,
+ false)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+ ApplicationReport report
+ report = waitForClusterLive(sliderClient, 30000)
+ describe "Waiting for the cluster to fail"
+ sleep(40000)
+ // end of process
+ report = sliderClient.applicationReport
+ log.info(report.diagnostics)
+ assert report.currentApplicationAttemptId.attemptId == threshold
+ assert YarnApplicationState.FAILED == report.yarnApplicationState
+ assert FinalApplicationStatus.FAILED == report.finalApplicationStatus
+ }
+
+ /**
+ * Get a restartable configuration
+ * @param restarts
+ * @return
+ */
+ public YarnConfiguration getRestartableConfiguration(int restarts) {
+ def conf = new YarnConfiguration(configuration)
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, restarts)
+ conf.setInt(SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, restarts)
+ conf
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/53028eaa/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy
index 1322fd3..b3aaa48 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/failures/TestRegionServerFailureThreshold.groovy
@@ -149,8 +149,9 @@ class TestRegionServerFailureThreshold extends HBaseMiniClusterTestBase {
}
}
} catch (BadClusterStateException e) {
- assert e.toString().contains(ErrorStrings.E_APPLICATION_NOT_RUNNING)
- assert e.exitCode == SliderExitCodes.EXIT_BAD_STATE
+ assertExceptionDetails(e,
+ SliderExitCodes.EXIT_BAD_STATE,
+ ErrorStrings.E_APPLICATION_NOT_RUNNING)
}
ApplicationReport report = client.applicationReport
log.info(report.diagnostics)
[35/50] [abbrv] git commit: SLIDER-320 add unregister operation,
rename PublishRegistryDetails. AM queues an unregister on container
completion
Posted by st...@apache.org.
SLIDER-320 add unregister operation, rename PublishRegistryDetails. AM queues an unregister on container completion
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/01c7390e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/01c7390e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/01c7390e
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 01c7390e52fedc9ac32123a9ddd1fc0f2d0f12b7
Parents: 31bfae2
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 15 15:33:26 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 15 15:33:26 2014 +0100
----------------------------------------------------------------------
.../providers/agent/AgentProviderService.java | 4 +-
.../server/appmaster/SliderAppMaster.java | 24 ++++++----
.../actions/PublishRegistryDetails.java | 44 ------------------
.../actions/RegisterComponentInstance.java | 48 ++++++++++++++++++++
.../actions/UnregisterComponentInstance.java | 46 +++++++++++++++++++
5 files changed, 111 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/01c7390e/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 680074d..60c99ac 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -63,7 +63,7 @@ import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.providers.agent.application.metadata.OSPackage;
import org.apache.slider.providers.agent.application.metadata.OSSpecific;
import org.apache.slider.server.appmaster.actions.ProviderReportedContainerLoss;
-import org.apache.slider.server.appmaster.actions.PublishRegistryDetails;
+import org.apache.slider.server.appmaster.actions.RegisterComponentInstance;
import org.apache.slider.server.appmaster.state.ContainerPriority;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
@@ -572,7 +572,7 @@ public class AgentProviderService extends AbstractProviderService implements
// and update registration entries
if (instance != null) {
- queueAccess.put(new PublishRegistryDetails(instance.getId(), 0,
+ queueAccess.put(new RegisterComponentInstance(instance.getId(), 0,
TimeUnit.MILLISECONDS));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/01c7390e/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index c54a268..93adfb2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -98,7 +98,7 @@ import org.apache.slider.providers.SliderProviderFactory;
import org.apache.slider.providers.slideram.SliderAMClientProvider;
import org.apache.slider.providers.slideram.SliderAMProviderService;
import org.apache.slider.server.appmaster.actions.ActionKillContainer;
-import org.apache.slider.server.appmaster.actions.PublishRegistryDetails;
+import org.apache.slider.server.appmaster.actions.RegisterComponentInstance;
import org.apache.slider.server.appmaster.actions.QueueExecutor;
import org.apache.slider.server.appmaster.actions.ActionHalt;
import org.apache.slider.server.appmaster.actions.QueueService;
@@ -106,6 +106,7 @@ import org.apache.slider.server.appmaster.actions.ActionStopSlider;
import org.apache.slider.server.appmaster.actions.AsyncAction;
import org.apache.slider.server.appmaster.actions.RenewingAction;
import org.apache.slider.server.appmaster.actions.ResetFailureWindow;
+import org.apache.slider.server.appmaster.actions.UnregisterComponentInstance;
import org.apache.slider.server.appmaster.monkey.ChaosKillAM;
import org.apache.slider.server.appmaster.monkey.ChaosKillContainer;
import org.apache.slider.server.appmaster.monkey.ChaosMonkeyService;
@@ -910,11 +911,22 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
return false;
}
// this is where component registrations will go
+ log.info("Registering component {}", id);
return true;
}
/**
+ * unregister a component. At the time this message is received,
+ * the component may already been deleted from/never added to
+ * the app state
+ * @param id the component
+ */
+ public void unregisterComponent(ContainerId id) {
+ log.info("Unregistering component {}", id);
+ }
+
+ /**
* looks for a specific case where a token file is provided as an environment
* variable, yet the file is not there.
*
@@ -1178,15 +1190,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
getProviderService().notifyContainerCompleted(containerId);
+ queue(new UnregisterComponentInstance(containerId, 0, TimeUnit.MILLISECONDS));
}
- // ask for more containers if any failed
- // In the case of Slider, we don't expect containers to complete since
- // Slider is a long running application. Keep track of how many containers
- // are completing. If too many complete, abort the application
- // TODO: this needs to be better thought about (and maybe something to
- // better handle in Yarn for long running apps)
-
try {
reviewRequestAndReleaseNodes();
} catch (SliderInternalStateException e) {
@@ -1681,7 +1687,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
nmClientAsync.getContainerStatusAsync(containerId,
cinfo.container.getNodeId());
// push out a registration
- queue(new PublishRegistryDetails(containerId, 0, TimeUnit.MILLISECONDS));
+ queue(new RegisterComponentInstance(containerId, 0, TimeUnit.MILLISECONDS));
} else {
//this is a hypothetical path not seen. We react by warning
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/01c7390e/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java
deleted file mode 100644
index cb6d140..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.state.AppState;
-
-import java.util.concurrent.TimeUnit;
-
-public class PublishRegistryDetails extends AsyncAction {
-
-
- public final ContainerId containerId;
-
- public PublishRegistryDetails(ContainerId containerId, long delay,
- TimeUnit timeUnit) {
- super("PublishRegistryDetails :" + containerId.toString(),
- delay, timeUnit);
- this.containerId = containerId;
- }
-
- @Override
- public void execute(SliderAppMaster appMaster,
- QueueAccess queueService,
- AppState appState) throws Exception {
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/01c7390e/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
new file mode 100644
index 0000000..a8a6fe2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.state.AppState;
+
+import java.util.concurrent.TimeUnit;
+
+public class RegisterComponentInstance extends AsyncAction {
+
+
+ public final ContainerId containerId;
+
+ public RegisterComponentInstance(ContainerId containerId, long delay,
+ TimeUnit timeUnit) {
+ super("RegisterComponentInstance :" + containerId,
+ delay, timeUnit);
+ Preconditions.checkArgument(containerId != null);
+ this.containerId = containerId;
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster,
+ QueueAccess queueService,
+ AppState appState) throws Exception {
+
+ appMaster.registerComponent(containerId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/01c7390e/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java
new file mode 100644
index 0000000..78d9c1c
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/UnregisterComponentInstance.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.state.AppState;
+
+import java.util.concurrent.TimeUnit;
+
+public class UnregisterComponentInstance extends AsyncAction {
+
+
+ public final ContainerId containerId;
+
+ public UnregisterComponentInstance(ContainerId containerId, long delay,
+ TimeUnit timeUnit) {
+ super("UnregisterComponentInstance :" + containerId.toString(),
+ delay, timeUnit);
+ this.containerId = containerId;
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster,
+ QueueAccess queueService,
+ AppState appState) throws Exception {
+ appMaster.unregisterComponent(containerId);
+
+ }
+}
[40/50] [abbrv] git commit: SLIDER-324. Exports should be attributed
to individual master components rather than only one master component (app
changes)
Posted by st...@apache.org.
SLIDER-324. Exports should be attributed to individual master components rather than only one master component (app changes)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/14b54a75
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/14b54a75
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/14b54a75
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 14b54a75bed9f6e74e27b54c096cd1469b023b7c
Parents: 2a25ab9
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sun Aug 17 14:15:16 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sun Aug 17 14:15:16 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/metainfo.xml | 3 ++-
app-packages/hbase/appConfig.json | 1 +
app-packages/hbase/metainfo.xml | 30 ++++++++++++++++++++++--------
app-packages/hbase/pom.xml | 2 +-
4 files changed, 26 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/14b54a75/app-packages/accumulo/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/metainfo.xml b/app-packages/accumulo/metainfo.xml
index 4cf6c79..b1aa9de 100644
--- a/app-packages/accumulo/metainfo.xml
+++ b/app-packages/accumulo/metainfo.xml
@@ -40,7 +40,7 @@
</value>
</export>
<export>
- <name>org.apache.slider.jmx</name>
+ <name>app.jmx</name>
<value>
${site.global.monitor_protocol}://${ACCUMULO_MONITOR_HOST}:${site.accumulo-site.monitor.port.client}/xml
</value>
@@ -85,6 +85,7 @@
<name>ACCUMULO_MONITOR</name>
<category>MASTER</category>
<publishConfig>true</publishConfig>
+ <appExports>QuickLinks-app.jmx,QuickLinks-org.apache.slider.monitor</appExports>
<commandScript>
<script>scripts/accumulo_monitor.py</script>
<scriptType>PYTHON</scriptType>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/14b54a75/app-packages/hbase/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json
index 24d9a84..d00ae6d 100644
--- a/app-packages/hbase/appConfig.json
+++ b/app-packages/hbase/appConfig.json
@@ -25,6 +25,7 @@
"site.global.ganglia_server_id": "Application1",
"site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
"site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}",
+ "site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
"site.hbase-site.hbase.hstore.flush.retries.number": "120",
"site.hbase-site.hbase.client.keyvalue.maxsize": "10485760",
"site.hbase-site.hbase.hstore.compactionThreshold": "3",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/14b54a75/app-packages/hbase/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase/metainfo.xml b/app-packages/hbase/metainfo.xml
index a5dab0c..aae048d 100644
--- a/app-packages/hbase/metainfo.xml
+++ b/app-packages/hbase/metainfo.xml
@@ -53,11 +53,11 @@
<value>http://${HBASE_THRIFT_HOST}:${site.global.hbase_thrift_port}</value>
</export>
<export>
- <name>org.apache.slider.metrics</name>
+ <name>app.metrics</name>
<value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value>
</export>
<export>
- <name>org.apache.slider.ganglia</name>
+ <name>app.ganglia</name>
<value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value>
</export>
</exports>
@@ -81,6 +81,17 @@
<category>MASTER</category>
<minInstanceCount>1</minInstanceCount>
<maxInstanceCount>2</maxInstanceCount>
+ <appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor,QuickLinks-app.metrics,QuickLinks-app.ganglia</appExports>
+ <componentExports>
+ <componentExport>
+ <name>app.jmx</name>
+ <value>${THIS_HOST}:${site.hbase-site.hbase.master.info.port}/jmx</value>
+ </componentExport>
+ <componentExport>
+ <name>app.monitor</name>
+ <value>${THIS_HOST}:${site.hbase-site.hbase.master.info.port}/master-status</value>
+ </componentExport>
+ </componentExports>
<commandScript>
<script>scripts/hbase_master.py</script>
<scriptType>PYTHON</scriptType>
@@ -100,8 +111,9 @@
<component>
<name>HBASE_REST</name>
- <category>SLAVE</category>
- <minInstanceCount>1</minInstanceCount>
+ <category>MASTER</category>
+ <minInstanceCount>0</minInstanceCount>
+ <appExports>QuickLinks-org.apache.slider.hbase.rest</appExports>
<commandScript>
<script>scripts/hbase_rest.py</script>
<scriptType>PYTHON</scriptType>
@@ -110,8 +122,9 @@
<component>
<name>HBASE_THRIFT</name>
- <category>SLAVE</category>
- <minInstanceCount>1</minInstanceCount>
+ <category>MASTER</category>
+ <minInstanceCount>0</minInstanceCount>
+ <appExports>QuickLinks-org.apache.slider.hbase.thrift</appExports>
<commandScript>
<script>scripts/hbase_thrift.py</script>
<scriptType>PYTHON</scriptType>
@@ -120,8 +133,9 @@
<component>
<name>HBASE_THRIFT2</name>
- <category>SLAVE</category>
- <minInstanceCount>1</minInstanceCount>
+ <category>MASTER</category>
+ <minInstanceCount>0</minInstanceCount>
+ <appExports>QuickLinks-org.apache.slider.hbase.thrift2</appExports>
<commandScript>
<script>scripts/hbase_thrift2.py</script>
<scriptType>PYTHON</scriptType>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/14b54a75/app-packages/hbase/pom.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase/pom.xml b/app-packages/hbase/pom.xml
index 4450f8b..7dede6c 100644
--- a/app-packages/hbase/pom.xml
+++ b/app-packages/hbase/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.31.0-incubating-SNAPSHOT</version>
+ <version>0.41.0-incubating-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
[27/50] [abbrv] git commit: SLIDER-325 codahale metrics not being
uploaded to AM classpath
Posted by st...@apache.org.
SLIDER-325 codahale metrics not being uploaded to AM classpath
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/388805a1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/388805a1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/388805a1
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 388805a1db2342973f3dcc6355a4205f01bd7a5a
Parents: 599e972
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 15 13:05:53 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 15 13:05:53 2014 +0100
----------------------------------------------------------------------
.../slideram/SliderAMClientProvider.java | 28 ++++++++++----------
1 file changed, 14 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/388805a1/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
index ffff76c..dc84f02 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
@@ -19,6 +19,7 @@
package org.apache.slider.providers.slideram;
import com.beust.jcommander.JCommander;
+import com.codahale.metrics.MetricRegistry;
import com.google.gson.GsonBuilder;
import org.apache.curator.CuratorZookeeperClient;
import org.apache.curator.framework.CuratorFramework;
@@ -59,8 +60,8 @@ import java.util.Map;
* This keeps aspects of role, cluster validation and Clusterspec setup
* out of the core slider client
*/
-public class SliderAMClientProvider extends AbstractClientProvider implements
- SliderKeys {
+public class SliderAMClientProvider extends AbstractClientProvider
+ implements SliderKeys {
protected static final Logger log =
@@ -113,9 +114,8 @@ public class SliderAMClientProvider extends AbstractClientProvider implements
AggregateConf instanceDefinition,
Path clusterDirPath,
Path generatedConfDirPath,
- boolean secure) throws
- SliderException,
- IOException {
+ boolean secure)
+ throws SliderException, IOException {
super.preflightValidateClusterConfiguration(sliderFileSystem, clustername, configuration, instanceDefinition, clusterDirPath, generatedConfDirPath, secure);
//add a check for the directory being writeable by the current user
@@ -167,7 +167,7 @@ public class SliderAMClientProvider extends AbstractClientProvider implements
CuratorZookeeperClient.class,
ServiceInstance.class,
ServiceNames.class,
-
+ MetricRegistry.class
};
String[] jars =
{
@@ -179,6 +179,7 @@ public class SliderAMClientProvider extends AbstractClientProvider implements
"curator-client.jar",
"curator-x-discovery.jar",
"curator-x-discovery-service.jar",
+ "metrics-core.jar"
};
ProviderUtils.addDependencyJars(providerResources, fileSystem, tempPath,
libdir, jars,
@@ -209,11 +210,11 @@ public class SliderAMClientProvider extends AbstractClientProvider implements
* add them to the command line
*/
public void addJVMOptions(AggregateConf aggregateConf,
- JavaCommandLineBuilder cmdLine) throws
- BadConfigException {
-
+ JavaCommandLineBuilder cmdLine)
+ throws BadConfigException {
+
MapOperations sliderAM =
- aggregateConf.getAppConfOperations().getMandatoryComponent(
+ aggregateConf.getAppConfOperations().getMandatoryComponent(
SliderKeys.COMPONENT_AM);
cmdLine.forceIPv4().headless();
String heap = sliderAM.getOption(RoleKeys.JVM_HEAP,
@@ -227,11 +228,10 @@ public class SliderAMClientProvider extends AbstractClientProvider implements
@Override
- public void prepareInstanceConfiguration(AggregateConf aggregateConf) throws
- SliderException,
- IOException {
+ public void prepareInstanceConfiguration(AggregateConf aggregateConf)
+ throws SliderException, IOException {
mergeTemplates(aggregateConf,
- INTERNAL_JSON, RESOURCES_JSON, APPCONF_JSON
+ INTERNAL_JSON, RESOURCES_JSON, APPCONF_JSON
);
}
}
[41/50] [abbrv] git commit: Merge branch
'feature/SLIDER-324_Refactor_Exports' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-324_Refactor_Exports' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/72daf371
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/72daf371
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/72daf371
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 72daf3713141c6d17dd1e3aad5bb85c2a76c4e96
Parents: 8050466 14b54a7
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sun Aug 17 14:16:04 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sun Aug 17 14:16:04 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/metainfo.xml | 3 +-
app-packages/hbase/appConfig.json | 1 +
app-packages/hbase/metainfo.xml | 30 ++-
app-packages/hbase/pom.xml | 2 +-
app-packages/storm/metainfo.xml | 28 +--
.../providers/agent/AgentProviderService.java | 216 +++++++++++--------
.../agent/application/metadata/Component.java | 21 +-
.../application/metadata/ComponentExport.java | 54 +++++
.../application/metadata/MetainfoParser.java | 9 +-
.../agent/TestAgentProviderService.java | 55 +++--
.../metadata/MetainfoParserTest.java | 59 -----
.../metadata/TestMetainfoParser.java | 63 ++++++
.../agent/application/metadata/metainfo.xml | 74 ++++++-
13 files changed, 411 insertions(+), 204 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/72daf371/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
[26/50] [abbrv] git commit: Merging feature/SLIDER-181_Storm_Package
to develop (jdk6 change)
Posted by st...@apache.org.
Merging feature/SLIDER-181_Storm_Package to develop (jdk6 change)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/599e9721
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/599e9721
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/599e9721
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 599e9721e161feed526a9fe712791af10ce5ca03
Parents: 47822df
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Aug 14 18:07:20 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Aug 14 18:07:20 2014 -0700
----------------------------------------------------------------------
.../org/apache/slider/providers/agent/AgentProviderService.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/599e9721/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 8ce1f18..91bc0a6 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -1180,7 +1180,7 @@ public class AgentProviderService extends AbstractProviderService implements
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
- Map<String, String> roleParams = new TreeMap<>();
+ Map<String, String> roleParams = new TreeMap<String, String>();
cmd.setRoleParams(roleParams);
cmd.getRoleParams().put("auto_restart", Boolean.toString(isMarkedAutoRestart));
[02/50] [abbrv] git commit: SLIDER-202: add codahale metrics
Posted by st...@apache.org.
SLIDER-202: add codahale metrics
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8d4217e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8d4217e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8d4217e7
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 8d4217e7988e561d6e9432f86a76c86adcfaee87
Parents: d38b31a
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 12 20:43:00 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 12 20:43:00 2014 +0100
----------------------------------------------------------------------
pom.xml | 8 ++++++++
slider-core/pom.xml | 46 +++++-----------------------------------------
2 files changed, 13 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8d4217e7/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 253d40f..ed823c4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -143,11 +143,14 @@
<jackson.version>1.9.13</jackson.version>
<jcommander.version>1.30</jcommander.version>
+
<jersey.version>1.9</jersey.version>
<servlet-api.version>2.5</servlet-api.version>
<jsr311-api.version>1.1.1</jsr311-api.version>
+
<junit.version>4.11</junit.version>
<log4j.version>1.2.17</log4j.version>
+ <metrics.version>3.0.1</metrics.version>
<mockito.version>1.8.5</mockito.version>
<!-- ProtocolBuffer version, used to verify the protoc version and -->
@@ -1174,6 +1177,11 @@
<version>${jersey.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
<!-- ======================================================== -->
<!-- Mocking -->
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8d4217e7/slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index a44e3bc..d403034 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -309,48 +309,7 @@
<artifactId>hadoop-minicluster</artifactId>
<scope>test</scope>
</dependency>
-<!--
-
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-server</artifactId>
- <scope>test</scope>
- </dependency>
-
--->
-
-<!--
-
- <dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-core</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-minicluster</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-start</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-trace</artifactId>
- <scope>test</scope>
- </dependency>
--->
<dependency>
<groupId>junit</groupId>
@@ -398,6 +357,11 @@
<artifactId>commons-logging</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ </dependency>
+
<!-- ======================================================== -->
<!-- service registry -->
<!-- ======================================================== -->
[32/50] [abbrv] git commit: SLIDER-320 rename registration action
Posted by st...@apache.org.
SLIDER-320 rename registration action
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/41e0c5be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/41e0c5be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/41e0c5be
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 41e0c5bec76a922479147642f2132cb17d66615d
Parents: 95dcbc9
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 15 14:28:06 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 15 14:28:06 2014 +0100
----------------------------------------------------------------------
.../actions/PublishEndpointDetails.java | 45 --------------------
.../actions/PublishRegistryDetails.java | 45 ++++++++++++++++++++
2 files changed, 45 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41e0c5be/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishEndpointDetails.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishEndpointDetails.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishEndpointDetails.java
deleted file mode 100644
index 67198a2..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishEndpointDetails.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.state.AppState;
-
-import java.util.concurrent.TimeUnit;
-
-public class PublishEndpointDetails extends AsyncAction {
-
-
- public final ContainerId containerId;
-
- public PublishEndpointDetails(ContainerId containerId, long delay,
- TimeUnit timeUnit) {
- super("PublishEndpointDetails :" + containerId.toString(),
- delay, timeUnit);
- this.containerId = containerId;
- }
-
- @Override
- public void execute(SliderAppMaster appMaster,
- QueueAccess queueService,
- AppState appState) throws Exception {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/41e0c5be/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java
new file mode 100644
index 0000000..f96183f
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.state.AppState;
+
+import java.util.concurrent.TimeUnit;
+
+public class PublishRegistryDetails extends AsyncAction {
+
+
+ public final ContainerId containerId;
+
+ public PublishRegistryDetails(ContainerId containerId, long delay,
+ TimeUnit timeUnit) {
+ super("PublishEndpointDetails :" + containerId.toString(),
+ delay, timeUnit);
+ this.containerId = containerId;
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster,
+ QueueAccess queueService,
+ AppState appState) throws Exception {
+
+ }
+}
[09/50] [abbrv] git commit: Merge branch
'feature/SLIDER-202_chaos_monkey' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-202_chaos_monkey' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9b20e47b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9b20e47b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9b20e47b
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 9b20e47b50d6e3f51467c0517f3b0d21242f1f17
Parents: d38b31a 53028ea
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 13 18:10:32 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 13 18:10:32 2014 +0100
----------------------------------------------------------------------
pom.xml | 8 +
slider-core/pom.xml | 46 +-----
.../api/ClusterDescriptionOperations.java | 12 +-
.../org/apache/slider/api/InternalKeys.java | 137 ++++++++++++++++
.../java/org/apache/slider/api/OptionKeys.java | 69 +-------
.../org/apache/slider/api/ResourceKeys.java | 26 +--
.../org/apache/slider/client/SliderClient.java | 7 +-
.../apache/slider/common/tools/SliderUtils.java | 8 +-
.../slider/core/build/InstanceBuilder.java | 3 +-
.../apache/slider/core/conf/MapOperations.java | 54 ++++++-
.../apache/slider/providers/ProviderUtils.java | 9 +-
.../providers/agent/AgentClientProvider.java | 8 +-
.../providers/agent/AgentProviderService.java | 7 +-
.../slideram/SliderAMClientProvider.java | 4 +-
.../server/appmaster/SliderAppMaster.java | 157 ++++++++++++------
.../server/appmaster/actions/ActionHalt.java | 4 +-
.../appmaster/actions/ActionKillContainer.java | 67 ++++++++
.../server/appmaster/actions/QueueService.java | 8 +-
.../server/appmaster/monkey/ChaosEntry.java | 86 ++++++++++
.../server/appmaster/monkey/ChaosKillAM.java | 48 ++++++
.../appmaster/monkey/ChaosKillContainer.java | 75 +++++++++
.../appmaster/monkey/ChaosMonkeyService.java | 86 ++++++++++
.../server/appmaster/monkey/ChaosTarget.java | 24 +++
.../appmaster/monkey/MonkeyPlayAction.java | 48 ++++++
.../slider/server/appmaster/state/AppState.java | 14 +-
.../TestStandaloneAMMonkeyRestart.groovy | 94 +++++++++++
.../standalone/TestStandaloneAgentAM.groovy | 2 +-
.../TestMockAppStateContainerFailure.groovy | 2 +-
.../TestMockAppStateDynamicRoles.groovy | 2 +-
.../TestMockAppStateFlexDynamicRoles.groovy | 2 +-
.../appstate/TestMockAppStateFlexing.groovy | 2 +-
.../TestMockAppStateRMOperations.groovy | 2 +-
.../TestMockAppStateRebuildOnAMRestart.groovy | 2 +-
.../TestMockAppStateRolePlacement.groovy | 2 +-
.../appstate/TestMockAppStateRoleRelease.groovy | 2 +-
.../TestMockContainerResourceAllocations.groovy | 2 +-
.../model/mock/BaseMockAppStateTest.groovy | 8 +-
.../model/mock/MockRMOperationHandler.groovy | 7 +-
.../model/monkey/TestMockMonkey.groovy | 160 +++++++++++++++++++
.../apache/slider/test/SliderTestBase.groovy | 7 +-
.../agent/TestAgentProviderService.java | 9 +-
.../accumulo/AccumuloClientProvider.java | 3 +-
.../accumulo/AccumuloProviderService.java | 9 +-
.../providers/hbase/HBaseClientProvider.java | 3 +-
.../providers/hbase/HBaseProviderService.java | 4 +-
.../TestRegionServerFailureThreshold.groovy | 5 +-
...reezeReconfigureThawLiveRegionService.groovy | 4 +-
47 files changed, 1097 insertions(+), 251 deletions(-)
----------------------------------------------------------------------
[46/50] [abbrv] git commit: SLIDER-306 in sync w/ trunk and JDK6
Posted by st...@apache.org.
SLIDER-306 in sync w/ trunk and JDK6
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f06b37cf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f06b37cf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f06b37cf
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: f06b37cf415ed0a602f38454096b19d74492a3d4
Parents: 5d59454
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 19 11:07:03 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 19 11:07:03 2014 +0100
----------------------------------------------------------------------
.../src/main/java/org/apache/slider/client/SliderClient.java | 2 +-
.../main/java/org/apache/slider/common/tools/SliderUtils.java | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f06b37cf/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index fb06454..d608278 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -321,7 +321,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
serviceArgs.getActionAMSuicideArgs());
} else if (ACTION_LIST.equals(action)) {
- exitCode = actionList(clusterName);
+ exitCode = actionList(clusterName, serviceArgs.getActionListArgs());
} else if (ACTION_REGISTRY.equals(action)) {
exitCode = actionRegistry(
serviceArgs.getActionRegistryArgs());
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f06b37cf/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 6aff38a..3f83855 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -579,8 +579,8 @@ public final class SliderUtils {
if (instances.size() <= 1) {
return;
}
- List<ApplicationReport> nonLiveInstance = new ArrayList<>(instances.size());
- List<ApplicationReport> liveInstance = new ArrayList<>(instances.size());
+ List<ApplicationReport> nonLiveInstance = new ArrayList<ApplicationReport>(instances.size());
+ List<ApplicationReport> liveInstance = new ArrayList<ApplicationReport>(instances.size());
for (ApplicationReport report : instances) {
if (report.getYarnApplicationState() == YarnApplicationState.RUNNING
[07/50] [abbrv] git commit: SLIDER-202 monkey has mock tests and is
set to run in AM if enabled
Posted by st...@apache.org.
SLIDER-202 monkey has mock tests and is set to run in AM if enabled
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/52bd11b5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/52bd11b5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/52bd11b5
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 52bd11b5d4d3a17f231e5343c0bfa6c39492fbb3
Parents: 8ed52a9
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 13 15:06:11 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 13 15:06:11 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/api/ResourceKeys.java | 26 +--
.../apache/slider/common/tools/SliderUtils.java | 6 +-
.../apache/slider/core/conf/MapOperations.java | 54 ++++++-
.../providers/agent/AgentClientProvider.java | 8 +-
.../providers/agent/AgentProviderService.java | 7 +-
.../server/appmaster/SliderAppMaster.java | 88 ++++++----
.../appmaster/actions/ActionKillContainer.java | 24 ++-
.../server/appmaster/actions/QueueService.java | 8 +-
.../server/appmaster/monkey/ChaosEntry.java | 12 +-
.../server/appmaster/monkey/ChaosKillAM.java | 12 +-
.../appmaster/monkey/ChaosKillContainer.java | 25 +--
.../appmaster/monkey/ChaosMonkeyService.java | 23 ++-
.../model/mock/MockRMOperationHandler.groovy | 7 +-
.../model/monkey/TestMockMonkey.groovy | 160 +++++++++++++++++++
.../apache/slider/test/SliderTestBase.groovy | 7 +-
15 files changed, 360 insertions(+), 107 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
index c2b78cc..3d54140 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
@@ -112,34 +112,14 @@ public interface ResourceKeys {
"yarn.container.failure.threshold";
/**
- * Minutes range of the container failure reset window.
- * This is combined with the hour and day values to
- * produce the full window.
+ * prefix for the time of the container failure reset window.
* {@value}
*/
- String CONTAINER_FAILURE_WINDOW_MINUTES =
- "yarn.container.failure.window.minutes";
-
-
- /**
- * Hours range of the container failure reset window
- * This is combined with the minute and day values to
- * produce the full window.
- * {@value}
- */
- String CONTAINER_FAILURE_WINDOW_HOURS =
- "yarn.container.failure.window.hours";
+ String CONTAINER_FAILURE_WINDOW =
+ "yarn.container.failure.window";
- /**
- * Hours range of the container failure reset window
- * This is combined with the hour and minute values to
- * produce the full window.
- * {@value}
- */
- String CONTAINER_FAILURE_WINDOW_DAYS =
- "yarn.container.failure.window.days";
int DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS = 0;
int DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS = 6;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index edbf2b2..d7f159c 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.RoleKeys;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
@@ -1303,8 +1303,8 @@ public final class SliderUtils {
SliderException, IOException {
Path imagePath;
String imagePathOption =
- internalOptions.get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
- String appHomeOption = internalOptions.get(OptionKeys.INTERNAL_APPLICATION_HOME);
+ internalOptions.get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ String appHomeOption = internalOptions.get(InternalKeys.INTERNAL_APPLICATION_HOME);
if (!isUnset(imagePathOption)) {
imagePath = fs.createPathThatMustExist(imagePathOption);
} else {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
index bb57b94..98a51bc 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
@@ -18,6 +18,7 @@
package org.apache.slider.core.conf;
+import com.google.common.base.Preconditions;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadConfigException;
import org.slf4j.Logger;
@@ -61,12 +62,11 @@ public class MapOperations implements Map<String, String> {
this.name = name;
}
-
/**
- * Get a cluster option or value
+ * Get an option value
*
- * @param key
- * @param defVal
+ * @param key key
+ * @param defVal default value
* @return option in map or the default
*/
public String getOption(String key, String defVal) {
@@ -74,14 +74,27 @@ public class MapOperations implements Map<String, String> {
return val != null ? val : defVal;
}
+ /**
+ * Get a boolean option
+ *
+ * @param key option key
+ * @param defVal default value
+ * @return option true if the option equals "true", or the default value
+ * if the option was not defined at all.
+ */
+ public Boolean getOptionBool(String key, boolean defVal) {
+ String val = getOption(key, Boolean.toString(defVal));
+ return Boolean.valueOf(val);
+ }
/**
* Get a cluster option or value
*
- * @param key
+ * @param key option key
* @return the value
* @throws BadConfigException if the option is missing
*/
+
public String getMandatoryOption(String key) throws BadConfigException {
String val = options.get(key);
if (val == null) {
@@ -247,4 +260,35 @@ public class MapOperations implements Map<String, String> {
}
return builder.toString();
}
+
+ /**
+ * Get the time range of a set of keys
+ * @param basekey
+ * @param defDays
+ * @param defHours
+ * @param defMins
+ * @param defSecs
+ * @return
+ */
+ public long getTimeRange(String basekey,
+ int defDays,
+ int defHours,
+ int defMins,
+ int defSecs) {
+ Preconditions.checkArgument(basekey != null);
+ int days = getOptionInt(basekey + ".days", defDays);
+ int hours = getOptionInt(basekey + ".hours", defHours);
+
+ int minutes = getOptionInt(basekey + ".minutes", defMins);
+ int seconds = getOptionInt(basekey + ".seconds", defSecs);
+ // range check
+ Preconditions.checkState(days >= 0 && hours >= 0 && minutes >= 0
+ && seconds >= 0,
+ "Time range for %s has negative time component %s:%s:%s:%s",
+ basekey, days, hours, minutes, seconds);
+
+ // calculate total time, schedule the reset if expected
+ long totalMinutes = days * 24 * 60 + hours * 24 + minutes;
+ return totalMinutes * 60 + seconds;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 2ce16c7..72b0a42 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -21,7 +21,7 @@ package org.apache.slider.providers.agent;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.SliderFileSystem;
@@ -109,7 +109,7 @@ public class AgentClientProvider extends AbstractClientProvider
getGlobalOptions().get(AgentKeys.PACKAGE_PATH);
if (SliderUtils.isUnset(appHome)) {
String agentImage = instanceDefinition.getInternalOperations().
- get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
sliderFileSystem.verifyFileExists(new Path(agentImage));
}
}
@@ -174,12 +174,12 @@ public class AgentClientProvider extends AbstractClientProvider
String appHome = instanceDefinition.getAppConfOperations().
getGlobalOptions().get(AgentKeys.PACKAGE_PATH);
String agentImage = instanceDefinition.getInternalOperations().
- get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
if (SliderUtils.isUnset(appHome) && SliderUtils.isUnset(agentImage)) {
throw new BadConfigException("Either agent package path " +
AgentKeys.PACKAGE_PATH + " or image root " +
- OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH
+ InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH
+ " must be provided.");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 59a6e29..0c57b79 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterDescriptionKeys;
import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.OptionKeys;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.common.SliderKeys;
@@ -62,8 +63,6 @@ import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.providers.agent.application.metadata.OSPackage;
import org.apache.slider.providers.agent.application.metadata.OSSpecific;
import org.apache.slider.server.appmaster.actions.ProviderReportedContainerLoss;
-import org.apache.slider.server.appmaster.state.ContainerPriority;
-import org.apache.slider.server.appmaster.state.RoleStatus;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
@@ -265,7 +264,7 @@ public class AgentProviderService extends AbstractProviderService implements
log.info("PYTHONPATH set to {}", pythonPath);
String agentImage = instanceDefinition.getInternalOperations().
- get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
if (agentImage != null) {
LocalResource agentImageRes = fileSystem.createAmResource(new Path(agentImage), LocalResourceType.ARCHIVE);
launcher.addLocalResource(AgentKeys.AGENT_INSTALL_DIR, agentImageRes);
@@ -1169,7 +1168,7 @@ public class AgentProviderService extends AbstractProviderService implements
tokens.put("${DEFAULT_DATA_DIR}", getAmState()
.getInternalsSnapshot()
.getGlobalOptions()
- .getMandatoryOption(OptionKeys.INTERNAL_DATA_DIR_PATH));
+ .getMandatoryOption(InternalKeys.INTERNAL_DATA_DIR_PATH));
return tokens;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index e6c7aba..1642cc5 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -19,6 +19,7 @@
package org.apache.slider.server.appmaster;
import com.codahale.metrics.MetricRegistry;
+import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.BlockingService;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -1195,31 +1196,19 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
ResetFailureWindow reset = new ResetFailureWindow();
ConfTreeOperations ops = new ConfTreeOperations(resources);
MapOperations globals = ops.getGlobalOptions();
- int days = globals.getOptionInt(ResourceKeys.CONTAINER_FAILURE_WINDOW_DAYS,
- ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS);
- int hours = globals.getOptionInt(
- ResourceKeys.CONTAINER_FAILURE_WINDOW_HOURS,
- ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS);
- int minutes = globals.getOptionInt(
- ResourceKeys.CONTAINER_FAILURE_WINDOW_MINUTES,
- ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES);
- // range check
- if (days < 0 || hours < 0 || minutes < 0) {
- throw new BadConfigException("Failure window contains negative values:"
- + "days=%d hours=%d, minutes=%d",
- days, hours, minutes);
- }
- // calculate total time, schedule the reset if expected
- int totalMinutes = days * 24 * 60 + hours * 24 + minutes;
- if (totalMinutes > 0) {
+ long seconds = globals.getTimeRange(ResourceKeys.CONTAINER_FAILURE_WINDOW,
+ ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_DAYS,
+ ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_HOURS,
+ ResourceKeys.DEFAULT_CONTAINER_FAILURE_WINDOW_MINUTES, 0);
+ if (seconds > 0) {
log.info(
- "Scheduling the failure window reset interval to every {} minutes",
- totalMinutes);
+ "Scheduling the failure window reset interval to every {} seconds",
+ seconds);
RenewingAction<ResetFailureWindow> renew = new RenewingAction<>(
- reset, totalMinutes, totalMinutes, TimeUnit.MINUTES, 0);
+ reset, seconds, seconds, TimeUnit.SECONDS, 0);
actionQueues.renewing("failures", renew);
} else {
- log.warn("Failure window reset interval is not set");
+ log.info("Failure window reset interval is not set");
}
}
@@ -1458,7 +1447,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//throws NoSuchNodeException if it is missing
RoleInstance instance =
appState.getLiveInstanceByContainerID(containerID);
- queue(new ActionKillContainer(instance.getId(), 0, TimeUnit.MILLISECONDS));
+ queue(new ActionKillContainer(instance.getId(), 0, TimeUnit.MILLISECONDS,
+ rmOperationHandler));
Messages.KillContainerResponseProto.Builder builder =
Messages.KillContainerResponseProto.newBuilder();
builder.setSuccess(true);
@@ -1469,6 +1459,15 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
rmOperationHandler.execute(operations);
}
+
+ /**
+ * Get the RM operations handler for direct scheduling of work.
+ */
+ @VisibleForTesting
+ public RMOperationHandler getRmOperationHandler() {
+ return rmOperationHandler;
+ }
+
@Override
public Messages.AMSuicideResponseProto amSuicide(
Messages.AMSuicideRequestProto request)
@@ -1742,18 +1741,45 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
public boolean maybeStartMonkey() {
+ MapOperations internals = getGlobalInternalOptions();
+
+ Boolean enabled =
+ internals.getOptionBool(InternalKeys.INTERNAL_CHAOS_MONKEY_ENABLED,
+ InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_ENABLED);
+ if (!enabled) {
+ log.info("Chaos monkey disabled");
+ }
+
+ long monkeyInterval = internals.getTimeRange(
+ InternalKeys.INTERNAL_CHAOS_MONKEY_RATE,
+ InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_DAYS,
+ InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_HOURS,
+ InternalKeys.DEFAULT_INTERNAL_CHAOS_MONKEY_RATE_MINUTES,
+ 0);
+ log.info("Adding Chaos Monkey scheduled every {} seconds ({} hours)",
+ monkeyInterval, monkeyInterval/(60*60));
monkey = new ChaosMonkeyService(metrics, actionQueues);
- int amKillProbability = 100;
- int containerKillProbability = 200;
- monkey.addTarget("AM killer",
- new ChaosKillAM(this, actionQueues, -1),
- amKillProbability);
- monkey.addTarget("Container killer",
- new ChaosKillContainer(this, actionQueues),
- amKillProbability);
+ int amKillProbability = internals.getOptionInt(
+ InternalKeys.INTERNAL_CHAOS_MONKEY_PROBABILITY_AM_FAILURE,
+ InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE);
+ if (amKillProbability > 0) {
+ log.info("Adding AM killer with probability %f", amKillProbability/100.0);
+ monkey.addTarget("AM killer",
+ new ChaosKillAM(actionQueues, -1), amKillProbability
+ );
+ }
+ int containerKillProbability = internals.getOptionInt(
+ InternalKeys.INTERNAL_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE,
+ InternalKeys.DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE);
+ if (containerKillProbability > 0) {
+ monkey.addTarget("Container killer",
+ new ChaosKillContainer(appState, actionQueues, rmOperationHandler),
+ amKillProbability
+ );
+ }
initAndAddService(monkey);
// and schedule it
- schedule(monkey.getChaosAction(60, TimeUnit.SECONDS));
+ schedule(monkey.getChaosAction(monkeyInterval, TimeUnit.SECONDS));
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
index 7668bd5..95edca4 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
@@ -18,10 +18,12 @@
package org.apache.slider.server.appmaster.actions;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
+import org.apache.slider.server.appmaster.operations.RMOperationHandler;
import org.apache.slider.server.appmaster.state.AppState;
import java.util.LinkedList;
@@ -31,13 +33,27 @@ import java.util.concurrent.TimeUnit;
public class ActionKillContainer extends AsyncAction {
private final ContainerId containerId;
-
- public ActionKillContainer(ContainerId containerId, long delay,
- TimeUnit timeUnit) {
+ private final RMOperationHandler operationHandler;
+ public ActionKillContainer(
+ ContainerId containerId,
+ long delay,
+ TimeUnit timeUnit,
+ RMOperationHandler operationHandler) {
super("kill container", delay, timeUnit);
+ this.operationHandler = operationHandler;
+ Preconditions.checkArgument(containerId != null);
+
this.containerId = containerId;
}
+ /**
+ * Get the container ID to kill
+ * @return
+ */
+ public ContainerId getContainerId() {
+ return containerId;
+ }
+
@Override
public void execute(SliderAppMaster appMaster,
QueueAccess queueService,
@@ -46,6 +62,6 @@ public class ActionKillContainer extends AsyncAction {
ContainerReleaseOperation release = new ContainerReleaseOperation(containerId);
opsList.add(release);
//now apply the operations
- appMaster.executeRMOperations(opsList);
+ operationHandler.execute(opsList);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
index 1154a44..ac59096 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
@@ -55,18 +55,14 @@ implements Runnable, QueueAccess {
/**
* Immediate actions.
- * Package scoped for testing.
*/
- @VisibleForTesting
- final BlockingDeque<AsyncAction> actionQueue =
+ public final BlockingDeque<AsyncAction> actionQueue =
new LinkedBlockingDeque<>();
/**
* Actions to be scheduled in the future
- * Package scoped for testing.
*/
- @VisibleForTesting
- final DelayQueue<AsyncAction> scheduledActions = new DelayQueue<>();
+ public final DelayQueue<AsyncAction> scheduledActions = new DelayQueue<>();
/**
* Map of renewing actions by name ... this is to allow them to
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
index 13bda57..5905d2f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosEntry.java
@@ -41,7 +41,7 @@ public class ChaosEntry {
/**
- * Constructor -includes validaton of all arguments
+ * Constructor -includes validation of all arguments
* @param name
* @param target
* @param probability
@@ -51,7 +51,8 @@ public class ChaosEntry {
Preconditions.checkArgument(!StringUtils.isEmpty(name), "missing name");
Preconditions.checkArgument(target != null, "null target");
Preconditions.checkArgument(probability > 0, "negative probability");
- Preconditions.checkArgument(probability < 10000, "probability over 100%");
+ Preconditions.checkArgument(probability <= ChaosMonkeyService.PERCENT_100,
+ "probability over 100%");
this.name = name;
this.target = target;
this.probability = probability;
@@ -71,11 +72,12 @@ public class ChaosEntry {
/**
* Invoke Chaos if the trigger value is in range of the probability
- * @param trigger trigger value, 0-10K
+ * @param value trigger value, 0-10K
* @return true if the chaos method was invoked
*/
- public boolean maybeInvokeChaos(long trigger) {
- if (probability < 0) {
+ public boolean maybeInvokeChaos(long value) {
+ log.debug("Probability {} trigger={}", probability, value);
+ if (value < probability) {
invokeChaos();
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java
index c464ac3..3c1a914 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillAM.java
@@ -18,21 +18,21 @@
package org.apache.slider.server.appmaster.monkey;
-import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.actions.ActionHalt;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import java.util.concurrent.TimeUnit;
+/**
+ * Kill the AM
+ */
public class ChaosKillAM implements ChaosTarget {
- private final SliderAppMaster appMaster;
+ public static final int DELAY = 1000;
private final QueueAccess queues;
private final int exitCode;
- public ChaosKillAM(SliderAppMaster appMaster,
- QueueAccess queues, int exitCode) {
- this.appMaster = appMaster;
+ public ChaosKillAM(QueueAccess queues, int exitCode) {
this.queues = queues;
this.exitCode = exitCode;
}
@@ -42,7 +42,7 @@ public class ChaosKillAM implements ChaosTarget {
*/
@Override
public void chaosAction() {
- queues.schedule(new ActionHalt(exitCode, "Chaos invoked halt", 1000,
+ queues.schedule(new ActionHalt(exitCode, "Chaos invoked halt", DELAY,
TimeUnit.MILLISECONDS));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
index 0ffcc8a..daf2590 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
@@ -18,9 +18,11 @@
package org.apache.slider.server.appmaster.monkey;
-import org.apache.slider.server.appmaster.SliderAppMaster;
+import com.google.common.base.Preconditions;
import org.apache.slider.server.appmaster.actions.ActionKillContainer;
import org.apache.slider.server.appmaster.actions.QueueAccess;
+import org.apache.slider.server.appmaster.operations.RMOperationHandler;
+import org.apache.slider.server.appmaster.state.AppState;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,24 +37,29 @@ import java.util.concurrent.TimeUnit;
public class ChaosKillContainer implements ChaosTarget {
protected static final Logger log =
LoggerFactory.getLogger(ChaosKillContainer.class);
- private final SliderAppMaster appMaster;
+ public static final int DELAY = 100;
+ private final AppState appState;
private final QueueAccess queues;
private final Random random = new Random();
+ private final RMOperationHandler operationHandler;
- public ChaosKillContainer(SliderAppMaster appMaster,
- QueueAccess queues) {
- this.appMaster = appMaster;
+ public ChaosKillContainer(AppState appState,
+ QueueAccess queues,
+ RMOperationHandler operationHandler) {
+ Preconditions.checkNotNull(appState);
+ Preconditions.checkNotNull(queues);
+ this.appState = appState;
this.queues = queues;
+ this.operationHandler = operationHandler;
}
/**
- * Trigger a container kill halt
+ * Trigger a container kill
*/
-
@Override
public void chaosAction() {
List<RoleInstance> liveContainers =
- appMaster.getAppState().cloneLiveContainerInfoList();
+ appState.cloneLiveContainerInfoList();
int size = liveContainers.size();
if (size == 0) {
log.info("No containers to kill");
@@ -63,6 +70,6 @@ public class ChaosKillContainer implements ChaosTarget {
log.info("Killing {}", roleInstance);
queues.schedule(new ActionKillContainer(roleInstance.getId(),
- 100, TimeUnit.MILLISECONDS));
+ DELAY, TimeUnit.MILLISECONDS, operationHandler));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
index c525d02..cd4b46b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
@@ -20,7 +20,6 @@ package org.apache.slider.server.appmaster.monkey;
import com.codahale.metrics.MetricRegistry;
import org.apache.hadoop.service.AbstractService;
-import org.apache.slider.server.appmaster.actions.AsyncAction;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.actions.RenewingAction;
import org.slf4j.Logger;
@@ -28,6 +27,7 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
+import java.util.Random;
import java.util.concurrent.TimeUnit;
/**
@@ -36,8 +36,16 @@ import java.util.concurrent.TimeUnit;
public class ChaosMonkeyService extends AbstractService {
protected static final Logger log =
LoggerFactory.getLogger(ChaosMonkeyService.class);
+ public static final int PERCENT_1 = 100;
+ public static final double PERCENT_1D = 100.0;
+
+ /**
+ * the percentage value as multiplied up
+ */
+ public static final int PERCENT_100 = 100 * PERCENT_1;
private final MetricRegistry metrics;
private final QueueAccess queues;
+ private final Random random = new Random();
private static final List<ChaosEntry> chaosEntries =
new ArrayList<ChaosEntry>();
@@ -50,14 +58,19 @@ public class ChaosMonkeyService extends AbstractService {
public synchronized void addTarget(String name,
- ChaosTarget target,
- long probability) {
-
+ ChaosTarget target, long probability) {
+ log.info("Adding {} with probability {}", name, probability / PERCENT_1);
chaosEntries.add(new ChaosEntry(name, target, probability, metrics));
}
+ /**
+ * Iterate through all the entries and invoke chaos on those wanted
+ */
public void play() {
-
+ for (ChaosEntry chaosEntry : chaosEntries) {
+ long p = random.nextInt(PERCENT_100);
+ chaosEntry.maybeInvokeChaos(p);
+ }
}
public RenewingAction<MonkeyPlayAction> getChaosAction(long time, TimeUnit timeUnit) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
index 8ccd463..0fdba6b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
@@ -29,17 +29,20 @@ import org.apache.slider.server.appmaster.operations.RMOperationHandler
@Slf4j
class MockRMOperationHandler extends RMOperationHandler {
public List<AbstractRMOperation> operations = [];
-
+ int requests, releases;
+
@Override
public void releaseAssignedContainer(ContainerId containerId) {
operations.add(new ContainerReleaseOperation(containerId))
log.info("Releasing container ID " + containerId.getId())
+ releases++;
}
@Override
public void addContainerRequest(AMRMClient.ContainerRequest req) {
operations.add(new ContainerRequestOperation(req))
log.info("Requesting container role #" + req.priority);
+ requests++;
}
/**
@@ -47,5 +50,7 @@ class MockRMOperationHandler extends RMOperationHandler {
*/
public void clear() {
operations.clear()
+ releases = 0;
+ requests = 0;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
new file mode 100644
index 0000000..c789011
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.monkey
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.server.appmaster.actions.ActionHalt
+import org.apache.slider.server.appmaster.actions.ActionKillContainer
+import org.apache.slider.server.appmaster.actions.QueueService
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler
+import org.apache.slider.server.appmaster.monkey.ChaosKillAM
+import org.apache.slider.server.appmaster.monkey.ChaosKillContainer
+import org.apache.slider.server.appmaster.monkey.ChaosMonkeyService
+import org.apache.slider.server.appmaster.monkey.ChaosTarget
+import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.junit.Before
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+class TestMockMonkey extends BaseMockAppStateTest {
+
+ /**
+ * This queue service is NOT started; tests need to poll the queue
+ * rather than expect them to execute
+ */
+ QueueService queues = new QueueService();
+ ChaosMonkeyService monkey = new ChaosMonkeyService(metricRegistry,
+ queues)
+
+ @Before
+ public void init() {
+ def configuration = new YarnConfiguration()
+ queues.init(configuration)
+ monkey.init(configuration)
+ }
+
+ @Test
+ public void testMonkeyStart() throws Throwable {
+ monkey.start()
+ monkey.stop()
+ }
+
+
+ @Test
+ public void testMonkeyPlay() throws Throwable {
+ ChaosCounter counter = new ChaosCounter()
+ monkey.addTarget("target", counter, ChaosMonkeyService.PERCENT_100)
+
+ monkey.play()
+ assert counter.count == 1
+ }
+
+ @Test
+ public void testMonkeyPlaySometimes() throws Throwable {
+ ChaosCounter counter = new ChaosCounter()
+ ChaosCounter counter2 = new ChaosCounter()
+ monkey.addTarget("target1", counter, ChaosMonkeyService.PERCENT_1 * 50)
+ monkey.addTarget("target2", counter2, ChaosMonkeyService.PERCENT_1 * 25)
+
+ for (int i = 0; i < 100; i++) {
+ monkey.play()
+ }
+ log.info("Counter1 = ${counter.count} counter2 = ${counter2.count}")
+ /*
+ * Relying on probability here to give approximate answers
+ */
+ assert counter.count > 25
+ assert counter.count < 75
+ assert counter2.count < counter.count
+ }
+
+ @Test
+ public void testAMKiller() throws Throwable {
+
+ def chaos = new ChaosKillAM(queues, -1)
+ chaos.chaosAction();
+ assert queues.scheduledActions.size() == 1
+ def action = queues.scheduledActions.take()
+ assert action instanceof ActionHalt
+ }
+
+
+ @Test
+ public void testContainerKillerEmptyApp() throws Throwable {
+
+
+ def chaos = new ChaosKillContainer(appState,
+ queues,
+ new MockRMOperationHandler())
+ chaos.chaosAction();
+ assert queues.scheduledActions.size() == 0
+ }
+
+
+
+ @Test
+ public void testContainerKiller() throws Throwable {
+ MockRMOperationHandler ops = new MockRMOperationHandler();
+ role0Status.desired = 1
+ List<RoleInstance> instances = createAndStartNodes()
+ assert instances.size() == 1
+ def instance = instances[0]
+
+ def chaos = new ChaosKillContainer(appState, queues, ops)
+ chaos.chaosAction();
+ assert queues.scheduledActions.size() == 1
+ def action = queues.scheduledActions.take()
+ ActionKillContainer killer = (ActionKillContainer) action
+ assert killer.containerId == instance.containerId;
+ killer.execute(null, queues, appState)
+ assert ops.releases == 1;
+
+ ContainerReleaseOperation operation = (ContainerReleaseOperation) ops.operations[0]
+ assert operation.containerId == instance.containerId
+ }
+
+
+
+ /**
+ * Chaos target that just implement a counter
+ */
+ private static class ChaosCounter implements ChaosTarget {
+ int count;
+
+ @Override
+ void chaosAction() {
+ count++;
+ }
+
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "ChaosCounter{");
+ sb.append("count=").append(count);
+ sb.append('}');
+ return sb.toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/52bd11b5/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
index d7b328f..28b484f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
@@ -18,6 +18,7 @@
package org.apache.slider.test
+import com.codahale.metrics.MetricRegistry
import groovy.transform.CompileStatic
import org.apache.hadoop.fs.FileUtil
import org.apache.slider.common.SliderXMLConfKeysForTesting
@@ -38,7 +39,11 @@ import java.nio.file.Files
@CompileStatic
public abstract class SliderTestBase extends SliderTestUtils {
-
+ /**
+ * Singleton metric registry
+ */
+ public static final MetricRegistry metricRegistry = new MetricRegistry()
+
@Rule
public TestName methodName = new TestName();
[43/50] [abbrv] git commit: SLIDER-331 fixed incorrect headers,
added missing dependencies
Posted by st...@apache.org.
SLIDER-331 fixed incorrect headers, added missing dependencies
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/80eec9b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/80eec9b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/80eec9b1
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 80eec9b105968f5c13f97155f1d241067a6c924d
Parents: a34cca8
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Mon Aug 18 17:07:34 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Mon Aug 18 17:07:34 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/pom.xml | 11 +++++++++
pom.xml | 24 ++++++++++++++++---
slider-assembly/pom.xml | 14 -----------
slider-core/pom.xml | 16 ++++++++++++-
.../accumulo/accumulo-funtests/pom.xml | 23 ++++++++++++++++++
slider-providers/hbase/hbase-funtests/pom.xml | 5 ++++
src/test/clusters/c6401/slider/log4j.properties | 25 +++++++++++---------
.../clusters/morzine/slider/log4j.properties | 25 +++++++++++---------
.../clusters/offline/slider/log4j.properties | 25 +++++++++++---------
.../clusters/remote/slider/log4j.properties | 25 +++++++++++---------
.../clusters/sandbox/slider/log4j.properties | 25 +++++++++++---------
11 files changed, 145 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80eec9b1/app-packages/accumulo/pom.xml
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml
index 801d90f..02bf668 100644
--- a/app-packages/accumulo/pom.xml
+++ b/app-packages/accumulo/pom.xml
@@ -169,6 +169,12 @@
</dependency>
<dependency>
<groupId>org.apache.slider</groupId>
+ <artifactId>slider-core</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.slider</groupId>
<artifactId>slider-funtest</artifactId>
<scope>test</scope>
</dependency>
@@ -177,6 +183,11 @@
<artifactId>groovy-all</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80eec9b1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 223ff2b..22f56b1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -145,9 +145,11 @@
<jackson.version>1.9.13</jackson.version>
<jcommander.version>1.30</jcommander.version>
+ <jetty.version>6.1.26</jetty.version>
<jersey.version>1.9</jersey.version>
<servlet-api.version>2.5</servlet-api.version>
<jsr311-api.version>1.1.1</jsr311-api.version>
+ <jaxb-api.version>2.2.7</jaxb-api.version>
<junit.version>4.11</junit.version>
<log4j.version>1.2.17</log4j.version>
@@ -1119,6 +1121,12 @@
</dependency>
<dependency>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ <version>${jaxb-api.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
<version>${jersey.version}</version>
@@ -1205,9 +1213,19 @@
<!-- ======================================================== -->
<dependency>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-sslengine</artifactId>
- <version>6.1.26</version>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-sslengine</artifactId>
+ <version>${jetty.version}</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80eec9b1/slider-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/slider-assembly/pom.xml b/slider-assembly/pom.xml
index d5299ee..e9defb5 100644
--- a/slider-assembly/pom.xml
+++ b/slider-assembly/pom.xml
@@ -141,26 +141,12 @@
</configuration>
<executions>
<execution>
- <id>clean</id>
- <goals>
- <goal>enforce</goal>
- </goals>
- <phase>pre-clean</phase>
- </execution>
- <execution>
<id>default</id>
<goals>
<goal>enforce</goal>
</goals>
<phase>validate</phase>
</execution>
- <execution>
- <id>site</id>
- <goals>
- <goal>enforce</goal>
- </goals>
- <phase>pre-site</phase>
- </execution>
</executions>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80eec9b1/slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index d403034..b2bdeed 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -406,6 +406,11 @@
</dependency>
<dependency>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-client</artifactId>
</dependency>
@@ -466,8 +471,17 @@
<dependency>
<groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-sslengine</artifactId>
- <scope>compile</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80eec9b1/slider-providers/accumulo/accumulo-funtests/pom.xml
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/accumulo-funtests/pom.xml b/slider-providers/accumulo/accumulo-funtests/pom.xml
index 0f11618..73d9ae0 100644
--- a/slider-providers/accumulo/accumulo-funtests/pom.xml
+++ b/slider-providers/accumulo/accumulo-funtests/pom.xml
@@ -143,6 +143,16 @@
<dependency>
<groupId>org.apache.slider</groupId>
<artifactId>slider-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-core</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
@@ -207,9 +217,22 @@
<dependency>
<groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-fate</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-test</artifactId>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ <version>0.9.0</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80eec9b1/slider-providers/hbase/hbase-funtests/pom.xml
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/pom.xml b/slider-providers/hbase/hbase-funtests/pom.xml
index d5e9d0e..d143408 100644
--- a/slider-providers/hbase/hbase-funtests/pom.xml
+++ b/slider-providers/hbase/hbase-funtests/pom.xml
@@ -140,6 +140,11 @@
<dependency>
<groupId>org.apache.slider</groupId>
<artifactId>slider-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-core</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80eec9b1/src/test/clusters/c6401/slider/log4j.properties
----------------------------------------------------------------------
diff --git a/src/test/clusters/c6401/slider/log4j.properties b/src/test/clusters/c6401/slider/log4j.properties
index d814f14..4682a96 100644
--- a/src/test/clusters/c6401/slider/log4j.properties
+++ b/src/test/clusters/c6401/slider/log4j.properties
@@ -1,15 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. See accompanying LICENSE file.
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
# log4j configuration used during build and unit tests
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80eec9b1/src/test/clusters/morzine/slider/log4j.properties
----------------------------------------------------------------------
diff --git a/src/test/clusters/morzine/slider/log4j.properties b/src/test/clusters/morzine/slider/log4j.properties
index d814f14..4682a96 100644
--- a/src/test/clusters/morzine/slider/log4j.properties
+++ b/src/test/clusters/morzine/slider/log4j.properties
@@ -1,15 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. See accompanying LICENSE file.
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
# log4j configuration used during build and unit tests
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80eec9b1/src/test/clusters/offline/slider/log4j.properties
----------------------------------------------------------------------
diff --git a/src/test/clusters/offline/slider/log4j.properties b/src/test/clusters/offline/slider/log4j.properties
index d814f14..4682a96 100644
--- a/src/test/clusters/offline/slider/log4j.properties
+++ b/src/test/clusters/offline/slider/log4j.properties
@@ -1,15 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. See accompanying LICENSE file.
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
# log4j configuration used during build and unit tests
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80eec9b1/src/test/clusters/remote/slider/log4j.properties
----------------------------------------------------------------------
diff --git a/src/test/clusters/remote/slider/log4j.properties b/src/test/clusters/remote/slider/log4j.properties
index f672472..5b59190 100644
--- a/src/test/clusters/remote/slider/log4j.properties
+++ b/src/test/clusters/remote/slider/log4j.properties
@@ -1,15 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. See accompanying LICENSE file.
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
# log4j configuration used during build and unit tests
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80eec9b1/src/test/clusters/sandbox/slider/log4j.properties
----------------------------------------------------------------------
diff --git a/src/test/clusters/sandbox/slider/log4j.properties b/src/test/clusters/sandbox/slider/log4j.properties
index d814f14..4682a96 100644
--- a/src/test/clusters/sandbox/slider/log4j.properties
+++ b/src/test/clusters/sandbox/slider/log4j.properties
@@ -1,15 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License. See accompanying LICENSE file.
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
# log4j configuration used during build and unit tests
[34/50] [abbrv] git commit: SLIDER-320 agent provider registers
endpoints and triggers a publish operation,
as does app state on an NM container start event. Nothing happens at the
moment.
Posted by st...@apache.org.
SLIDER-320 agent provider registers endpoints and triggers a publish operation, as does app state on an NM container start event. Nothing happens at the moment.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/31bfae26
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/31bfae26
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/31bfae26
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 31bfae26a5ee44794cd72fd681dbf69d237e18f8
Parents: 7e2ce27
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 15 15:18:42 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 15 15:18:42 2014 +0100
----------------------------------------------------------------------
.../providers/agent/AgentProviderService.java | 32 +++++++++++++--
.../server/appmaster/SliderAppMaster.java | 24 +++++++++--
.../actions/PublishRegistryDetails.java | 3 +-
.../slider/server/appmaster/state/AppState.java | 43 +++++++++++++++++---
.../appmaster/state/ProviderAppState.java | 5 +++
.../server/appmaster/state/RoleInstance.java | 42 +++++++++++++++++--
.../state/StateAccessForProviders.java | 7 ++++
.../providers/hbase/HBaseProviderService.java | 11 ++---
8 files changed, 144 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/31bfae26/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 91bc0a6..680074d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -42,6 +42,7 @@ import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.NoSuchNodeException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.CommandLineBuilder;
import org.apache.slider.core.launch.ContainerLauncher;
@@ -62,7 +63,9 @@ import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.providers.agent.application.metadata.OSPackage;
import org.apache.slider.providers.agent.application.metadata.OSSpecific;
import org.apache.slider.server.appmaster.actions.ProviderReportedContainerLoss;
+import org.apache.slider.server.appmaster.actions.PublishRegistryDetails;
import org.apache.slider.server.appmaster.state.ContainerPriority;
+import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
@@ -96,6 +99,7 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_AGENTS;
@@ -541,14 +545,36 @@ public class AgentProviderService extends AbstractProviderService implements
String roleName,
String containerId,
Map<String, String> ports) {
+ RoleInstance instance;
+ try {
+ instance = getAmState().getOwnedContainer(containerId);
+ } catch (NoSuchNodeException e) {
+ log.warn("Failed to locate instance of container {}: {}", containerId, e);
+ instance = null;
+ }
for (Map.Entry<String, String> port : ports.entrySet()) {
- log.info("Recording allocated port for {} as {}", port.getKey(), port.getValue());
- this.getAllocatedPorts().put(port.getKey(), port.getValue());
- this.getAllocatedPorts(containerId).put(port.getKey(), port.getValue());
+ String portname = port.getKey();
+ String portNo = port.getValue();
+ log.info("Recording allocated port for {} as {}", portname, portNo);
+ this.getAllocatedPorts().put(portname, portNo);
+ this.getAllocatedPorts(containerId).put(portname, portNo);
+ if (instance!=null) {
+ try {
+ instance.registerPortEndpoint(Integer.valueOf(portNo), portname, "");
+ } catch (NumberFormatException e) {
+ log.warn("Failed to parse {}: {}", portNo, e);
+ }
+ }
}
// component specific publishes
processAndPublishComponentSpecificData(ports, containerId, fqdn, roleName);
+
+ // and update registration entries
+ if (instance != null) {
+ queueAccess.put(new PublishRegistryDetails(instance.getId(), 0,
+ TimeUnit.MILLISECONDS));
+ }
}
private void updateComponentStatusWithAgentState(
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/31bfae26/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 9a16188..c54a268 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -84,7 +84,6 @@ import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.exceptions.SliderInternalStateException;
import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
import org.apache.slider.core.main.ExitCodeProvider;
-import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.slider.core.main.RunService;
import org.apache.slider.core.main.ServiceLauncher;
import org.apache.slider.core.persist.ConfTreeSerDeser;
@@ -99,6 +98,7 @@ import org.apache.slider.providers.SliderProviderFactory;
import org.apache.slider.providers.slideram.SliderAMClientProvider;
import org.apache.slider.providers.slideram.SliderAMProviderService;
import org.apache.slider.server.appmaster.actions.ActionKillContainer;
+import org.apache.slider.server.appmaster.actions.PublishRegistryDetails;
import org.apache.slider.server.appmaster.actions.QueueExecutor;
import org.apache.slider.server.appmaster.actions.ActionHalt;
import org.apache.slider.server.appmaster.actions.QueueService;
@@ -901,6 +901,20 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
/**
+ * Register/re-register a component (that is already in the app state
+ * @param id the component
+ */
+ public boolean registerComponent(ContainerId id) {
+ RoleInstance instance = appState.getOwnedContainer(id);
+ if (instance == null) {
+ return false;
+ }
+ // this is where component registrations will go
+
+ return true;
+ }
+
+ /**
* looks for a specific case where a token file is provided as an environment
* variable, yet the file is not there.
*
@@ -1192,9 +1206,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
throws IOException, SliderInternalStateException, BadConfigException {
appState.updateResourceDefinitions(resources);
- appState.resetFailureCounts();
- // reset the scheduled window resetter...the values
+
+ // reset the scheduled windows...the values
// may have changed
+ appState.resetFailureCounts();
@@ -1665,6 +1680,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//trigger an async container status
nmClientAsync.getContainerStatusAsync(containerId,
cinfo.container.getNodeId());
+ // push out a registration
+ queue(new PublishRegistryDetails(containerId, 0, TimeUnit.MILLISECONDS));
+
} else {
//this is a hypothetical path not seen. We react by warning
log.error("Notified of started container that isn't pending {} - releasing",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/31bfae26/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java
index f96183f..cb6d140 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/PublishRegistryDetails.java
@@ -31,7 +31,7 @@ public class PublishRegistryDetails extends AsyncAction {
public PublishRegistryDetails(ContainerId containerId, long delay,
TimeUnit timeUnit) {
- super("PublishEndpointDetails :" + containerId.toString(),
+ super("PublishRegistryDetails :" + containerId.toString(),
delay, timeUnit);
this.containerId = containerId;
}
@@ -40,6 +40,5 @@ public class PublishRegistryDetails extends AsyncAction {
public void execute(SliderAppMaster appMaster,
QueueAccess queueService,
AppState appState) throws Exception {
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/31bfae26/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 2f2eadd..07976ef 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -850,20 +850,53 @@ public class AppState {
/**
* Lookup live instance by string value of container ID
- * @param containerId container ID
+ * @param containerId container ID as a string
* @return the role instance for that container
* @throws NoSuchNodeException if it does not exist
*/
public synchronized RoleInstance getLiveInstanceByContainerID(String containerId)
- throws NoSuchNodeException {
+ throws NoSuchNodeException {
Collection<RoleInstance> nodes = getLiveNodes().values();
+ return findNodeInCollection(containerId, nodes);
+ }
+
+ /**
+ * Lookup owned instance by string value of container ID
+ * @param containerId container ID as a string
+ * @return the role instance for that container
+ * @throws NoSuchNodeException if it does not exist
+ */
+ public synchronized RoleInstance getOwnedInstanceByContainerID(String containerId)
+ throws NoSuchNodeException {
+ Collection<RoleInstance> nodes = ownedContainers.values();
+ return findNodeInCollection(containerId, nodes);
+ }
+
+
+
+ /**
+ * Iterate through a collection of role instances to find one with a
+ * specific (string) container ID
+ * @param containerId container ID as a string
+ * @param nodes collection
+ * @return
+ * @throws NoSuchNodeException if there was no match
+ */
+ private RoleInstance findNodeInCollection(String containerId,
+ Collection<RoleInstance> nodes) throws NoSuchNodeException {
+ RoleInstance found = null;
for (RoleInstance node : nodes) {
if (containerId.equals(node.id)) {
- return node;
+ found = node;
+ break;
}
}
- //at this point: no node
- throw new NoSuchNodeException(containerId);
+ if (found != null) {
+ return found;
+ } else {
+ //at this point: no node
+ throw new NoSuchNodeException(containerId);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/31bfae26/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
index e7384c8..a0871ae 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
@@ -179,6 +179,11 @@ public class ProviderAppState implements StateAccessForProviders {
}
@Override
+ public RoleInstance getOwnedContainer(String id) throws NoSuchNodeException {
+ return appState.getOwnedInstanceByContainerID(id);
+ }
+
+ @Override
public List<RoleInstance> cloneLiveContainerInfoList() {
return appState.cloneLiveContainerInfoList();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/31bfae26/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
index 8b6ed8e..e373843 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
@@ -25,8 +25,12 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.proto.Messages;
import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.registry.info.RegisteredEndpoint;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
/**
* Tracking information about a container
@@ -82,11 +86,12 @@ public final class RoleInstance implements Cloneable {
public String host;
public String hostURL;
+
/**
- * Any information the provider wishes to retain on the state of
- * an instance
+ * A list of registered endpoints.
*/
- public Object providerInfo;
+ private List<RegisteredEndpoint> endpoints =
+ new ArrayList<RegisteredEndpoint>(2);
public RoleInstance(Container container) {
Preconditions.checkNotNull(container, "Null container");
@@ -188,8 +193,39 @@ public final class RoleInstance implements Cloneable {
@Override
public Object clone() throws CloneNotSupportedException {
RoleInstance cloned = (RoleInstance) super.clone();
+ // clone the endpoint list, but not the values
+ cloned.endpoints = new ArrayList<RegisteredEndpoint>(this.endpoints);
return cloned;
}
+ /**
+ * Get the list of endpoints.
+ * @return the endpoint list.
+ */
+ public List<RegisteredEndpoint> getEndpoints() {
+ return endpoints;
+ }
+
+ /**
+ * Add an endpoint registration
+ * @param endpoint
+ */
+ public void addEndpoint(RegisteredEndpoint endpoint) {
+ Preconditions.checkArgument(endpoint != null);
+ endpoints.add(endpoint);
+ }
+ /**
+ * Register a port endpoint as an inet-addr formatted endpoint, using the
+ * hostname as the first part of the address
+ * @param port
+ * @param protocol
+ * @param text
+ */
+ public void registerPortEndpoint(int port, String protocol, String text) {
+ InetSocketAddress addr = new InetSocketAddress(host, port);
+ RegisteredEndpoint epr = new RegisteredEndpoint(addr, protocol, text);
+ addEndpoint(epr);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/31bfae26/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
index b236dfc..1714f75 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
@@ -160,6 +160,13 @@ public interface StateAccessForProviders {
RoleInstance getOwnedContainer(ContainerId id);
/**
+ * Get any active container with the given ID
+ * @param id container Id
+ * @return the active container or null if it is not found
+ */
+ RoleInstance getOwnedContainer(String id) throws NoSuchNodeException;
+
+ /**
* Create a clone of the list of live cluster nodes.
* @return the list of nodes, may be empty
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/31bfae26/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index 1fb392c..82e535f 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -68,11 +68,8 @@ import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_
* This class implements the server-side aspects
* of an HBase Cluster
*/
-public class HBaseProviderService extends AbstractProviderService implements
- ProviderCore,
- HBaseKeys,
- SliderKeys,
- AgentRestOperations{
+public class HBaseProviderService extends AbstractProviderService
+ implements ProviderCore, HBaseKeys, SliderKeys, AgentRestOperations{
protected static final Logger log =
LoggerFactory.getLogger(HBaseProviderService.class);
@@ -109,8 +106,8 @@ public class HBaseProviderService extends AbstractProviderService implements
* @param instanceDefinition the instance definition to validate
*/
@Override // Client and Server
- public void validateInstanceDefinition(AggregateConf instanceDefinition) throws
- SliderException {
+ public void validateInstanceDefinition(AggregateConf instanceDefinition)
+ throws SliderException {
clientProvider.validateInstanceDefinition(instanceDefinition);
}
[20/50] [abbrv] git commit: Merge branch
'feature/SLIDER-322_Slider_to_briefly_support_Java_1.6' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-322_Slider_to_briefly_support_Java_1.6' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/966c8c83
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/966c8c83
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/966c8c83
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: 966c8c83d84bf0a127b646608821ca9316b179ac
Parents: 9aa0299 b6782e1
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 14 18:23:46 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 14 18:23:46 2014 +0100
----------------------------------------------------------------------
pom.xml | 8 +-
.../src/main/java/org/apache/slider/Slider.java | 2 +-
.../apache/slider/api/ClusterDescription.java | 14 +--
.../org/apache/slider/client/SliderClient.java | 100 +++++++++++++++++--
.../slider/client/SliderYarnClientImpl.java | 8 +-
.../common/params/AbstractActionArgs.java | 6 +-
.../params/AppAndResouceOptionArgsDelegate.java | 8 +-
.../org/apache/slider/common/params/ArgOps.java | 8 +-
.../apache/slider/common/params/CommonArgs.java | 4 +-
.../common/params/DontSplitArguments.java | 2 +-
.../slider/common/tools/ConfigHelper.java | 60 +++++++----
.../slider/common/tools/CoreFileSystem.java | 2 +-
.../apache/slider/common/tools/SliderUtils.java | 10 +-
.../org/apache/slider/core/conf/ConfTree.java | 6 +-
.../slider/core/conf/ConfTreeOperations.java | 4 +-
.../apache/slider/core/conf/MapOperations.java | 2 +-
.../slider/core/launch/AbstractLauncher.java | 10 +-
.../core/launch/ClasspathConstructor.java | 2 +-
.../slider/core/launch/CommandLineBuilder.java | 2 +-
.../slider/core/main/ServiceLauncher.java | 2 +-
.../slider/core/persist/JsonSerDeser.java | 27 ++++-
.../registry/docstore/PublishedConfigSet.java | 2 +-
.../docstore/PublishedConfiguration.java | 4 +-
.../PublishedConfigurationOutputter.java | 12 +++
.../slider/core/registry/docstore/UriMap.java | 2 +-
.../slider/core/registry/info/RegistryView.java | 3 +-
.../core/registry/info/ServiceInstanceData.java | 4 +-
.../apache/slider/core/zk/ZKIntegration.java | 2 +-
.../apache/slider/core/zk/ZookeeperUtils.java | 6 +-
.../providers/AbstractProviderService.java | 4 +-
.../apache/slider/providers/ProviderUtils.java | 2 +-
.../providers/agent/AgentClientProvider.java | 2 +-
.../providers/agent/AgentLaunchParameter.java | 2 +-
.../providers/agent/AgentProviderService.java | 47 +++++----
.../providers/agent/ComponentCommandOrder.java | 9 +-
.../agent/application/metadata/Application.java | 8 +-
.../agent/application/metadata/Component.java | 2 +-
.../slideram/SliderAMClientProvider.java | 4 +-
.../slideram/SliderAMProviderService.java | 2 +-
.../server/appmaster/SliderAppMaster.java | 37 +++++--
.../appmaster/actions/ActionKillContainer.java | 2 +-
.../server/appmaster/actions/QueueService.java | 6 +-
.../appmaster/monkey/ChaosMonkeyService.java | 2 +-
.../slider/server/appmaster/state/AppState.java | 56 +++++------
.../MostRecentContainerReleaseSelector.java | 2 +-
.../server/appmaster/state/NodeInstance.java | 2 +-
.../slider/server/appmaster/state/NodeMap.java | 2 +-
.../state/OutstandingRequestTracker.java | 6 +-
.../appmaster/state/ProviderAppState.java | 4 +-
.../server/appmaster/state/RoleHistory.java | 15 ++-
.../server/appmaster/state/RoleStatus.java | 2 +-
.../server/appmaster/web/SliderAMWebApp.java | 2 +-
.../web/SliderAmFilterInitializer.java | 2 +-
.../server/appmaster/web/SliderAmIpFilter.java | 8 +-
.../server/appmaster/web/WebAppApiImpl.java | 2 +-
.../web/rest/publisher/PublisherResource.java | 4 +-
.../slider/server/avro/RoleHistoryWriter.java | 6 +-
.../server/servicemonitor/MonitorUtils.java | 2 +-
.../services/curator/RegistryBinderService.java | 6 +-
.../registry/RegistryRestResources.java | 3 +-
.../registry/SliderRegistryService.java | 2 +-
.../agent/TestAgentProviderService.java | 45 +++++----
.../agent/TestComponentInstanceState.java | 11 +-
.../providers/agent/TestHeartbeatMonitor.java | 4 +-
.../management/TestAMManagementWebServices.java | 9 +-
.../publisher/TestAgentProviderService.java | 2 +-
.../workflow/ProcessCommandFactory.java | 6 +-
.../slider/test/MiniZooKeeperCluster.java | 6 +-
.../accumulo/AccumuloProviderService.java | 19 +++-
.../providers/hbase/HBaseClientProvider.java | 6 +-
.../providers/hbase/HBaseProviderService.java | 31 +++++-
71 files changed, 476 insertions(+), 240 deletions(-)
----------------------------------------------------------------------
[48/50] [abbrv] git commit: SLIDER-306 changes pulled in,
reviewed tests set up and working
Posted by st...@apache.org.
SLIDER-306 changes pulled in, reviewed tests set up and working
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fa289e8d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fa289e8d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fa289e8d
Branch: refs/heads/feature/SLIDER-306_Slider_list_command_live_and_history_options
Commit: fa289e8d700aee1fad27670fd6875468038bafde
Parents: 7b589d9
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 19 12:52:16 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 19 12:52:16 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 60 +++++++++++++-------
.../apache/slider/common/tools/SliderUtils.java | 6 +-
.../slider/agent/actions/TestActionList.groovy | 9 +--
.../agent/freezethaw/TestFreezeCommands.groovy | 3 +-
.../standalone/TestStandaloneAMDestroy.groovy | 16 ++++++
.../slider/client/TestCommonArgParsing.groovy | 3 +-
6 files changed, 66 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fa289e8d/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index d608278..dc186ce 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1420,29 +1420,42 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
String user = UserGroupInformation.getCurrentUser().getUserName();
List<ApplicationReport> instances = listSliderInstances(user);
SliderUtils.sortApplicationReport(instances);
+ boolean live = args.live;
+ boolean history = args.history;
if (isUnset(clustername)) {
+ // no cluster name: list all
log.info("Instances for {}: {}",
- (user != null ? user : "all users"),
- instances.size());
+ (user != null ? user : "all users"),
+ instances.size());
for (ApplicationReport report : instances) {
- logAppReport(report, args.live, args.history);
+ logAppReport(report, live, history);
}
+ // and always succeed
return EXIT_SUCCESS;
} else {
+ // cluster name provided
SliderUtils.validateClusterName(clustername);
- log.debug("Listing cluster named {}", clustername);
- if (args.history) {
+ log.debug("Listing cluster named {}, live={}, history={}",
+ clustername, live, history);
+ boolean instanceFound = false;
+ if (history) {
for (ApplicationReport report : instances) {
if (report.getName().equals(clustername)) {
- logAppReport(report, args.live, args.history);
+ logAppReport(report, live, true);
+ instanceFound = true;
}
}
- return EXIT_SUCCESS;
+ } else {
+ // no history flag, only list live value
+ ApplicationReport report =
+ findClusterInInstanceList(instances, clustername);
+ if (report != null) {
+ logAppReport(report, true, true);
+ instanceFound = true;
+ }
}
- ApplicationReport report = findClusterInInstanceList(instances,
- clustername);
- if (report != null) {
- logAppReport(report, true, false);
+ // exit code if the instance was found
+ if (instanceFound) {
return EXIT_SUCCESS;
} else {
throw unknownClusterException(clustername);
@@ -1453,21 +1466,30 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
/**
* Log the application report at INFO
* @param report report to log
+ * @param live only list live apps
+ * @param history list historical containers
*/
public void logAppReport(ApplicationReport report, boolean live, boolean history) {
- boolean active = report.getYarnApplicationState() == YarnApplicationState.RUNNING
- || report.getYarnApplicationState() ==
- YarnApplicationState.ACCEPTED;
- if (active && live && !history) {
- log.info(SliderUtils.appReportToString(report, "\n"));
- } else if (!active && history && !live) {
- log.info(SliderUtils.appReportToString(report, "\n"));
- } else {
+ // app is active if it is accepted or running
+ boolean active = isApplicationActive(report);
+
+ boolean toLog = (active && live) || (!active && history);
+ if (toLog) {
log.info(SliderUtils.appReportToString(report, "\n"));
}
}
/**
+ * Is an application active: accepted or running
+ * @param report the application report
+ * @return true if it is running or scheduled to run.
+ */
+ public boolean isApplicationActive(ApplicationReport report) {
+ return report.getYarnApplicationState() == YarnApplicationState.RUNNING
+ || report.getYarnApplicationState() == YarnApplicationState.ACCEPTED;
+ }
+
+ /**
* Implement the islive action: probe for a cluster of the given name existing
* @return exit code
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fa289e8d/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 3f83855..f740d3d 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -571,9 +571,9 @@ public final class SliderUtils {
* Sorts the given list of application reports
* Finished instances are ordered by finished time and running/accepted instances are
* ordered by start time
- * Finally Instance are order by finished instances and running instances
+ * Finally Instance are order by finished instances coming after running instances
*
- * @param instances
+ * @param instances list of intances
*/
public static void sortApplicationReport(List<ApplicationReport> instances) {
if (instances.size() <= 1) {
@@ -610,8 +610,8 @@ public final class SliderUtils {
Collections.sort(nonLiveInstance, nonLiveInstanceComparator);
}
instances.clear();
- instances.addAll(nonLiveInstance);
instances.addAll(liveInstance);
+ instances.addAll(nonLiveInstance);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fa289e8d/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
index 5ccaeca..b80658d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
@@ -173,15 +173,11 @@ class TestActionList extends AgentMiniClusterTestBase {
args.live = true;
args.history = false;
- try {
- int exitCode = sliderClient.actionList(clustername, args);
- fail("expected an exception, got a status code $exitCode")
- } catch (UnknownApplicationInstanceException expected) {
- }
+ assert 0 == sliderClient.actionList(clustername, args);
// historical list will work
args.history = true;
- assert sliderClient.actionList(clustername, args) == 0;
+ assert 0 == sliderClient.actionList(clustername, args)
// thaw
ActionThawArgs thawArgs = new ActionThawArgs();
@@ -192,7 +188,6 @@ class TestActionList extends AgentMiniClusterTestBase {
args.live = true;
args.history = false;
assert 0 == sliderClient.actionList(clustername, args);
- assert launcher.serviceExitCode == 0
//Listing all the instance both history (previously freezed instance) and live
args.live = true
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fa289e8d/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy
index f5eff25..14541f3 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy
@@ -107,7 +107,8 @@ class TestFreezeCommands extends AgentMiniClusterTestBase {
ServiceLauncher thawCommand = execSliderCommand(conf, commands);
assertSucceeded(thawCommand)
assertSucceeded(execSliderCommand(conf,
- [SliderActions.ACTION_LIST, clustername]))
+ [SliderActions.ACTION_LIST, clustername,
+ Arguments.ARG_LIVE]))
assertSucceeded(execSliderCommand(conf,
[SliderActions.ACTION_EXISTS, clustername]))
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fa289e8d/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
index 17e4ff2..07da54b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
@@ -57,6 +57,22 @@ class TestStandaloneAMDestroy extends AgentMiniClusterTestBase {
])
assert launcher1.serviceExitCode == 0
+ // try to list it and expect failures
+ try {
+ launchClientAgainstMiniMR(
+ configuration,
+ [
+ SliderActions.ACTION_LIST,
+ "no-cluster-of-this-name",
+ Arguments.ARG_LIVE, Arguments.ARG_HISTORY
+ ])
+ fail("expected a failure")
+ } catch (UnknownApplicationInstanceException e) {
+ assertExceptionDetails(e,
+ SliderExitCodes.EXIT_UNKNOWN_INSTANCE,
+ ErrorStrings.E_UNKNOWN_INSTANCE)
+ }
+
ServiceLauncher<SliderClient> launcher = createStandaloneAM(
clustername,
true,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fa289e8d/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy
index d94cd54..2083576 100644
--- a/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy
@@ -74,7 +74,8 @@ class TestCommonArgParsing implements SliderActions, Arguments {
@Test
public void testSliderBasePath() throws Throwable {
- ClientArgs clientArgs = createClientArgs([ACTION_LIST, "--basepath", "/projects/slider/clusters"])
+ ClientArgs clientArgs = createClientArgs([ACTION_LIST,
+ ARG_BASE_PATH, "/projects/slider/clusters"])
assert clientArgs.basePath == new Path("/projects/slider/clusters")
}