You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/08/09 23:51:47 UTC
[01/14] git commit: SLIDER-240 Detect discrepancy between file
content of {application.def} and {package_list} value
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-181_Storm_Package bc4a5acc6 -> ee8bf826b
SLIDER-240 Detect discrepancy between file content of {application.def} and {package_list} value
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/66cbd376
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/66cbd376
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/66cbd376
Branch: refs/heads/feature/SLIDER-181_Storm_Package
Commit: 66cbd3761ead0fa39132b2a07833c9e6b2f4642d
Parents: bd69a4c
Author: tedyu <yu...@gmail.com>
Authored: Thu Aug 7 11:41:23 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Thu Aug 7 11:41:23 2014 -0700
----------------------------------------------------------------------
.../slider/common/tools/CoreFileSystem.java | 33 ++++++++++++++++++++
.../providers/agent/AgentClientProvider.java | 6 ++++
2 files changed, 39 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66cbd376/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index cada732..93de4d9 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -48,8 +48,11 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.charset.Charset;
+import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
import static org.apache.slider.common.SliderXmlConfKeys.CLUSTER_DIRECTORY_PERMISSIONS;
import static org.apache.slider.common.SliderXmlConfKeys.DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS;
@@ -304,6 +307,36 @@ public class CoreFileSystem {
}
/**
+ * Verify that a file exists in the zip file given by path
+ * @param path path to zip file
+ * @param file file expected to be in zip
+ * @throws FileNotFoundException file not found or is not a zip file
+ * @throws IOException trouble with FS
+ */
+ public void verifyFileExistsInZip(Path path, String file) throws IOException {
+ fileSystem.copyToLocalFile(path, new Path("/tmp"));
+ File dst = new File((new Path("/tmp", path.getName())).toString());
+ Enumeration<? extends ZipEntry> entries;
+ ZipFile zipFile = new ZipFile(dst);
+ boolean found = false;
+
+ try {
+ entries = zipFile.entries();
+ while (entries.hasMoreElements()) {
+ ZipEntry entry = entries.nextElement();
+ String nm = entry.getName();
+ if (nm.endsWith(file)) {
+ found = true;
+ break;
+ }
+ }
+ } finally {
+ zipFile.close();
+ }
+ dst.delete();
+ if (!found) throw new FileNotFoundException("file: " + file + " not found in " + path);
+ }
+ /**
* Create the application-instance specific temporary directory
* in the DFS
*
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66cbd376/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 3835df6..2ce16c7 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -92,6 +92,12 @@ public class AgentClientProvider extends AbstractClientProvider
getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
Path appDefPath = new Path(appDef);
sliderFileSystem.verifyFileExists(appDefPath);
+
+ String pkgList = instanceDefinition.getAppConfOperations().
+ getGlobalOptions().getOption(AgentKeys.PACKAGE_LIST, null);
+ if (pkgList != null) {
+ sliderFileSystem.verifyFileExistsInZip(appDefPath, pkgList);
+ }
String agentConf = instanceDefinition.getAppConfOperations().
getGlobalOptions().getOption(AgentKeys.AGENT_CONF, "");
[12/14] git commit: Merge branch 'develop' of
https://git-wip-us.apache.org/repos/asf/incubator-slider into develop
Posted by sm...@apache.org.
Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/05bf45d1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/05bf45d1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/05bf45d1
Branch: refs/heads/feature/SLIDER-181_Storm_Package
Commit: 05bf45d179cd52ea0d9de79ffd1a84dec90958fd
Parents: 6f912df c215045
Author: tedyu <yu...@gmail.com>
Authored: Fri Aug 8 11:24:31 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Fri Aug 8 11:24:31 2014 -0700
----------------------------------------------------------------------
.../apache/slider/common/tools/SliderUtils.java | 17 +-
.../providers/AbstractProviderService.java | 12 +-
.../slider/providers/ProviderService.java | 16 +-
.../providers/agent/AgentProviderService.java | 15 +-
.../providers/agent/HeartbeatMonitor.java | 9 +-
.../server/appmaster/AMViewForProviders.java | 46 -----
.../appmaster/ContainerStartOperation.java | 40 -----
.../server/appmaster/RoleLaunchService.java | 40 +++--
.../server/appmaster/SliderAppMaster.java | 134 ++++++++++----
.../server/appmaster/actions/ActionHalt.java | 41 +++++
.../appmaster/actions/ActionStartContainer.java | 47 +++++
.../appmaster/actions/ActionStopQueue.java | 36 ++++
.../appmaster/actions/ActionStopSlider.java | 39 ++++
.../server/appmaster/actions/AsyncAction.java | 124 +++++++++++++
.../actions/ProviderReportedContainerLoss.java | 47 +++++
.../actions/ProviderStartupCompleted.java | 37 ++++
.../server/appmaster/actions/QueueAccess.java | 35 ++++
.../server/appmaster/actions/QueueExecutor.java | 75 ++++++++
.../server/appmaster/actions/QueueService.java | 76 ++++++++
.../appmaster/actions/StartContainerAction.java | 41 +++++
.../AbstractWorkflowExecutorService.java | 110 ------------
.../services/workflow/ForkedProcessService.java | 6 +-
.../workflow/WorkflowCallbackService.java | 3 +-
.../workflow/WorkflowExecutorService.java | 113 ++++++++++++
.../WorkflowScheduledExecutorService.java | 38 ++++
.../server/services/workflow/package-info.java | 2 +-
.../standalone/TestDestroyStandaloneAM.groovy | 156 ----------------
.../standalone/TestKillStandaloneAM.groovy | 73 --------
.../standalone/TestRestartStandaloneAM.groovy | 96 ----------
.../standalone/TestStandaloneAMDestroy.groovy | 154 ++++++++++++++++
.../standalone/TestStandaloneAMKill.groovy | 73 ++++++++
.../standalone/TestStandaloneAMRestart.groovy | 96 ++++++++++
.../server/appmaster/actions/TestActions.groovy | 180 +++++++++++++++++++
.../model/history/TestRoleHistoryRW.groovy | 2 +-
.../model/mock/MockProviderService.groovy | 4 +-
.../providers/agent/TestHeartbeatMonitor.java | 6 -
.../publisher/TestAgentProviderService.java | 6 +-
.../workflow/TestWorkflowExecutorService.java | 9 +-
38 files changed, 1428 insertions(+), 626 deletions(-)
----------------------------------------------------------------------
[08/14] git commit: SLIDER-302 reorder constructors for IDE
subclassing ease
Posted by sm...@apache.org.
SLIDER-302 reorder constructors for IDE subclassing ease
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/cf2aa70e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/cf2aa70e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/cf2aa70e
Branch: refs/heads/feature/SLIDER-181_Storm_Package
Commit: cf2aa70e49d606a9c2bfee8194b84746166b835a
Parents: b7b9a15
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 8 19:16:55 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 8 19:16:55 2014 +0100
----------------------------------------------------------------------
.../slider/server/appmaster/actions/AsyncAction.java | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cf2aa70e/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
index a755064..5e325a0 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
@@ -43,6 +43,13 @@ public abstract class AsyncAction implements Delayed {
}
protected AsyncAction(String name,
+ int delayMillis) {
+ this.name = name;
+ this.nanos = convertAndOffset(delayMillis);
+ attrs = EnumSet.noneOf(ActionAttributes.class);
+ }
+
+ protected AsyncAction(String name,
int delayMillis, EnumSet<ActionAttributes> attrs) {
this.name = name;
this.nanos = convertAndOffset(delayMillis);
@@ -56,13 +63,6 @@ public abstract class AsyncAction implements Delayed {
Collections.addAll(attrs, attributes);
}
- protected AsyncAction(String name,
- int delayMillis) {
- this.name = name;
- this.nanos = convertAndOffset(delayMillis);
- attrs = EnumSet.noneOf(ActionAttributes.class);
- }
-
private long convertAndOffset(int delay) {
return now() + TimeUnit.NANOSECONDS.convert(delay, TimeUnit.MILLISECONDS);
}
[03/14] git commit: SLIDER-302: propagation working with correct
ordering
Posted by sm...@apache.org.
SLIDER-302: propagation working with correct ordering
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3d02e1cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3d02e1cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3d02e1cb
Branch: refs/heads/feature/SLIDER-181_Storm_Package
Commit: 3d02e1cbb072a795b58ac7e263b30d7370f5b133
Parents: 7ef046b
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 8 17:07:24 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 8 17:07:24 2014 +0100
----------------------------------------------------------------------
.../slider/server/appmaster/actions/QueueService.java | 10 +++++++---
.../slider/server/appmaster/actions/TestActions.groovy | 12 ++++++------
2 files changed, 13 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3d02e1cb/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
index dd39de0..69cb9f7 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
@@ -41,11 +41,13 @@ public class QueueService extends AbstractService
super("action queue");
}
- public void putImmediately(AsyncAction action) {
+ public void put(AsyncAction action) {
+ log.debug("Queueing {}", action);
actionQueue.add(action);
}
public void putDelayed(AsyncAction action) {
+ log.debug("Delayed Queueing {}", action);
delayedActions.add(action);
}
@@ -56,13 +58,15 @@ public class QueueService extends AbstractService
public void run() {
try {
- log.info("ActionExecutor Queue terminated");
+ log.info("QueueService processor started");
AsyncAction take;
do {
take = delayedActions.take();
- actionQueue.push(take);
+ log.debug("Propagating {}", take);
+ actionQueue.put(take);
} while (!(take instanceof ActionStopQueue));
+ log.info("QueueService processor terminated");
} catch (InterruptedException e) {
//game over
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3d02e1cb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
index 24d4c40..7e6338b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
@@ -103,8 +103,8 @@ class TestActions {
@Test
public void testImmediateQueue() throws Throwable {
ActionNoteExecuted noteExecuted = new ActionNoteExecuted("executed", 0)
- queues.putImmediately(noteExecuted)
- queues.putImmediately(new ActionStopQueue(0))
+ queues.put(noteExecuted)
+ queues.put(new ActionStopQueue(0))
QueueExecutor ex = new QueueExecutor(queues)
ex.run();
assert queues.actionQueue.empty
@@ -120,9 +120,9 @@ class TestActions {
List<AsyncAction> actions = [note1, stop, note2]
Collections.sort(actions)
- assert actions.get(0) == note1
- assert actions.get(1) == note2
- assert actions.get(3) == stop
+ assert actions[0] == note1
+ assert actions[1] == note2
+ assert actions[2] == stop
}
@Test
@@ -136,8 +136,8 @@ class TestActions {
assert note1.nanos < note2.nanos
assert note2.nanos < stop.nanos
queues.putDelayed(note1)
- queues.putDelayed(stop)
queues.putDelayed(note2)
+ queues.putDelayed(stop)
// async to sync expected to run in order
queues.run();
assert queues.delayedActions.empty
[14/14] git commit: Merge branch 'develop' into
feature/SLIDER-181_Storm_Package
Posted by sm...@apache.org.
Merge branch 'develop' into feature/SLIDER-181_Storm_Package
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ee8bf826
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ee8bf826
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ee8bf826
Branch: refs/heads/feature/SLIDER-181_Storm_Package
Commit: ee8bf826ba27418052c2262c14e20c2bb530fffa
Parents: bc4a5ac e464bba
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sat Aug 9 14:51:05 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sat Aug 9 14:51:05 2014 -0700
----------------------------------------------------------------------
app-packages/hbase/appConfig.json | 1 +
app-packages/hbase/get-hbase-site.sh | 26 +--
app-packages/hbase/links.py | 35 ++++
app-packages/hbase/metainfo.xml | 15 ++
.../hbase/package/scripts/hbase_service.py | 2 +
.../hbase/package/scripts/hbase_thrift2.py | 62 +++++++
app-packages/hbase/package/scripts/params.py | 1 +
app-packages/hbase/resources.json | 9 +-
.../slider/common/tools/CoreFileSystem.java | 34 ++++
.../apache/slider/common/tools/SliderUtils.java | 17 +-
.../providers/AbstractProviderService.java | 12 +-
.../slider/providers/ProviderService.java | 16 +-
.../providers/agent/AgentClientProvider.java | 6 +
.../providers/agent/AgentProviderService.java | 15 +-
.../providers/agent/HeartbeatMonitor.java | 9 +-
.../server/appmaster/AMViewForProviders.java | 46 -----
.../appmaster/ContainerStartOperation.java | 40 -----
.../server/appmaster/RoleLaunchService.java | 40 +++--
.../server/appmaster/SliderAppMaster.java | 134 ++++++++++----
.../server/appmaster/actions/ActionHalt.java | 41 +++++
.../appmaster/actions/ActionStartContainer.java | 47 +++++
.../appmaster/actions/ActionStopQueue.java | 36 ++++
.../appmaster/actions/ActionStopSlider.java | 39 ++++
.../server/appmaster/actions/AsyncAction.java | 124 +++++++++++++
.../actions/ProviderReportedContainerLoss.java | 47 +++++
.../actions/ProviderStartupCompleted.java | 37 ++++
.../server/appmaster/actions/QueueAccess.java | 35 ++++
.../server/appmaster/actions/QueueExecutor.java | 75 ++++++++
.../server/appmaster/actions/QueueService.java | 76 ++++++++
.../appmaster/actions/StartContainerAction.java | 41 +++++
.../AbstractWorkflowExecutorService.java | 110 ------------
.../services/workflow/ForkedProcessService.java | 6 +-
.../workflow/WorkflowCallbackService.java | 3 +-
.../workflow/WorkflowExecutorService.java | 113 ++++++++++++
.../WorkflowScheduledExecutorService.java | 38 ++++
.../server/services/workflow/package-info.java | 2 +-
.../standalone/TestDestroyStandaloneAM.groovy | 156 ----------------
.../standalone/TestKillStandaloneAM.groovy | 73 --------
.../standalone/TestRestartStandaloneAM.groovy | 96 ----------
.../standalone/TestStandaloneAMDestroy.groovy | 154 ++++++++++++++++
.../standalone/TestStandaloneAMKill.groovy | 73 ++++++++
.../standalone/TestStandaloneAMRestart.groovy | 96 ++++++++++
.../server/appmaster/actions/TestActions.groovy | 180 +++++++++++++++++++
.../model/history/TestRoleHistoryRW.groovy | 2 +-
.../model/mock/MockProviderService.groovy | 4 +-
.../providers/agent/TestHeartbeatMonitor.java | 6 -
.../publisher/TestAgentProviderService.java | 6 +-
.../workflow/TestWorkflowExecutorService.java | 9 +-
.../providers/hbase/HBaseClientProvider.java | 4 +
.../slider/providers/hbase/HBaseKeys.java | 5 +
.../providers/hbase/HBaseProviderService.java | 5 +
.../slider/providers/hbase/HBaseRoles.java | 2 +
52 files changed, 1616 insertions(+), 645 deletions(-)
----------------------------------------------------------------------
[06/14] git commit: SLIDER-303 Enhance get-hbase-site.sh to obtain
information about gateways
Posted by sm...@apache.org.
SLIDER-303 Enhance get-hbase-site.sh to obtain information about gateways
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a1bbf4db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a1bbf4db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a1bbf4db
Branch: refs/heads/feature/SLIDER-181_Storm_Package
Commit: a1bbf4dba7613ca2d8b0942ed2b3df3a8bebe17a
Parents: 66cbd37
Author: tedyu <yu...@gmail.com>
Authored: Fri Aug 8 10:45:48 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Fri Aug 8 10:45:48 2014 -0700
----------------------------------------------------------------------
app-packages/hbase/get-hbase-site.sh | 26 +++++++++-----------------
app-packages/hbase/links.py | 15 +++++++++++++++
2 files changed, 24 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a1bbf4db/app-packages/hbase/get-hbase-site.sh
----------------------------------------------------------------------
diff --git a/app-packages/hbase/get-hbase-site.sh b/app-packages/hbase/get-hbase-site.sh
index 04f9d3d..5211d83 100755
--- a/app-packages/hbase/get-hbase-site.sh
+++ b/app-packages/hbase/get-hbase-site.sh
@@ -1,25 +1,17 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
tuple=`slider status $1 | grep "info.am.web.url"`
+echo $tuple
FS=":"
url=`echo $tuple | awk '{split($0,array,": ")} END{print array[2]}'`
url="${url%,}"
url="${url%\"}"
url="${url#\"}"
-url="${url}ws/v1/slider/publisher/slider/hbase-site.xml"
-curl -k -o hbase-site.dnld $url
+siteurl="${url}ws/v1/slider/publisher/slider/hbase-site.xml"
+curl -k -o hbase-site.dnld $siteurl
grep -v 'hbase.tmp.dir' hbase-site.dnld > hbase-site.xml
+
+linksurl="${url}ws/v1/slider/publisher/slider/quicklinks"
+curl -k -o links.json $linksurl
+python $DIR/links.py
+#| sed -e 's/\/\///g' | awk 'BEGIN { FS = ":" } ; { print $2 }'
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a1bbf4db/app-packages/hbase/links.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/links.py b/app-packages/hbase/links.py
new file mode 100644
index 0000000..c77e620
--- /dev/null
+++ b/app-packages/hbase/links.py
@@ -0,0 +1,15 @@
+from __future__ import print_function
+import logging
+import json
+
+file = open("links.json")
+links = json.load(file)
+file.close()
+if links.has_key("entries"):
+ entries = links["entries"]
+ if entries.has_key("org.apache.slider.hbase.rest"):
+ print("org.apache.slider.hbase.rest : %s" % entries["org.apache.slider.hbase.rest"])
+ if entries.has_key("org.apache.slider.hbase.thrift"):
+ print("org.apache.slider.hbase.thrift : %s" % entries["org.apache.slider.hbase.thrift"])
+ if entries.has_key("org.apache.slider.hbase.thrift2"):
+ print("org.apache.slider.hbase.thrift2 : %s" % entries["org.apache.slider.hbase.thrift2"])
[04/14] git commit: SLIDER-302 role launch moves to async model
Posted by sm...@apache.org.
SLIDER-302 role launch moves to async model
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e75f605f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e75f605f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e75f605f
Branch: refs/heads/feature/SLIDER-181_Storm_Package
Commit: e75f605fb569eacab9d8e9e449e8f764e9fec13e
Parents: 3d02e1c
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 8 17:21:56 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 8 17:21:56 2014 +0100
----------------------------------------------------------------------
.../server/appmaster/RoleLaunchService.java | 18 +++++-----
.../server/appmaster/SliderAppMaster.java | 3 +-
.../server/appmaster/actions/QueueAccess.java | 35 ++++++++++++++++++++
.../server/appmaster/actions/QueueService.java | 4 ++-
4 files changed, 47 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e75f605f/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
index b83c905..e8b6802 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
@@ -30,6 +30,7 @@ import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.actions.ActionStartContainer;
import org.apache.slider.server.appmaster.actions.AsyncAction;
+import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.RoleStatus;
import org.apache.slider.server.services.workflow.WorkflowExecutorService;
@@ -52,13 +53,11 @@ public class RoleLaunchService
public static final String ROLE_LAUNCH_SERVICE = "RoleLaunchService";
+
/**
- * Callback to whatever has the task of actually running the container
- * start operation
+ * Queue submission API
*/
- private final ContainerStartOperation containerStarter;
-
- private final Queue<AsyncAction> actionQueue;
+ private final QueueAccess actionQueue;
/**
* Provider bulding up the command
@@ -93,15 +92,14 @@ public class RoleLaunchService
* @param envVars environment variables
* @param launcherTmpDirPath path for a temporary data in the launch process
*/
- public RoleLaunchService(ContainerStartOperation startOperation,
- Queue<AsyncAction> actionQueue, ProviderService provider,
+ public RoleLaunchService(QueueAccess queueAccess,
+ ProviderService provider,
SliderFileSystem fs,
Path generatedConfDirPath,
Map<String, String> envVars,
Path launcherTmpDirPath) {
super(ROLE_LAUNCH_SERVICE);
- containerStarter = startOperation;
- this.actionQueue = actionQueue;
+ this.actionQueue = queueAccess;
this.fs = fs;
this.generatedConfDirPath = generatedConfDirPath;
this.launcherTmpDirPath = launcherTmpDirPath;
@@ -216,7 +214,7 @@ public class RoleLaunchService
instance.role = containerRole;
instance.roleId = role.id;
instance.environment = envDescription;
- actionQueue.add(new ActionStartContainer("starting " + containerRole,
+ actionQueue.put(new ActionStartContainer("starting " + containerRole,
0, container,
containerLauncher.completeContainerLaunch(),
instance));
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e75f605f/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index b1ac6bb..b82967f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -750,8 +750,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
fs.getFileSystem().mkdirs(launcherTmpDirPath);
//launcher service
- launchService = new RoleLaunchService(this,
- actionQueues.actionQueue,
+ launchService = new RoleLaunchService(actionQueues,
providerService,
fs,
new Path(getGeneratedConfDir()),
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e75f605f/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
new file mode 100644
index 0000000..f717085
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+public interface QueueAccess {
+ /**
+ * Put an action on the immediate queue -to be executed when the queue
+ * reaches it.
+ * @param action action to queue
+ */
+ void put(AsyncAction action);
+
+ /**
+ * Put a delayed action: this will only be added to the main queue
+ * after its action time has been reached
+ * @param action action to queue
+ */
+ void putDelayed(AsyncAction action);
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e75f605f/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
index 69cb9f7..9e4e8d0 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
@@ -28,7 +28,7 @@ import java.util.concurrent.DelayQueue;
import java.util.concurrent.LinkedBlockingDeque;
public class QueueService extends AbstractService
- implements Runnable {
+ implements Runnable, QueueAccess {
private static final Logger log =
LoggerFactory.getLogger(QueueService.class);
@@ -41,11 +41,13 @@ public class QueueService extends AbstractService
super("action queue");
}
+ @Override
public void put(AsyncAction action) {
log.debug("Queueing {}", action);
actionQueue.add(action);
}
+ @Override
public void putDelayed(AsyncAction action) {
log.debug("Delayed Queueing {}", action);
delayedActions.add(action);
[05/14] git commit: SLIDER-302 providers switch to using queued
action, methods and interfaces changed appropriately
Posted by sm...@apache.org.
SLIDER-302 providers switch to using queued action, methods and interfaces changed appropriately
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8f777f31
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8f777f31
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8f777f31
Branch: refs/heads/feature/SLIDER-181_Storm_Package
Commit: 8f777f31ceb46e7b84a32524789b48664080bf41
Parents: e75f605
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 8 17:53:21 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 8 17:53:21 2014 +0100
----------------------------------------------------------------------
.../providers/AbstractProviderService.java | 12 ++---
.../slider/providers/ProviderService.java | 16 ++-----
.../providers/agent/AgentProviderService.java | 15 +++----
.../providers/agent/HeartbeatMonitor.java | 9 +---
.../server/appmaster/AMViewForProviders.java | 46 -------------------
.../appmaster/ContainerStartOperation.java | 40 -----------------
.../server/appmaster/SliderAppMaster.java | 39 +++++++++-------
.../server/appmaster/actions/ActionHalt.java | 4 +-
.../appmaster/actions/ActionStartContainer.java | 4 +-
.../appmaster/actions/ActionStopQueue.java | 4 +-
.../appmaster/actions/ActionStopSlider.java | 4 +-
.../server/appmaster/actions/AsyncAction.java | 2 +-
.../actions/ProviderReportedContainerLoss.java | 47 ++++++++++++++++++++
.../server/appmaster/actions/QueueExecutor.java | 14 +++---
.../server/appmaster/actions/TestActions.groovy | 2 +-
.../model/mock/MockProviderService.groovy | 4 +-
.../providers/agent/TestHeartbeatMonitor.java | 6 ---
.../publisher/TestAgentProviderService.java | 6 +--
18 files changed, 106 insertions(+), 168 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index b431a10..89b862e 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -33,7 +33,7 @@ import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.main.ExitCodeProvider;
import org.apache.slider.core.registry.info.RegisteredEndpoint;
import org.apache.slider.core.registry.info.ServiceInstanceData;
-import org.apache.slider.server.appmaster.AMViewForProviders;
+import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
@@ -71,8 +71,8 @@ public abstract class AbstractProviderService
protected AgentRestOperations restOps;
protected RegistryViewForProviders registry;
protected ServiceInstanceData registryInstanceData;
- protected AMViewForProviders amView;
protected URL amWebAPI;
+ protected QueueAccess queueAccess;
public AbstractProviderService(String name) {
super(name);
@@ -87,8 +87,8 @@ public abstract class AbstractProviderService
return amState;
}
- public AMViewForProviders getAppMaster() {
- return amView;
+ public QueueAccess getQueueAccess() {
+ return queueAccess;
}
public void setAmState(StateAccessForProviders amState) {
@@ -98,11 +98,11 @@ public abstract class AbstractProviderService
@Override
public void bind(StateAccessForProviders stateAccessor,
RegistryViewForProviders reg,
- AMViewForProviders amView,
+ QueueAccess queueAccess,
List<Container> liveContainers) {
this.amState = stateAccessor;
this.registry = reg;
- this.amView = amView;
+ this.queueAccess = queueAccess;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index 1466350..e717158 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -32,7 +32,7 @@ import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.core.main.ExitCodeProvider;
import org.apache.slider.core.registry.info.ServiceInstanceData;
-import org.apache.slider.server.appmaster.AMViewForProviders;
+import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
@@ -146,17 +146,9 @@ public interface ProviderService extends ProviderCore,
*/
Map<String, String> buildMonitorDetails(ClusterDescription clusterSpec);
- /**
- * bind operation -invoked before the service is started
- * @param stateAccessor interface offering read access to the state
- * @param registry registry view
- * @param amView AM access
- * @param liveContainers list of live containers. If non-empty, the AM was
- * restarted with one more running containers.
- */
- void bind(StateAccessForProviders stateAccessor,
- RegistryViewForProviders registry,
- AMViewForProviders amView,
+ public void bind(StateAccessForProviders stateAccessor,
+ RegistryViewForProviders reg,
+ QueueAccess queueAccess,
List<Container> liveContainers);
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 4d54509..419a454 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -58,7 +58,7 @@ import org.apache.slider.providers.agent.application.metadata.ExportGroup;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.providers.agent.application.metadata.OSPackage;
import org.apache.slider.providers.agent.application.metadata.OSSpecific;
-import org.apache.slider.server.appmaster.AMViewForProviders;
+import org.apache.slider.server.appmaster.actions.ProviderReportedContainerLoss;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
@@ -591,19 +591,16 @@ public class AgentProviderService extends AbstractProviderService implements
/**
* Lost heartbeat from the container - release it and ask for a replacement
- *
- *
- * @param label
+ * (async operation)
+ * @param label
* @param containerId
*
- * @return outcome of the operation
*/
- protected AMViewForProviders.ContainerLossReportOutcomes lostContainer(
+ protected void lostContainer(
String label,
- ContainerId containerId) throws
- SliderException {
+ ContainerId containerId) {
getComponentStatuses().remove(label);
- return getAppMaster().providerLostContainer(containerId);
+ getQueueAccess().put(new ProviderReportedContainerLoss(containerId));
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java b/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java
index cb22d2e..0a1beca 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java
@@ -19,7 +19,6 @@ package org.apache.slider.providers.agent;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.core.exceptions.SliderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -114,13 +113,7 @@ public class HeartbeatMonitor implements Runnable {
timeSinceLastHeartbeat);
ContainerId containerId =
componentInstanceState.getContainerId();
- try {
- provider.lostContainer(containerLabel, containerId);
- } catch (SliderException e) {
- log.info(
- "Error while requesting container release for {}. Message: {}",
- containerId, e, e);
- }
+ provider.lostContainer(containerLabel, containerId);
}
break;
case HEARTBEAT_LOST:
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/server/appmaster/AMViewForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/AMViewForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/AMViewForProviders.java
deleted file mode 100644
index f426551..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/AMViewForProviders.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.core.exceptions.SliderException;
-
-/** Operations available to a provider from AppMaster */
-public interface AMViewForProviders {
-
- /**
- * Outcomes from container loss
- */
- enum ContainerLossReportOutcomes {
- /**
- * The container doesn't exist...either it wasn't in use or it
- * has been released
- */
- CONTAINER_NOT_IN_USE,
-
- /**
- * The container is known about and a review has been initated
- */
- CONTAINER_LOST_REVIEW_INITIATED,
- }
-
- /** Provider can ask AppMaster to release a specific container */
- ContainerLossReportOutcomes providerLostContainer(ContainerId containerId) throws SliderException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/server/appmaster/ContainerStartOperation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/ContainerStartOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/ContainerStartOperation.java
deleted file mode 100644
index 50c99f3..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/ContainerStartOperation.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-
-/**
- * Callback for container start requests
- */
-public interface ContainerStartOperation {
- /**
- * Add a node to the list of starting
- * nodes then trigger the NM start operation with the given
- * launch context
- * @param container container
- * @param ctx context
- * @param instance node details
- */
- void startContainer(Container container,
- ContainerLaunchContext ctx,
- RoleInstance instance) ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index b82967f..52f7f8f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -80,6 +80,7 @@ import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.exceptions.SliderInternalStateException;
import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
+import org.apache.slider.core.main.ExitCodeProvider;
import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.slider.core.main.RunService;
import org.apache.slider.core.main.ServiceLauncher;
@@ -168,9 +169,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
SliderClusterProtocol,
ServiceStateChangeListener,
RoleKeys,
- ProviderCompleted,
- ContainerStartOperation,
- AMViewForProviders {
+ ProviderCompleted {
protected static final Logger log =
LoggerFactory.getLogger(SliderAppMaster.class);
@@ -763,9 +762,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//Give the provider restricted access to the state, registry
- providerService.bind(stateForProviders, registry, this,
+ providerService.bind(stateForProviders, registry, actionQueues,
liveContainers);
- sliderAMProvider.bind(stateForProviders, registry, this,
+ sliderAMProvider.bind(stateForProviders, registry, actionQueues,
liveContainers);
// now do the registration
@@ -1219,7 +1218,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
public void onError(Throwable e) {
//callback says it's time to finish
LOG_YARN.error("AMRMClientAsync.onError() received " + e, e);
- signalAMComplete(EXIT_EXCEPTION_THROWN, "AMRMClientAsync.onError() received " + e);
+ signalAMComplete(EXIT_EXCEPTION_THROWN,
+ "AMRMClientAsync.onError() received " + e);
}
/* =================================================================== */
@@ -1484,18 +1484,13 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
}
-
- /* =================================================================== */
- /* ProviderAMOperations */
- /* =================================================================== */
-
/**
* report container loss. If this isn't already known about, react
*
* @param containerId id of the container which has failed
* @throws SliderException
*/
- public synchronized ContainerLossReportOutcomes providerLostContainer(
+ public synchronized void providerLostContainer(
ContainerId containerId)
throws SliderException {
log.info("containerLostContactWithProvider: container {} lost",
@@ -1506,10 +1501,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// ask for more containers if needed
log.info("Container released; triggering review");
reviewRequestAndReleaseNodes();
- return ContainerLossReportOutcomes.CONTAINER_LOST_REVIEW_INITIATED;
} else {
log.info("Container not in active set - ignoring");
- return ContainerLossReportOutcomes.CONTAINER_NOT_IN_USE;
}
}
@@ -1570,7 +1563,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* @param ctx context
* @param instance node details
*/
- @Override // ContainerStartOperation
public void startContainer(Container container,
ContainerLaunchContext ctx,
RoleInstance instance) {
@@ -1676,6 +1668,22 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
actionQueues.delayedActions.add(action);
}
+
+ /**
+ * Handle any exception in a thread. If the exception provides an exit
+ * code, that is the one that will be used
+ * @param thread thread throwing the exception
+ * @param exception exception
+ */
+ public void onExceptionInThread(Thread thread, Exception exception) {
+ log.error("Exception in {}: {}", thread.getName(), exception, exception);
+ int exitCode = EXIT_EXCEPTION_THROWN;
+ if (exception instanceof ExitCodeProvider) {
+ exitCode = ((ExitCodeProvider) exception).getExitCode();
+ }
+ signalAMComplete(exitCode, exception.toString());
+ }
+
/**
* This is the main entry point for the service launcher.
* @param args command line arguments.
@@ -1691,4 +1699,5 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//now have the service launcher do its work
ServiceLauncher.serviceMain(extendedArgs);
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
index a16d207..1468cd8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
@@ -21,8 +21,6 @@ package org.apache.slider.server.appmaster.actions;
import org.apache.hadoop.util.ExitUtil;
import org.apache.slider.server.appmaster.SliderAppMaster;
-import java.io.IOException;
-
public class ActionHalt extends AsyncAction {
private final int status;
@@ -37,7 +35,7 @@ public class ActionHalt extends AsyncAction {
}
@Override
- public void execute(SliderAppMaster appMaster) throws IOException {
+ public void execute(SliderAppMaster appMaster) throws Exception {
ExitUtil.halt(status, text);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
index 2d643ed..6179f35 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
@@ -23,8 +23,6 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.state.RoleInstance;
-import java.io.IOException;
-
public class ActionStartContainer extends AsyncAction {
private final Container container;
@@ -43,7 +41,7 @@ public class ActionStartContainer extends AsyncAction {
}
@Override
- public void execute(SliderAppMaster appMaster) throws IOException {
+ public void execute(SliderAppMaster appMaster) throws Exception {
appMaster.startContainer(container, ctx, instance);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
index b64fd73..a11bccb 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
@@ -20,8 +20,6 @@ package org.apache.slider.server.appmaster.actions;
import org.apache.slider.server.appmaster.SliderAppMaster;
-import java.io.IOException;
-
/**
* Action to tell a queue executor to stop -after handing this on/executing it
*/
@@ -32,7 +30,7 @@ public class ActionStopQueue extends AsyncAction {
}
@Override
- public void execute(SliderAppMaster appMaster) throws IOException {
+ public void execute(SliderAppMaster appMaster) throws Exception {
// no-op
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
index cffa3f2..0e6dc25 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
@@ -21,8 +21,6 @@ package org.apache.slider.server.appmaster.actions;
import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.slider.server.appmaster.SliderAppMaster;
-import java.io.IOException;
-
public class ActionStopSlider extends AsyncAction {
public ActionStopSlider(String message,
int delay) {
@@ -30,7 +28,7 @@ public class ActionStopSlider extends AsyncAction {
}
@Override
- public void execute(SliderAppMaster appMaster) throws IOException {
+ public void execute(SliderAppMaster appMaster) throws Exception {
String message = name;
SliderAppMaster.getLog().info("SliderAppMasterApi.stopCluster: {}",
message);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
index 09bb517..a755064 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
@@ -112,7 +112,7 @@ public abstract class AsyncAction implements Delayed {
* @param appMaster
* @throws IOException
*/
- public abstract void execute(SliderAppMaster appMaster) throws IOException;
+ public abstract void execute(SliderAppMaster appMaster) throws Exception;
public enum ActionAttributes {
SHRINKS_CLUSTER,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
new file mode 100644
index 0000000..63b40d5
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+
+/**
+ * Report container loss to the AM
+ * {@link SliderAppMaster#providerLostContainer(ContainerId)}
+ */
+public class ProviderReportedContainerLoss extends AsyncAction {
+
+ private final ContainerId containerId;
+
+ public ProviderReportedContainerLoss(ContainerId containerId) {
+ super("lost container " + containerId);
+ this.containerId = containerId;
+ }
+
+ public ProviderReportedContainerLoss(
+ ContainerId containerId, int delayMillis) {
+ super("lost container " + containerId, delayMillis);
+ this.containerId = containerId;
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster) throws Exception {
+ appMaster.providerLostContainer(containerId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
index 7eeaddf..072f324 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
@@ -24,8 +24,6 @@ import org.apache.slider.server.appmaster.SliderAppMaster;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-
/**
* Executor for async actions - hands them off to the AM as
* appropriate
@@ -41,7 +39,7 @@ public class QueueExecutor implements Runnable {
QueueService actionQueues) {
Preconditions.checkNotNull(appMaster);
Preconditions.checkNotNull(actionQueues);
-
+
this.appMaster = appMaster;
this.actionQueues = actionQueues;
}
@@ -52,7 +50,7 @@ public class QueueExecutor implements Runnable {
this.appMaster = null;
this.actionQueues = actionQueues;
}
-
+
/**
* Run until the queue has been told to stop
*/
@@ -67,9 +65,11 @@ public class QueueExecutor implements Runnable {
take.execute(appMaster);
} while (!(take instanceof ActionStopQueue));
log.info("Queue Executor run() stopped");
- } catch (InterruptedException | IOException e) {
- log.error("Exception processing {}: {}", take, e, e);
-
+ } catch (Exception e) {
+ log.error("Exception processing {}: {}", take, e, e);
+ if (appMaster != null) {
+ appMaster.onExceptionInThread(Thread.currentThread(), e);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
index 7e6338b..5dffea0 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
@@ -159,7 +159,7 @@ class TestActions {
}
@Override
- public void execute(SliderAppMaster appMaster) throws IOException {
+ public void execute(SliderAppMaster appMaster) throws Exception {
log.info("Executing $name");
executed.set(true);
executionTimeNanos.set(System.nanoTime())
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
index c46b828..458cfac 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
@@ -35,7 +35,7 @@ import org.apache.slider.core.launch.ContainerLauncher
import org.apache.slider.core.registry.info.ServiceInstanceData
import org.apache.slider.providers.ProviderRole
import org.apache.slider.providers.ProviderService
-import org.apache.slider.server.appmaster.AMViewForProviders
+import org.apache.slider.server.appmaster.actions.QueueAccess
import org.apache.slider.server.appmaster.state.ContainerReleaseSelector
import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector
import org.apache.slider.server.appmaster.state.StateAccessForProviders
@@ -202,7 +202,7 @@ class MockProviderService implements ProviderService {
void bind(
StateAccessForProviders stateAccessor,
RegistryViewForProviders registry,
- AMViewForProviders amView,
+ QueueAccess queueAccess,
List<Container> liveContainers) {
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
index b16b145..1ace44e 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
@@ -17,7 +17,6 @@
package org.apache.slider.providers.agent;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.server.appmaster.AMViewForProviders;
import org.apache.slider.server.appmaster.model.mock.MockContainerId;
import org.junit.Assert;
import org.junit.Test;
@@ -98,11 +97,6 @@ public class TestHeartbeatMonitor {
slaveState.setState(State.STARTED);
slaveState.heartbeat(now);
expect(provider.getComponentStatuses()).andReturn(statuses).anyTimes();
- expect(provider.lostContainer(slaveLabel, slaveContainer)).andReturn(
- AMViewForProviders.ContainerLossReportOutcomes.CONTAINER_LOST_REVIEW_INITIATED).once();
- // expect the second iteration to report no container any more
- expect(provider.lostContainer(slaveLabel, slaveContainer)).andReturn(
- AMViewForProviders.ContainerLossReportOutcomes.CONTAINER_NOT_IN_USE).once();
replay(provider);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
index 5579592..a97e351 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
@@ -18,7 +18,7 @@ package org.apache.slider.server.appmaster.web.rest.publisher;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.slider.providers.agent.AgentProviderService;
-import org.apache.slider.server.appmaster.AMViewForProviders;
+import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.services.registry.RegistryViewForProviders;
import org.slf4j.Logger;
@@ -43,9 +43,9 @@ public class TestAgentProviderService extends AgentProviderService {
@Override
public void bind(StateAccessForProviders stateAccessor,
RegistryViewForProviders reg,
- AMViewForProviders amView,
+ QueueAccess queueAccess,
List<Container> liveContainers) {
- super.bind(stateAccessor, reg, amView, liveContainers);
+ super.bind(stateAccessor, reg, queueAccess, liveContainers);
Map<String,String> dummyProps = new HashMap<>();
dummyProps.put("prop1", "val1");
dummyProps.put("prop2", "val2");
[02/14] git commit: SLIDER-302 queue and actions,
tests not quite working
Posted by sm...@apache.org.
SLIDER-302 queue and actions, tests not quite working
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7ef046b6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7ef046b6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7ef046b6
Branch: refs/heads/feature/SLIDER-181_Storm_Package
Commit: 7ef046b61ca0230b1485b24eb9a4f1dc55ba7e0e
Parents: 66cbd37
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 8 14:17:38 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 8 16:53:50 2014 +0100
----------------------------------------------------------------------
.../apache/slider/common/tools/SliderUtils.java | 17 +-
.../server/appmaster/RoleLaunchService.java | 32 +++-
.../server/appmaster/SliderAppMaster.java | 94 +++++++---
.../server/appmaster/actions/ActionHalt.java | 43 +++++
.../appmaster/actions/ActionStartContainer.java | 49 +++++
.../appmaster/actions/ActionStopQueue.java | 38 ++++
.../appmaster/actions/ActionStopSlider.java | 41 +++++
.../server/appmaster/actions/AsyncAction.java | 124 +++++++++++++
.../server/appmaster/actions/QueueExecutor.java | 75 ++++++++
.../server/appmaster/actions/QueueService.java | 70 ++++++++
.../appmaster/actions/StartContainerAction.java | 41 +++++
.../AbstractWorkflowExecutorService.java | 110 ------------
.../services/workflow/ForkedProcessService.java | 6 +-
.../workflow/WorkflowCallbackService.java | 3 +-
.../workflow/WorkflowExecutorService.java | 113 ++++++++++++
.../WorkflowScheduledExecutorService.java | 38 ++++
.../server/services/workflow/package-info.java | 2 +-
.../standalone/TestDestroyStandaloneAM.groovy | 156 ----------------
.../standalone/TestKillStandaloneAM.groovy | 73 --------
.../standalone/TestRestartStandaloneAM.groovy | 96 ----------
.../standalone/TestStandaloneAMDestroy.groovy | 154 ++++++++++++++++
.../standalone/TestStandaloneAMKill.groovy | 73 ++++++++
.../standalone/TestStandaloneAMRestart.groovy | 96 ++++++++++
.../server/appmaster/actions/TestActions.groovy | 180 +++++++++++++++++++
.../model/history/TestRoleHistoryRW.groovy | 2 +-
.../workflow/TestWorkflowExecutorService.java | 9 +-
26 files changed, 1262 insertions(+), 473 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 8e412ee..cda0cf5 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -1073,7 +1073,7 @@ public final class SliderUtils {
UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
return currentUser;
} catch (IOException e) {
- log.info("Failed to grt user info", e);
+ log.info("Failed to get user info", e);
throw e;
}
}
@@ -1415,6 +1415,21 @@ public final class SliderUtils {
}
/**
+ * A compareTo function that converts the result of a long
+ * comparision into the integer that <code>Comparable</code>
+ * expects.
+ * @param left left side
+ * @param right right side
+ * @return -1, 0, 1 depending on the diff
+ */
+ public static int compareTo(long left, long right) {
+ long diff = left - right;
+ if (diff < 0) return -1;
+ if (diff > 0) return 1;
+ return 0;
+ }
+
+ /**
* This wrapps ApplicationReports and generates a string version
* iff the toString() operator is invoked
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
index 5a5baaa..b83c905 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
@@ -28,20 +28,25 @@ import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderService;
+import org.apache.slider.server.appmaster.actions.ActionStartContainer;
+import org.apache.slider.server.appmaster.actions.AsyncAction;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.apache.slider.server.services.workflow.AbstractWorkflowExecutorService;
+import org.apache.slider.server.services.workflow.WorkflowExecutorService;
import org.apache.slider.server.services.workflow.ServiceThreadFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Map;
+import java.util.Queue;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* A service for launching containers
*/
-public class RoleLaunchService extends AbstractWorkflowExecutorService {
+public class RoleLaunchService
+ extends WorkflowExecutorService<ExecutorService> {
protected static final Logger log =
LoggerFactory.getLogger(RoleLaunchService.class);
@@ -53,7 +58,13 @@ public class RoleLaunchService extends AbstractWorkflowExecutorService {
*/
private final ContainerStartOperation containerStarter;
+ private final Queue<AsyncAction> actionQueue;
+
+ /**
+ * Provider bulding up the command
+ */
private final ProviderService provider;
+
/**
* Filesystem to use for the launch
*/
@@ -75,6 +86,7 @@ public class RoleLaunchService extends AbstractWorkflowExecutorService {
/**
* Construct an instance of the launcher
* @param startOperation the callback to start the opreation
+ * @param actionQueue
* @param provider the provider
* @param fs filesystem
* @param generatedConfDirPath path in the FS for the generated dir
@@ -82,13 +94,14 @@ public class RoleLaunchService extends AbstractWorkflowExecutorService {
* @param launcherTmpDirPath path for a temporary data in the launch process
*/
public RoleLaunchService(ContainerStartOperation startOperation,
- ProviderService provider,
- SliderFileSystem fs,
- Path generatedConfDirPath,
- Map<String, String> envVars,
+ Queue<AsyncAction> actionQueue, ProviderService provider,
+ SliderFileSystem fs,
+ Path generatedConfDirPath,
+ Map<String, String> envVars,
Path launcherTmpDirPath) {
super(ROLE_LAUNCH_SERVICE);
containerStarter = startOperation;
+ this.actionQueue = actionQueue;
this.fs = fs;
this.generatedConfDirPath = generatedConfDirPath;
this.launcherTmpDirPath = launcherTmpDirPath;
@@ -203,9 +216,10 @@ public class RoleLaunchService extends AbstractWorkflowExecutorService {
instance.role = containerRole;
instance.roleId = role.id;
instance.environment = envDescription;
- containerStarter.startContainer(container,
- containerLauncher.completeContainerLaunch(),
- instance);
+ actionQueue.add(new ActionStartContainer("starting " + containerRole,
+ 0, container,
+ containerLauncher.completeContainerLaunch(),
+ instance));
} catch (Exception e) {
log.error("Exception thrown while trying to start {}: {}",
containerRole, e);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index bc63351..b1ac6bb 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -94,6 +94,11 @@ import org.apache.slider.providers.ProviderService;
import org.apache.slider.providers.SliderProviderFactory;
import org.apache.slider.providers.slideram.SliderAMClientProvider;
import org.apache.slider.providers.slideram.SliderAMProviderService;
+import org.apache.slider.server.appmaster.actions.QueueExecutor;
+import org.apache.slider.server.appmaster.actions.ActionHalt;
+import org.apache.slider.server.appmaster.actions.QueueService;
+import org.apache.slider.server.appmaster.actions.ActionStopSlider;
+import org.apache.slider.server.appmaster.actions.AsyncAction;
import org.apache.slider.server.appmaster.operations.AsyncRMOperationHandler;
import org.apache.slider.server.appmaster.operations.ProviderNotifyingOperationHandler;
import org.apache.slider.server.appmaster.rpc.RpcBinder;
@@ -120,7 +125,10 @@ import org.apache.slider.server.services.registry.SliderRegistryService;
import org.apache.slider.server.services.security.CertificateManager;
import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
import org.apache.slider.server.services.utility.WebAppService;
+import org.apache.slider.server.services.workflow.ServiceThreadFactory;
+import org.apache.slider.server.services.workflow.WorkflowExecutorService;
import org.apache.slider.server.services.workflow.WorkflowRpcService;
+import org.apache.slider.server.services.workflow.WorkflowScheduledExecutorService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -133,12 +141,14 @@ import java.net.URLClassLoader;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
@@ -183,6 +193,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
public static final int HEARTBEAT_INTERVAL = 1000;
public static final int NUM_RPC_HANDLERS = 5;
+ public static final int SCHEDULED_EXECUTOR_POOL_SIZE = 1;
/** YARN RPC to communicate with the Resource Manager or Node Manager */
private YarnRPC yarnRPC;
@@ -299,7 +310,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
*/
private int containerMaxCores;
-
/**
* limit container memory
*/
@@ -321,6 +331,14 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
private String agentAccessUrl;
private CertificateManager certificateManager;
+ private WorkflowScheduledExecutorService<ScheduledExecutorService>
+ scheduledExecutors;
+ private WorkflowExecutorService<ExecutorService>
+ executorService;
+
+
+ private final QueueService actionQueues = new QueueService();
+
/**
* Service Constructor
*/
@@ -328,9 +346,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
super(SERVICE_CLASSNAME_SHORT);
}
-
-
- /* =================================================================== */
+/* =================================================================== */
/* service lifecycle methods */
/* =================================================================== */
@@ -374,11 +390,33 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//look at settings of Hadoop Auth, to pick up a problem seen once
checkAndWarnForAuthTokenProblems();
+ scheduledExecutors =
+ new WorkflowScheduledExecutorService<>(
+ "AmExecutor",
+ Executors.newScheduledThreadPool(
+ SCHEDULED_EXECUTOR_POOL_SIZE,
+ new ServiceThreadFactory("AmScheduledExecutor", true)));
+ addService(scheduledExecutors);
+
+ executorService = new WorkflowExecutorService<>("AmExecutor",
+ Executors.newCachedThreadPool(
+ new ServiceThreadFactory("AmExecutor", true)));
+ addService(executorService);
+
+
+ addService(actionQueues);
//init all child services
super.serviceInit(conf);
}
+
+ @Override
+ protected void serviceStart() throws Exception {
+ super.serviceStart();
+ executorService.execute(new QueueExecutor(this, actionQueues));
+ executorService.execute(actionQueues);
+ }
-/* =================================================================== */
+ /* =================================================================== */
/* RunService methods called from ServiceLauncher */
/* =================================================================== */
@@ -713,6 +751,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//launcher service
launchService = new RoleLaunchService(this,
+ actionQueues.actionQueue,
providerService,
fs,
new Path(getGeneratedConfDir()),
@@ -893,6 +932,14 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
/**
+ * Get the AM log
+ * @return the log of the AM
+ */
+ public static Logger getLog() {
+ return log;
+ }
+
+ /**
* Block until it is signalled that the AM is done
*/
private void waitForAMCompletionSignal() {
@@ -1208,8 +1255,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
YarnException {
SliderUtils.getCurrentUser();
String message = request.getMessage();
- log.info("SliderAppMasterApi.stopCluster: {}",message);
- signalAMComplete(EXIT_CLIENT_INITIATED_SHUTDOWN, message);
+ log.info("SliderAppMasterApi.stopCluster: {}", message);
+ queue(new ActionStopSlider(message, 1000));
return Messages.StopClusterResponseProto.getDefaultInstance();
}
@@ -1369,17 +1416,16 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
@Override
- public Messages.AMSuicideResponseProto amSuicide(Messages.AMSuicideRequestProto request) throws
- IOException,
- YarnException {
+ public Messages.AMSuicideResponseProto amSuicide(
+ Messages.AMSuicideRequestProto request)
+ throws IOException, YarnException {
int signal = request.getSignal();
String text = request.getText();
int delay = request.getDelay();
log.info("AM Suicide with signal {}, message {} delay = {}", signal, text, delay);
- SliderUtils.haltAM(signal, text, delay);
- Messages.AMSuicideResponseProto.Builder builder =
- Messages.AMSuicideResponseProto.newBuilder();
- return builder.build();
+ ActionHalt action = new ActionHalt(signal, text, delay);
+ schedule(action);
+ return Messages.AMSuicideResponseProto.getDefaultInstance();
}
/* =================================================================== */
@@ -1576,7 +1622,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
public void onContainerStatusReceived(ContainerId containerId,
ContainerStatus containerStatus) {
LOG_YARN.debug("Container Status: id={}, status={}", containerId,
- containerStatus);
+ containerStatus);
}
@Override // NMClientAsync.CallbackHandler
@@ -1616,11 +1662,19 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
/**
- * Get the username for the slider cluster as set in the environment
- * @return the username or null if none was set/it is a secure cluster
+ * Queue an action for immediate execution in the executor thread
+ * @param action action to execute
+ */
+ public void queue(AsyncAction action) {
+ actionQueues.actionQueue.add(action);
+ }
+
+ /**
+ * Schedule an action
+ * @param action for delayed execution
*/
- public String getHadoop_user_name() {
- return hadoop_user_name;
+ public void schedule(AsyncAction action) {
+ actionQueues.delayedActions.add(action);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
new file mode 100644
index 0000000..a16d207
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+
+import java.io.IOException;
+
+public class ActionHalt extends AsyncAction {
+
+ private final int status;
+ private final String text;
+
+ public ActionHalt(
+ int status,
+ String text, int delay) {
+ super("Halt", delay, ActionAttributes.HALTS_CLUSTER);
+ this.status = status;
+ this.text = text;
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster) throws IOException {
+ ExitUtil.halt(status, text);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
new file mode 100644
index 0000000..2d643ed
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+
+import java.io.IOException;
+
+public class ActionStartContainer extends AsyncAction {
+
+ private final Container container;
+ private final ContainerLaunchContext ctx;
+ private final RoleInstance instance;
+
+ public ActionStartContainer(String name,
+ int delay,
+ Container container,
+ ContainerLaunchContext ctx,
+ RoleInstance instance) {
+ super(name, delay);
+ this.container = container;
+ this.ctx = ctx;
+ this.instance = instance;
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster) throws IOException {
+ appMaster.startContainer(container, ctx, instance);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
new file mode 100644
index 0000000..b64fd73
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.slider.server.appmaster.SliderAppMaster;
+
+import java.io.IOException;
+
+/**
+ * Action to tell a queue executor to stop -after handing this on/executing it
+ */
+public class ActionStopQueue extends AsyncAction {
+
+ public ActionStopQueue(int delay) {
+ super("stop queue", delay);
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster) throws IOException {
+ // no-op
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
new file mode 100644
index 0000000..cffa3f2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.slider.core.main.LauncherExitCodes;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+
+import java.io.IOException;
+
+public class ActionStopSlider extends AsyncAction {
+ public ActionStopSlider(String message,
+ int delay) {
+ super(message, delay, ActionAttributes.HALTS_CLUSTER);
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster) throws IOException {
+ String message = name;
+ SliderAppMaster.getLog().info("SliderAppMasterApi.stopCluster: {}",
+ message);
+ appMaster.signalAMComplete(
+ LauncherExitCodes.EXIT_CLIENT_INITIATED_SHUTDOWN,
+ message);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
new file mode 100644
index 0000000..09bb517
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.concurrent.Delayed;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+public abstract class AsyncAction implements Delayed {
+
+ private static final AtomicLong sequencer = new AtomicLong(0);
+
+ public final String name;
+ public final long nanos;
+ private final EnumSet<ActionAttributes> attrs;
+ private final long sequenceNumber = sequencer.incrementAndGet();
+
+
+ protected AsyncAction(String name) {
+ this(name, 0);
+ }
+
+ protected AsyncAction(String name,
+ int delayMillis, EnumSet<ActionAttributes> attrs) {
+ this.name = name;
+ this.nanos = convertAndOffset(delayMillis);
+ this.attrs = attrs;
+ }
+
+ protected AsyncAction(String name,
+ int delayMillis,
+ ActionAttributes... attributes) {
+ this(name, delayMillis);
+ Collections.addAll(attrs, attributes);
+ }
+
+ protected AsyncAction(String name,
+ int delayMillis) {
+ this.name = name;
+ this.nanos = convertAndOffset(delayMillis);
+ attrs = EnumSet.noneOf(ActionAttributes.class);
+ }
+
+ private long convertAndOffset(int delay) {
+ return now() + TimeUnit.NANOSECONDS.convert(delay, TimeUnit.MILLISECONDS);
+ }
+
+ protected long now() {
+ return System.nanoTime();
+ }
+
+ @Override
+ public long getDelay(TimeUnit unit) {
+ return unit.convert(nanos - now(), TimeUnit.NANOSECONDS);
+ }
+
+ @Override
+ public int compareTo(Delayed that) {
+ if (this == that) {
+ return 0;
+ }
+ return SliderUtils.compareTo(
+ getDelay(TimeUnit.NANOSECONDS),
+ that.getDelay(TimeUnit.NANOSECONDS));
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder(super.toString());
+ sb.append(" name='").append(name).append('\'');
+ sb.append(", nanos=").append(nanos);
+ sb.append(", attrs=").append(attrs);
+ sb.append(", sequenceNumber=").append(sequenceNumber);
+ sb.append('}');
+ return sb.toString();
+ }
+
+ /**
+ * Ask if an action has a specific attribute
+ * @param attr attribute
+ * @return true iff the action has the specific attribute
+ */
+ public boolean hasAttr(ActionAttributes attr) {
+ return attrs.contains(attr);
+ }
+
+ /**
+ * Actual application
+ * @param appMaster
+ * @throws IOException
+ */
+ public abstract void execute(SliderAppMaster appMaster) throws IOException;
+
+ public enum ActionAttributes {
+ SHRINKS_CLUSTER,
+ EXPANDS_CLUSTER,
+ HALTS_CLUSTER,
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
new file mode 100644
index 0000000..7eeaddf
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ * Executor for async actions - hands them off to the AM as
+ * appropriate
+ */
+public class QueueExecutor implements Runnable {
+ private static final Logger log =
+ LoggerFactory.getLogger(QueueExecutor.class);
+
+ private final SliderAppMaster appMaster;
+ private final QueueService actionQueues;
+
+ public QueueExecutor(SliderAppMaster appMaster,
+ QueueService actionQueues) {
+ Preconditions.checkNotNull(appMaster);
+ Preconditions.checkNotNull(actionQueues);
+
+ this.appMaster = appMaster;
+ this.actionQueues = actionQueues;
+ }
+
+ @VisibleForTesting
+ QueueExecutor(QueueService actionQueues) {
+ Preconditions.checkNotNull(actionQueues);
+ this.appMaster = null;
+ this.actionQueues = actionQueues;
+ }
+
+ /**
+ * Run until the queue has been told to stop
+ */
+ @Override
+ public void run() {
+ AsyncAction take = null;
+ try {
+ log.info("Queue Executor run() started");
+ do {
+ take = actionQueues.actionQueue.take();
+ log.debug("Executing {}", take);
+ take.execute(appMaster);
+ } while (!(take instanceof ActionStopQueue));
+ log.info("Queue Executor run() stopped");
+ } catch (InterruptedException | IOException e) {
+ log.error("Exception processing {}: {}", take, e, e);
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
new file mode 100644
index 0000000..dd39de0
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+
+import org.apache.hadoop.service.AbstractService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.DelayQueue;
+import java.util.concurrent.LinkedBlockingDeque;
+
+public class QueueService extends AbstractService
+ implements Runnable {
+ private static final Logger log =
+ LoggerFactory.getLogger(QueueService.class);
+
+ public final DelayQueue<AsyncAction> delayedActions = new DelayQueue<>();
+
+ public final BlockingDeque<AsyncAction> actionQueue =
+ new LinkedBlockingDeque<>();
+
+ public QueueService() {
+ super("action queue");
+ }
+
+ public void putImmediately(AsyncAction action) {
+ actionQueue.add(action);
+ }
+
+ public void putDelayed(AsyncAction action) {
+ delayedActions.add(action);
+ }
+
+ /**
+ * Run until the queue has been told to stop
+ */
+ @Override
+ public void run() {
+ try {
+
+ log.info("ActionExecutor Queue terminated");
+
+ AsyncAction take;
+ do {
+ take = delayedActions.take();
+ actionQueue.push(take);
+ } while (!(take instanceof ActionStopQueue));
+ } catch (InterruptedException e) {
+ //game over
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/StartContainerAction.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/StartContainerAction.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/StartContainerAction.java
new file mode 100644
index 0000000..95ecbe8
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/StartContainerAction.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+
+/**
+ * Start a container
+ */
+public class StartContainerAction {
+
+ private final Container container;
+ private final ContainerLaunchContext ctx;
+ private final RoleInstance instance;
+
+ public StartContainerAction(Container container,
+ ContainerLaunchContext ctx,
+ RoleInstance instance) {
+ this.container = container;
+ this.ctx = ctx;
+ this.instance = instance;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/services/workflow/AbstractWorkflowExecutorService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/AbstractWorkflowExecutorService.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/AbstractWorkflowExecutorService.java
deleted file mode 100644
index c26e3c4..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/AbstractWorkflowExecutorService.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import org.apache.hadoop.service.AbstractService;
-
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-
-/**
- * A service that hosts an executor -when the service is stopped,
- * {@link ExecutorService#shutdownNow()} is invoked.
- */
-public abstract class AbstractWorkflowExecutorService extends AbstractService {
-
- private ExecutorService executor;
-
- /**
- * Construct an instance with the given name -but
- * no executor
- * @param name service name
- */
- public AbstractWorkflowExecutorService(String name) {
- this(name, null);
- }
-
- /**
- * Construct an instance with the given name and executor
- * @param name service name
- * @param executor exectuor
- */
- protected AbstractWorkflowExecutorService(String name,
- ExecutorService executor) {
- super(name);
- this.executor = executor;
- }
-
- /**
- * Get the executor
- * @return the executor
- */
- public synchronized ExecutorService getExecutor() {
- return executor;
- }
-
- /**
- * Set the executor. This is protected as it
- * is intended to be restricted to subclasses
- * @param executor executor
- */
- protected synchronized void setExecutor(ExecutorService executor) {
- this.executor = executor;
- }
-
- /**
- * Execute the runnable with the executor (which
- * must have been created already)
- * @param runnable runnable to execute
- */
- public void execute(Runnable runnable) {
- getExecutor().execute(runnable);
- }
-
- /**
- * Submit a callable
- * @param callable callable
- * @param <V> type of the final get
- * @return a future to wait on
- */
- public <V> Future<V> submit(Callable<V> callable) {
- return getExecutor().submit(callable);
- }
- /**
- * Stop the service: halt the executor.
- * @throws Exception exception.
- */
- @Override
- protected void serviceStop() throws Exception {
- super.serviceStop();
- stopExecutor();
- }
-
- /**
- * Stop the executor if it is not null.
- * This uses {@link ExecutorService#shutdownNow()}
- * and so does not block until they have completed.
- */
- protected synchronized void stopExecutor() {
- if (executor != null) {
- executor.shutdownNow();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
index ccce6cb..46c724c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
@@ -27,6 +27,7 @@ import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -65,8 +66,9 @@ import java.util.concurrent.atomic.AtomicInteger;
* it calls its {@link #stop()} method. If the error code was non-zero,
* the service is logged as having failed.
*/
-public class ForkedProcessService extends AbstractWorkflowExecutorService implements
- LongLivedProcessLifecycleEvent, Runnable {
+public class ForkedProcessService
+ extends WorkflowExecutorService<ExecutorService>
+ implements LongLivedProcessLifecycleEvent, Runnable {
/**
* Log for the forked master process
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java
index 6c50798..fae3a38 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java
@@ -36,7 +36,7 @@ import java.util.concurrent.TimeUnit;
* started in this service's <code>start()</code> operation.
*/
public class WorkflowCallbackService<V> extends
- AbstractWorkflowExecutorService {
+ WorkflowScheduledExecutorService<ScheduledExecutorService> {
protected static final Logger LOG =
LoggerFactory.getLogger(WorkflowCallbackService.class);
private final Callable<V> callback;
@@ -48,7 +48,6 @@ public class WorkflowCallbackService<V> extends
private Callable<V> callable;
private ScheduledFuture<V> scheduledFuture;
-
/**
* Create an instance of the service
* @param name service name
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowExecutorService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowExecutorService.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowExecutorService.java
new file mode 100644
index 0000000..85d5330
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowExecutorService.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.workflow;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.service.AbstractService;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+
+/**
+ * A service that hosts an executor -when the service is stopped,
+ * {@link ExecutorService#shutdownNow()} is invoked.
+ */
+public class WorkflowExecutorService<E extends ExecutorService> extends AbstractService {
+
+ private E executor;
+
+ /**
+ * Construct an instance with the given name -but
+ * no executor
+ * @param name service name
+ */
+ public WorkflowExecutorService(String name) {
+ this(name, null);
+ }
+
+ /**
+ * Construct an instance with the given name and executor
+ * @param name service name
+ * @param executor exectuor
+ */
+ public WorkflowExecutorService(String name,
+ E executor) {
+ super(name);
+ this.executor = executor;
+ }
+
+ /**
+ * Get the executor
+ * @return the executor
+ */
+ public synchronized E getExecutor() {
+ return executor;
+ }
+
+ /**
+ * Set the executor. Only valid if the current one is null
+ * @param executor executor
+ */
+ public synchronized void setExecutor(E executor) {
+ Preconditions.checkState(this.executor == null,
+ "Executor already set");
+ this.executor = executor;
+ }
+
+ /**
+ * Execute the runnable with the executor (which
+ * must have been created already)
+ * @param runnable runnable to execute
+ */
+ public void execute(Runnable runnable) {
+ getExecutor().execute(runnable);
+ }
+
+ /**
+ * Submit a callable
+ * @param callable callable
+ * @param <V> type of the final get
+ * @return a future to wait on
+ */
+ public <V> Future<V> submit(Callable<V> callable) {
+ return getExecutor().submit(callable);
+ }
+
+ /**
+ * Stop the service: halt the executor.
+ * @throws Exception exception.
+ */
+ @Override
+ protected void serviceStop() throws Exception {
+ super.serviceStop();
+ stopExecutor();
+ }
+
+ /**
+ * Stop the executor if it is not null.
+ * This uses {@link ExecutorService#shutdownNow()}
+ * and so does not block until they have completed.
+ */
+ protected synchronized void stopExecutor() {
+ if (executor != null) {
+ executor.shutdownNow();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowScheduledExecutorService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowScheduledExecutorService.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowScheduledExecutorService.java
new file mode 100644
index 0000000..e9f53ed
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowScheduledExecutorService.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.workflow;
+
+import java.util.concurrent.ScheduledExecutorService;
+
+/**
+ * Scheduled executor or subclass thereof
+ * @param <E> scheduled executor service type
+ */
+public class WorkflowScheduledExecutorService<E extends ScheduledExecutorService>
+ extends WorkflowExecutorService<E> {
+
+ public WorkflowScheduledExecutorService(String name) {
+ super(name);
+ }
+
+ public WorkflowScheduledExecutorService(String name,
+ E executor) {
+ super(name, executor);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/services/workflow/package-info.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/package-info.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/package-info.java
index fab1b9f..36d059a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/package-info.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/package-info.java
@@ -154,7 +154,7 @@ and simply in applications which are built from composite YARN services.
then signals its owning service to stop once that runnable is completed.
Any exception raised in the run is stored.
</li>
- <li>{@link org.apache.slider.server.services.workflow.AbstractWorkflowExecutorService}:
+ <li>{@link org.apache.slider.server.services.workflow.WorkflowExecutorService}:
A base class for services that wish to have a {@link java.util.concurrent.ExecutorService}
with a lifespan mapped to that of a service. When the service is stopped, the
{@link java.util.concurrent.ExecutorService#shutdownNow()} method is called to
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestDestroyStandaloneAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestDestroyStandaloneAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestDestroyStandaloneAM.groovy
deleted file mode 100644
index ed0604b..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestDestroyStandaloneAM.groovy
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.agent.standalone
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.slider.agent.AgentMiniClusterTestBase
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.params.ActionEchoArgs
-import org.apache.slider.common.params.Arguments
-import org.apache.slider.common.params.SliderActions
-import org.apache.slider.common.tools.SliderFileSystem
-import org.apache.slider.core.exceptions.ErrorStrings
-import org.apache.slider.core.exceptions.SliderException
-import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
-import org.apache.slider.core.main.ServiceLauncher
-import org.junit.Test
-
-/**
- * destroy a masterless AM
- */
-@CompileStatic
-@Slf4j
-
-class TestDestroyStandaloneAM extends AgentMiniClusterTestBase {
-
- @Test
- public void testDestroyStandaloneAM() throws Throwable {
- String clustername = createMiniCluster("", configuration, 1, false)
-
- describe "create a Standalone AM, stop it, try to create" +
- "a second cluster with the same name, destroy it, try a third time"
-
- ServiceLauncher<SliderClient> launcher1 = launchClientAgainstMiniMR(
- configuration,
- [
- SliderActions.ACTION_DESTROY,
- "no-cluster-of-this-name",
- Arguments.ARG_FILESYSTEM, fsDefaultName,
- ])
- assert launcher1.serviceExitCode == 0
-
-
-
- ServiceLauncher<SliderClient> launcher = createStandaloneAM(
- clustername,
- true,
- true)
- SliderClient sliderClient = launcher.service
- addToTeardown(sliderClient);
-
- SliderFileSystem sliderFileSystem = createSliderFileSystem()
- def fs = sliderFileSystem.fileSystem
- def instanceDir = sliderFileSystem.buildClusterDirPath(clustername)
-
- assertPathExists(
- fs,
- "cluster path not found",
- instanceDir)
-
- sliderFileSystem.locateInstanceDefinition(clustername)
- clusterActionFreeze(sliderClient, clustername,"stopping first cluster")
- waitForAppToFinish(sliderClient)
-
-
- describe "Warnings below are expected"
-
- //now try to create instance #2, and expect an in-use failure
- try {
- createStandaloneAM(clustername, false, false)
- fail("expected a failure, got an AM")
- } catch (SliderException e) {
- assertExceptionDetails(e,
- SliderExitCodes.EXIT_INSTANCE_EXISTS,
- ErrorStrings.E_ALREADY_EXISTS)
- }
-
- describe "END EXPECTED WARNINGS"
-
-
- describe "destroying $clustername"
- //now: destroy it
-
- int exitCode = sliderClient.actionDestroy(clustername);
- assert 0 == exitCode
-
- describe "post destroy checks"
- sliderFileSystem.verifyDirectoryNonexistent(instanceDir)
-
- describe "thaw expected to fail"
- //expect thaw to now fail
- def ex = launchExpectingException(SliderClient,
- configuration,
- "",
- [
- SliderActions.ACTION_THAW,
- clustername,
- Arguments.ARG_FILESYSTEM, fsDefaultName,
- Arguments.ARG_MANAGER, RMAddr,
- ])
- assert ex instanceof UnknownApplicationInstanceException
-
- describe "thaw completed, checking dir is still absent"
- sliderFileSystem.verifyDirectoryNonexistent(instanceDir)
-
-
- describe "recreating $clustername"
-
- //and create a new cluster
- launcher = createStandaloneAM(clustername, false, true)
- SliderClient cluster2 = launcher.service
-
- // do an echo here of a large string
- // Hadoop RPC couldn't handle strings > 32K chars, this
- // check here allows us to be confident that large JSON Reports are handled
- StringBuilder sb = new StringBuilder()
- for (int i = 0; i < 65536; i++) {
- sb.append(Integer.toString(i, 16))
- }
- ActionEchoArgs args = new ActionEchoArgs();
- args.message = sb.toString();
- def echoed = cluster2.actionEcho(clustername, args)
- assert echoed == args.message
- log.info(
- "Successful echo of a text document ${echoed.size()} characters long")
- //try to destroy it while live
- try {
- int ec = cluster2.actionDestroy(clustername)
- fail("expected a failure from the destroy, got error code $ec")
- } catch (SliderException e) {
- assertFailureClusterInUse(e);
- }
-
- //and try to destroy a completely different cluster just for the fun of it
- assert 0 == sliderClient.actionDestroy("no-cluster-of-this-name")
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestKillStandaloneAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestKillStandaloneAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestKillStandaloneAM.groovy
deleted file mode 100644
index fef98a7..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestKillStandaloneAM.groovy
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.agent.standalone
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.ApplicationReport
-import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.agent.AgentMiniClusterTestBase
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderXmlConfKeys
-import org.apache.slider.common.params.Arguments
-import org.apache.slider.core.main.ServiceLauncher
-import org.junit.Test
-
-/**
- * kill a masterless AM and verify it shuts down. This test
- * also sets the retry count to 1 to stop recreation attempts
- */
-@CompileStatic
-@Slf4j
-
-class TestKillStandaloneAM extends AgentMiniClusterTestBase {
-
-
- @Test
- public void testKillStandaloneAM() throws Throwable {
- String clustername = createMiniCluster("", configuration, 1, true)
-
- describe "kill a Standalone AM and verify that it shuts down"
- ServiceLauncher<SliderClient> launcher =
- createStandaloneAMWithArgs(clustername,
- [
- Arguments.ARG_OPTION, SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, "1"
- ],
- true,
- false)
- SliderClient sliderClient = launcher.service
- addToTeardown(sliderClient);
- ApplicationReport report = waitForClusterLive(sliderClient)
-
- describe("listing Java processes")
- lsJavaProcesses();
- describe("killing AM")
- killAM(SIGTERM);
- waitWhileClusterLive(sliderClient);
- //give yarn some time to notice
- sleep(10000)
- describe("final listing")
- lsJavaProcesses();
- report = sliderClient.applicationReport
- assert YarnApplicationState.FAILED == report.yarnApplicationState;
- clusterActionFreeze(sliderClient, clustername)
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestRestartStandaloneAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestRestartStandaloneAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestRestartStandaloneAM.groovy
deleted file mode 100644
index 62dd1f9..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestRestartStandaloneAM.groovy
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.agent.standalone
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.ApplicationReport
-import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.slider.agent.AgentMiniClusterTestBase
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderXmlConfKeys
-import org.apache.slider.common.params.ActionAMSuicideArgs
-import org.apache.slider.common.params.Arguments
-import org.apache.slider.core.main.ServiceLauncher
-import org.junit.Test
-
-/**
- * kill a masterless AM and verify it shuts down. This test
- * also sets the retry count to 1 to stop recreation attempts
- */
-@CompileStatic
-@Slf4j
-
-class TestRestartStandaloneAM extends AgentMiniClusterTestBase {
-
-
- @Test
- public void testRestartStandaloneAM() throws Throwable {
- describe "kill a Standalone AM and verify that it restarts"
- // patch the configuration for AM restart
- YarnConfiguration conf = getRestartableConfiguration(5)
-
- String clustername = createMiniCluster("", conf, 1, true)
- ServiceLauncher<SliderClient> launcher =
- createStandaloneAMWithArgs(clustername,
- [
- Arguments.ARG_OPTION, SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, "4"
- ],
- true,
- false)
- SliderClient sliderClient = launcher.service
- addToTeardown(sliderClient);
-
- ApplicationReport report = waitForClusterLive(sliderClient)
- logReport(report)
- waitUntilClusterLive(sliderClient, 30000)
-
-
- ActionAMSuicideArgs args = new ActionAMSuicideArgs()
- args.message = "test AM iteration"
- args.waittime = 100
- args.exitcode = 1
- sliderClient.actionAmSuicide(clustername, args)
- waitWhileClusterLive(sliderClient);
- //give yarn some time to notice
- sleep(20000)
- waitUntilClusterLive(sliderClient, 40000)
-
-
- // app should be running here
- assert 0 == sliderClient.actionExists(clustername, true)
-
-
- clusterActionFreeze(sliderClient, clustername)
- }
-
- /**
- * Get a restartable configuration
- * @param restarts
- * @return
- */
- public YarnConfiguration getRestartableConfiguration(int restarts) {
- def conf = new YarnConfiguration(configuration)
- conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, restarts)
- conf.setInt(SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, restarts)
- conf
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
new file mode 100644
index 0000000..17e4ff2
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.params.ActionEchoArgs
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.common.tools.SliderFileSystem
+import org.apache.slider.core.exceptions.ErrorStrings
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * destroy a masterless AM
+ */
+@CompileStatic
+@Slf4j
+
+class TestStandaloneAMDestroy extends AgentMiniClusterTestBase {
+
+ @Test
+ public void testDestroyStandaloneAM() throws Throwable {
+ String clustername = createMiniCluster("", configuration, 1, false)
+
+ describe "create a Standalone AM, stop it, try to create" +
+ "a second cluster with the same name, destroy it, try a third time"
+
+ ServiceLauncher<SliderClient> launcher1 = launchClientAgainstMiniMR(
+ configuration,
+ [
+ SliderActions.ACTION_DESTROY,
+ "no-cluster-of-this-name",
+ Arguments.ARG_FILESYSTEM, fsDefaultName,
+ ])
+ assert launcher1.serviceExitCode == 0
+
+ ServiceLauncher<SliderClient> launcher = createStandaloneAM(
+ clustername,
+ true,
+ true)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+ SliderFileSystem sliderFileSystem = createSliderFileSystem()
+ def fs = sliderFileSystem.fileSystem
+ def instanceDir = sliderFileSystem.buildClusterDirPath(clustername)
+
+ assertPathExists(
+ fs,
+ "cluster path not found",
+ instanceDir)
+
+ sliderFileSystem.locateInstanceDefinition(clustername)
+ clusterActionFreeze(sliderClient, clustername,"stopping first cluster")
+ waitForAppToFinish(sliderClient)
+
+
+ describe "Warnings below are expected"
+
+ //now try to create instance #2, and expect an in-use failure
+ try {
+ createStandaloneAM(clustername, false, false)
+ fail("expected a failure, got an AM")
+ } catch (SliderException e) {
+ assertExceptionDetails(e,
+ SliderExitCodes.EXIT_INSTANCE_EXISTS,
+ ErrorStrings.E_ALREADY_EXISTS)
+ }
+
+ describe "END EXPECTED WARNINGS"
+
+
+ describe "destroying $clustername"
+ //now: destroy it
+
+ int exitCode = sliderClient.actionDestroy(clustername);
+ assert 0 == exitCode
+
+ describe "post destroy checks"
+ sliderFileSystem.verifyDirectoryNonexistent(instanceDir)
+
+ describe "thaw expected to fail"
+ //expect thaw to now fail
+ def ex = launchExpectingException(SliderClient,
+ configuration,
+ "",
+ [
+ SliderActions.ACTION_THAW,
+ clustername,
+ Arguments.ARG_FILESYSTEM, fsDefaultName,
+ Arguments.ARG_MANAGER, RMAddr,
+ ])
+ assert ex instanceof UnknownApplicationInstanceException
+
+ describe "thaw completed, checking dir is still absent"
+ sliderFileSystem.verifyDirectoryNonexistent(instanceDir)
+
+
+ describe "recreating $clustername"
+
+ //and create a new cluster
+ launcher = createStandaloneAM(clustername, false, true)
+ SliderClient cluster2 = launcher.service
+
+ // do an echo here of a large string
+ // Hadoop RPC couldn't handle strings > 32K chars, this
+ // check here allows us to be confident that large JSON Reports are handled
+ StringBuilder sb = new StringBuilder()
+ for (int i = 0; i < 65536; i++) {
+ sb.append(Integer.toString(i, 16))
+ }
+ ActionEchoArgs args = new ActionEchoArgs();
+ args.message = sb.toString();
+ def echoed = cluster2.actionEcho(clustername, args)
+ assert echoed == args.message
+ log.info(
+ "Successful echo of a text document ${echoed.size()} characters long")
+ //try to destroy it while live
+ try {
+ int ec = cluster2.actionDestroy(clustername)
+ fail("expected a failure from the destroy, got error code $ec")
+ } catch (SliderException e) {
+ assertFailureClusterInUse(e);
+ }
+
+ //and try to destroy a completely different cluster just for the fun of it
+ assert 0 == sliderClient.actionDestroy("no-cluster-of-this-name")
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
new file mode 100644
index 0000000..75f9a2c
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * kill a masterless AM and verify it shuts down. This test
+ * also sets the retry count to 1 to stop recreation attempts
+ */
+@CompileStatic
+@Slf4j
+
+class TestStandaloneAMKill extends AgentMiniClusterTestBase {
+
+
+ @Test
+ public void testKillStandaloneAM() throws Throwable {
+ String clustername = createMiniCluster("", configuration, 1, true)
+
+ describe "kill a Standalone AM and verify that it shuts down"
+ ServiceLauncher<SliderClient> launcher =
+ createStandaloneAMWithArgs(clustername,
+ [
+ Arguments.ARG_OPTION, SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, "1"
+ ],
+ true,
+ false)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+ ApplicationReport report = waitForClusterLive(sliderClient)
+
+ describe("listing Java processes")
+ lsJavaProcesses();
+ describe("killing AM")
+ killAM(SIGTERM);
+ waitWhileClusterLive(sliderClient);
+ //give yarn some time to notice
+ sleep(10000)
+ describe("final listing")
+ lsJavaProcesses();
+ report = sliderClient.applicationReport
+ assert YarnApplicationState.FAILED == report.yarnApplicationState;
+ clusterActionFreeze(sliderClient, clustername)
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMRestart.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMRestart.groovy
new file mode 100644
index 0000000..8d9318a
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMRestart.groovy
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.ActionAMSuicideArgs
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * kill a masterless AM and verify it shuts down. This test
+ * also sets the retry count to 1 to stop recreation attempts
+ */
+@CompileStatic
+@Slf4j
+
+class TestStandaloneAMRestart extends AgentMiniClusterTestBase {
+
+
+ @Test
+ public void testRestartStandaloneAM() throws Throwable {
+ describe "kill a Standalone AM and verify that it restarts"
+ // patch the configuration for AM restart
+ YarnConfiguration conf = getRestartableConfiguration(5)
+
+ String clustername = createMiniCluster("", conf, 1, true)
+ ServiceLauncher<SliderClient> launcher =
+ createStandaloneAMWithArgs(clustername,
+ [
+ Arguments.ARG_OPTION, SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, "4"
+ ],
+ true,
+ false)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+ ApplicationReport report = waitForClusterLive(sliderClient)
+ logReport(report)
+ waitUntilClusterLive(sliderClient, 30000)
+
+
+ ActionAMSuicideArgs args = new ActionAMSuicideArgs()
+ args.message = "test AM iteration"
+ args.waittime = 100
+ args.exitcode = 1
+ sliderClient.actionAmSuicide(clustername, args)
+ waitWhileClusterLive(sliderClient);
+ //give yarn some time to notice
+ sleep(20000)
+ waitUntilClusterLive(sliderClient, 40000)
+
+
+ // app should be running here
+ assert 0 == sliderClient.actionExists(clustername, true)
+
+
+ clusterActionFreeze(sliderClient, clustername)
+ }
+
+ /**
+ * Get a restartable configuration
+ * @param restarts
+ * @return
+ */
+ public YarnConfiguration getRestartableConfiguration(int restarts) {
+ def conf = new YarnConfiguration(configuration)
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, restarts)
+ conf.setInt(SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, restarts)
+ conf
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
new file mode 100644
index 0000000..24d4c40
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.service.ServiceOperations
+import org.apache.slider.server.appmaster.SliderAppMaster
+import org.apache.slider.server.services.workflow.ServiceThreadFactory
+import org.apache.slider.server.services.workflow.WorkflowExecutorService
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicBoolean
+import java.util.concurrent.atomic.AtomicLong
+
+@Slf4j
+//@CompileStatic
+class TestActions {
+
+ QueueService queues;
+ WorkflowExecutorService<ExecutorService> executorService;
+
+
+ @Before
+ void createService() {
+ queues = new QueueService();
+
+ def conf = new Configuration()
+ queues.init(conf)
+
+ queues.start();
+
+ executorService = new WorkflowExecutorService<>("AmExecutor",
+ Executors.newCachedThreadPool(
+ new ServiceThreadFactory("AmExecutor", true)));
+
+ executorService.init(conf)
+ executorService.start();
+ }
+
+ @After
+ void destroyService() {
+ ServiceOperations.stop(executorService);
+ ServiceOperations.stop(queues);
+ }
+
+ @Test
+ public void testBasicService() throws Throwable {
+ queues.start();
+ }
+
+ @Test
+ public void testDelayLogic() throws Throwable {
+ ActionNoteExecuted action = new ActionNoteExecuted("", 1000)
+ long now = System.currentTimeMillis();
+
+ def delay = action.getDelay(TimeUnit.MILLISECONDS)
+ assert delay >= 800
+ assert delay <= 1800
+
+ ActionNoteExecuted a2 = new ActionNoteExecuted("a2", 10000)
+ assert action.compareTo(a2) < 0
+ assert a2.compareTo(action) > 0
+ assert action.compareTo(action)== 0
+
+ }
+
+ @Test
+ public void testActionDelayedExecutorTermination() throws Throwable {
+ long start = System.currentTimeMillis()
+
+ ActionStopQueue stopAction = new ActionStopQueue(1000);
+ queues.delayedActions.add(stopAction);
+ queues.run();
+ AsyncAction take = queues.actionQueue.take();
+ assert take == stopAction
+ long stop = System.currentTimeMillis();
+ assert stop - start > 500
+ assert stop - start < 1500
+ }
+
+ @Test
+ public void testImmediateQueue() throws Throwable {
+ ActionNoteExecuted noteExecuted = new ActionNoteExecuted("executed", 0)
+ queues.putImmediately(noteExecuted)
+ queues.putImmediately(new ActionStopQueue(0))
+ QueueExecutor ex = new QueueExecutor(queues)
+ ex.run();
+ assert queues.actionQueue.empty
+ assert noteExecuted.executed.get()
+ }
+
+ @Test
+ public void testActionOrdering() throws Throwable {
+
+ ActionNoteExecuted note1 = new ActionNoteExecuted("note1", 500)
+ def stop = new ActionStopQueue(1500)
+ ActionNoteExecuted note2 = new ActionNoteExecuted("note2", 800)
+
+ List<AsyncAction> actions = [note1, stop, note2]
+ Collections.sort(actions)
+ assert actions.get(0) == note1
+ assert actions.get(1) == note2
+ assert actions.get(3) == stop
+ }
+
+ @Test
+ public void testDelayedQueueWithReschedule() throws Throwable {
+
+ ActionNoteExecuted note1 = new ActionNoteExecuted("note1", 500)
+ def stop = new ActionStopQueue(1500)
+ ActionNoteExecuted note2 = new ActionNoteExecuted("note2", 800)
+
+ assert note2.compareTo(stop) < 0
+ assert note1.nanos < note2.nanos
+ assert note2.nanos < stop.nanos
+ queues.putDelayed(note1)
+ queues.putDelayed(stop)
+ queues.putDelayed(note2)
+ // async to sync expected to run in order
+ queues.run();
+ assert queues.delayedActions.empty
+ assert !queues.actionQueue.empty
+ QueueExecutor ex = new QueueExecutor(queues)
+ ex.run();
+ assert queues.actionQueue.empty
+ assert note1.executed.get()
+ assert note2.executed.get()
+ }
+
+ public class ActionNoteExecuted extends AsyncAction {
+ public final AtomicBoolean executed = new AtomicBoolean(false);
+ public final AtomicLong executionTimeNanos = new AtomicLong()
+ public final AtomicLong executionCount = new AtomicLong()
+
+ public ActionNoteExecuted(String text, int delay) {
+ super(text, delay);
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster) throws IOException {
+ log.info("Executing $name");
+ executed.set(true);
+ executionTimeNanos.set(System.nanoTime())
+ executionCount.incrementAndGet()
+ synchronized (this) {
+ this.notify();
+ }
+ }
+
+ @Override
+ String toString() {
+ return super.toString() +
+ " executed=${executed.get()}; count=${executionCount.get()};"
+ }
+
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
index 0f4a4ca..4242ba1 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
@@ -31,7 +31,7 @@ import org.apache.slider.server.avro.RoleHistoryWriter
import org.junit.Test
@Slf4j
-@CompileStatic
+//@CompileStatic
class TestRoleHistoryRW extends BaseMockAppStateTest {
static long time = System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowExecutorService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowExecutorService.java b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowExecutorService.java
index 9514f47..dc160d9 100644
--- a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowExecutorService.java
+++ b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowExecutorService.java
@@ -20,9 +20,13 @@ package org.apache.slider.server.services.workflow;
import org.junit.Test;
+import java.util.concurrent.ExecutorService;
-public class TestWorkflowExecutorService extends WorkflowServiceTestBase {
+/**
+ * Basic tests for executor service
+ */
+public class TestWorkflowExecutorService extends WorkflowServiceTestBase {
@Test
public void testAsyncRun() throws Throwable {
@@ -51,7 +55,8 @@ public class TestWorkflowExecutorService extends WorkflowServiceTestBase {
assertNotNull(runnable.getException());
}
- private static class ExecutorSvc extends AbstractWorkflowExecutorService {
+ private static class ExecutorSvc
+ extends WorkflowExecutorService<ExecutorService> {
private ExecutorSvc() {
super("ExecutorService",
ServiceThreadFactory.singleThreadExecutor("test", true));
[07/14] git commit: SLIDER-302 Add a provider startup event...need to
think through how best to use this
Posted by sm...@apache.org.
SLIDER-302 Add a provider startup event...need to think through how best to use this
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b7b9a158
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b7b9a158
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b7b9a158
Branch: refs/heads/feature/SLIDER-181_Storm_Package
Commit: b7b9a158b7f7da3c4deb5009eb48c3a8a301838b
Parents: 8f777f3
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 8 19:15:58 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 8 19:16:30 2014 +0100
----------------------------------------------------------------------
.../actions/ProviderStartupCompleted.java | 37 ++++++++++++++++++++
1 file changed, 37 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b7b9a158/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
new file mode 100644
index 0000000..293492f
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.slider.server.appmaster.SliderAppMaster;
+
+public class ProviderStartupCompleted extends AsyncAction {
+
+ public ProviderStartupCompleted() {
+ super("ProviderStartupCompleted");
+ }
+
+ public ProviderStartupCompleted(int delayMillis) {
+ super("ProviderStartupCompleted", delayMillis);
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster) throws Exception {
+ appMaster.eventCallbackEvent(null);
+ }
+}
[10/14] git commit: Merge branch 'develop' of
https://git-wip-us.apache.org/repos/asf/incubator-slider into develop
Posted by sm...@apache.org.
Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c2150453
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c2150453
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c2150453
Branch: refs/heads/feature/SLIDER-181_Storm_Package
Commit: c2150453e0faa5ff4199a95e8885512b0f6a7b36
Parents: bfb4cbb a1bbf4d
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 8 19:17:32 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 8 19:17:32 2014 +0100
----------------------------------------------------------------------
app-packages/hbase/get-hbase-site.sh | 26 +++++++++-----------------
app-packages/hbase/links.py | 15 +++++++++++++++
2 files changed, 24 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
[11/14] git commit: SLIDER-288 Add role for HBase thrift2 gateway
Posted by sm...@apache.org.
SLIDER-288 Add role for HBase thrift2 gateway
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6f912df2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6f912df2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6f912df2
Branch: refs/heads/feature/SLIDER-181_Storm_Package
Commit: 6f912df214136a8cd903b0c5d17ee9d680e6cffc
Parents: a1bbf4d
Author: tedyu <yu...@gmail.com>
Authored: Fri Aug 8 11:24:28 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Fri Aug 8 11:24:28 2014 -0700
----------------------------------------------------------------------
app-packages/hbase/appConfig.json | 1 +
app-packages/hbase/links.py | 20 +++++++
app-packages/hbase/metainfo.xml | 15 +++++
.../hbase/package/scripts/hbase_service.py | 2 +
.../hbase/package/scripts/hbase_thrift2.py | 62 ++++++++++++++++++++
app-packages/hbase/package/scripts/params.py | 1 +
app-packages/hbase/resources.json | 9 ++-
.../providers/hbase/HBaseClientProvider.java | 4 ++
.../slider/providers/hbase/HBaseKeys.java | 5 ++
.../providers/hbase/HBaseProviderService.java | 5 ++
.../slider/providers/hbase/HBaseRoles.java | 2 +
11 files changed, 124 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json
index f0ab144..24d9a84 100644
--- a/app-packages/hbase/appConfig.json
+++ b/app-packages/hbase/appConfig.json
@@ -24,6 +24,7 @@
"site.global.ganglia_server_port": "8667",
"site.global.ganglia_server_id": "Application1",
"site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
+ "site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}",
"site.hbase-site.hbase.hstore.flush.retries.number": "120",
"site.hbase-site.hbase.client.keyvalue.maxsize": "10485760",
"site.hbase-site.hbase.hstore.compactionThreshold": "3",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/links.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/links.py b/app-packages/hbase/links.py
index c77e620..19b0d91 100644
--- a/app-packages/hbase/links.py
+++ b/app-packages/hbase/links.py
@@ -1,3 +1,23 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
from __future__ import print_function
import logging
import json
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase/metainfo.xml b/app-packages/hbase/metainfo.xml
index ee6ddea..a5dab0c 100644
--- a/app-packages/hbase/metainfo.xml
+++ b/app-packages/hbase/metainfo.xml
@@ -45,6 +45,10 @@
<value>http://${HBASE_REST_HOST}:${site.global.hbase_rest_port}</value>
</export>
<export>
+ <name>org.apache.slider.hbase.thrift2</name>
+ <value>http://${HBASE_THRIFT2_HOST}:${site.global.hbase_thrift2_port}</value>
+ </export>
+ <export>
<name>org.apache.slider.hbase.thrift</name>
<value>http://${HBASE_THRIFT_HOST}:${site.global.hbase_thrift_port}</value>
</export>
@@ -68,6 +72,7 @@
<command>HBASE_MASTER-START</command>
<requires>HBASE_REST-INSTALLED</requires>
<requires>HBASE_THRIFT-INSTALLED</requires>
+ <requires>HBASE_THRIFT2-INSTALLED</requires>
</commandOrder>
</commandOrders>
<components>
@@ -114,6 +119,16 @@
</component>
<component>
+ <name>HBASE_THRIFT2</name>
+ <category>SLAVE</category>
+ <minInstanceCount>1</minInstanceCount>
+ <commandScript>
+ <script>scripts/hbase_thrift2.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
+
+ <component>
<name>HBASE_CLIENT</name>
<category>CLIENT</category>
<minInstanceCount>0</minInstanceCount>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/package/scripts/hbase_service.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/scripts/hbase_service.py b/app-packages/hbase/package/scripts/hbase_service.py
index 01d38b3..96add84 100644
--- a/app-packages/hbase/package/scripts/hbase_service.py
+++ b/app-packages/hbase/package/scripts/hbase_service.py
@@ -39,6 +39,8 @@ def hbase_service(
daemon_cmd = format("{daemon_cmd} -p {rest_port}")
elif name == 'thrift':
daemon_cmd = format("{daemon_cmd} -p {thrift_port}")
+ elif name == 'thrift2':
+ daemon_cmd = format("{daemon_cmd} -p {thrift2_port}")
no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
elif action == 'stop':
daemon_cmd = format("{cmd} stop {role} && rm -f {pid_file}")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/package/scripts/hbase_thrift2.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/scripts/hbase_thrift2.py b/app-packages/hbase/package/scripts/hbase_thrift2.py
new file mode 100644
index 0000000..b72196c
--- /dev/null
+++ b/app-packages/hbase/package/scripts/hbase_thrift2.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+from hbase import hbase
+from hbase_service import hbase_service
+
+
+class HbaseThrift2(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ hbase(name='thrift2')
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env) # for security
+
+ hbase_service( 'thrift2',
+ action = 'start'
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ hbase_service( 'thrift2',
+ action = 'stop'
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ pid_file = format("{pid_dir}/hbase-{hbase_user}-thrift2.pid")
+ check_process_status(pid_file)
+
+if __name__ == "__main__":
+ HbaseThrift2().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/scripts/params.py b/app-packages/hbase/package/scripts/params.py
index 2d05289..1f25f68 100644
--- a/app-packages/hbase/package/scripts/params.py
+++ b/app-packages/hbase/package/scripts/params.py
@@ -59,6 +59,7 @@ ganglia_server_port = default('/configurations/global/ganglia_server_port', '866
rest_port = config['configurations']['global']['hbase_rest_port']
thrift_port = config['configurations']['global']['hbase_thrift_port']
+thrift2_port = config['configurations']['global']['hbase_thrift2_port']
if security_enabled:
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/resources.json b/app-packages/hbase/resources.json
index 2d2b4f9..d2fdbd8 100644
--- a/app-packages/hbase/resources.json
+++ b/app-packages/hbase/resources.json
@@ -21,9 +21,14 @@
"yarn.role.priority": "3",
"yarn.component.instances": "1",
"yarn.memory": "256"
- }
+ },
"HBASE_THRIFT": {
- "yarn.role.priority": "3",
+ "yarn.role.priority": "4",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ },
+ "HBASE_THRIFT2": {
+ "yarn.role.priority": "5",
"yarn.component.instances": "1",
"yarn.memory": "256"
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
index fde2bf7..cd104a2 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
@@ -239,8 +239,12 @@ public class HBaseClientProvider extends AbstractClientProvider implements
providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_MASTER,
0, -1);
+ providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_REST_GATEWAY,
+ 0, -1);
providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_THRIFT_GATEWAY,
0, -1);
+ providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_THRIFT2_GATEWAY,
+ 0, -1);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
index 9c38fb2..2a20438 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
@@ -30,6 +30,8 @@ public interface HBaseKeys {
String ROLE_THRIFT_GATEWAY = "THRIFT";
+ String ROLE_THRIFT2_GATEWAY = "THRIFT2";
+
/** {@value */
String REGION_SERVER = "regionserver";
@@ -38,6 +40,9 @@ public interface HBaseKeys {
/** {@value */
String THRIFT_GATEWAY = "thrift";
+ /** {@value */
+ String THRIFT2_GATEWAY = "thrift2";
+ /**
/**
* What is the command for hbase to print a version: {@value}
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index 953ee79..f72967f 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -201,6 +201,11 @@ public class HBaseProviderService extends AbstractProviderService implements
logfile = "/thrift-gateway.txt";
break;
+ case ROLE_THRIFT2_GATEWAY:
+ roleCommand = THRIFT2_GATEWAY;
+
+ logfile = "/thrift2-gateway.txt";
+ break;
default:
throw new SliderInternalStateException("Cannot start role %s", role);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
index 2cc457b..b2825b8 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
@@ -41,6 +41,7 @@ public class HBaseRoles {
public static final int KEY_THRIFT_GATEWAY = SliderKeys.ROLE_AM_PRIORITY_INDEX + 4;
+ public static final int KEY_THRIFT2_GATEWAY = SliderKeys.ROLE_AM_PRIORITY_INDEX + 5;
/**
* Initialize role list
*/
@@ -49,6 +50,7 @@ public class HBaseRoles {
ROLES.add(new ProviderRole(HBaseKeys.ROLE_MASTER, KEY_MASTER));
ROLES.add(new ProviderRole(HBaseKeys.ROLE_REST_GATEWAY, KEY_REST_GATEWAY));
ROLES.add(new ProviderRole(HBaseKeys.ROLE_THRIFT_GATEWAY, KEY_THRIFT_GATEWAY));
+ ROLES.add(new ProviderRole(HBaseKeys.ROLE_THRIFT_GATEWAY, KEY_THRIFT2_GATEWAY));
}
[09/14] git commit: Merge branch
'feature/SLIDER-302_scheduled_executor' into develop
Posted by sm...@apache.org.
Merge branch 'feature/SLIDER-302_scheduled_executor' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bfb4cbb0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bfb4cbb0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bfb4cbb0
Branch: refs/heads/feature/SLIDER-181_Storm_Package
Commit: bfb4cbb030a979edde16334342b72071e4d51e80
Parents: 66cbd37 cf2aa70
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 8 19:17:18 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 8 19:17:18 2014 +0100
----------------------------------------------------------------------
.../apache/slider/common/tools/SliderUtils.java | 17 +-
.../providers/AbstractProviderService.java | 12 +-
.../slider/providers/ProviderService.java | 16 +-
.../providers/agent/AgentProviderService.java | 15 +-
.../providers/agent/HeartbeatMonitor.java | 9 +-
.../server/appmaster/AMViewForProviders.java | 46 -----
.../appmaster/ContainerStartOperation.java | 40 -----
.../server/appmaster/RoleLaunchService.java | 40 +++--
.../server/appmaster/SliderAppMaster.java | 134 ++++++++++----
.../server/appmaster/actions/ActionHalt.java | 41 +++++
.../appmaster/actions/ActionStartContainer.java | 47 +++++
.../appmaster/actions/ActionStopQueue.java | 36 ++++
.../appmaster/actions/ActionStopSlider.java | 39 ++++
.../server/appmaster/actions/AsyncAction.java | 124 +++++++++++++
.../actions/ProviderReportedContainerLoss.java | 47 +++++
.../actions/ProviderStartupCompleted.java | 37 ++++
.../server/appmaster/actions/QueueAccess.java | 35 ++++
.../server/appmaster/actions/QueueExecutor.java | 75 ++++++++
.../server/appmaster/actions/QueueService.java | 76 ++++++++
.../appmaster/actions/StartContainerAction.java | 41 +++++
.../AbstractWorkflowExecutorService.java | 110 ------------
.../services/workflow/ForkedProcessService.java | 6 +-
.../workflow/WorkflowCallbackService.java | 3 +-
.../workflow/WorkflowExecutorService.java | 113 ++++++++++++
.../WorkflowScheduledExecutorService.java | 38 ++++
.../server/services/workflow/package-info.java | 2 +-
.../standalone/TestDestroyStandaloneAM.groovy | 156 ----------------
.../standalone/TestKillStandaloneAM.groovy | 73 --------
.../standalone/TestRestartStandaloneAM.groovy | 96 ----------
.../standalone/TestStandaloneAMDestroy.groovy | 154 ++++++++++++++++
.../standalone/TestStandaloneAMKill.groovy | 73 ++++++++
.../standalone/TestStandaloneAMRestart.groovy | 96 ++++++++++
.../server/appmaster/actions/TestActions.groovy | 180 +++++++++++++++++++
.../model/history/TestRoleHistoryRW.groovy | 2 +-
.../model/mock/MockProviderService.groovy | 4 +-
.../providers/agent/TestHeartbeatMonitor.java | 6 -
.../publisher/TestAgentProviderService.java | 6 +-
.../workflow/TestWorkflowExecutorService.java | 9 +-
38 files changed, 1428 insertions(+), 626 deletions(-)
----------------------------------------------------------------------
[13/14] git commit: SLIDER-240 Add log after verification passes
Posted by sm...@apache.org.
SLIDER-240 Add log after verification passes
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e464bbad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e464bbad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e464bbad
Branch: refs/heads/feature/SLIDER-181_Storm_Package
Commit: e464bbadf0c9a395c3d0c1a610838f6e6269f94e
Parents: 05bf45d
Author: tedyu <yu...@gmail.com>
Authored: Fri Aug 8 11:41:51 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Fri Aug 8 11:41:51 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/slider/common/tools/CoreFileSystem.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e464bbad/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index 93de4d9..9718be9 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -335,6 +335,7 @@ public class CoreFileSystem {
}
dst.delete();
if (!found) throw new FileNotFoundException("file: " + file + " not found in " + path);
+ log.info("Verification of " + path + " passed");
}
/**
* Create the application-instance specific temporary directory