You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/08/09 15:04:42 UTC
[01/50] [abbrv] git commit: SLIDER-90: HA NN, RM support
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-151_REST_API 621a981cb -> bfe71dc05
SLIDER-90: HA NN, RM support
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0b52d4b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0b52d4b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0b52d4b0
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 0b52d4b0e2fc0a3a5e98c34bb6248a84e359c3db
Parents: 7ab9ce9
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 5 12:04:41 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 5 12:04:41 2014 +0100
----------------------------------------------------------------------
.../java/org/apache/slider/client/SliderClient.java | 10 ++++------
.../slider/common/params/AbstractActionArgs.java | 14 +++++---------
.../java/org/apache/slider/common/params/ArgOps.java | 7 ++++---
.../org/apache/slider/common/params/ClientArgs.java | 5 +++--
.../org/apache/slider/common/params/CommonArgs.java | 11 +++++------
.../slider/server/appmaster/SliderAppMaster.java | 4 ++--
6 files changed, 23 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b52d4b0/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 40e01bb..ff2e865 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -182,7 +182,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
Configuration clientConf = SliderUtils.loadClientConfigurationResource();
ConfigHelper.mergeConfigurations(conf, clientConf, CLIENT_RESOURCE);
serviceArgs.applyDefinitions(conf);
- serviceArgs.applyFileSystemURL(conf);
+ serviceArgs.applyFileSystemBinding(conf);
// init security with our conf
if (SliderUtils.isHadoopClusterSecure(conf)) {
SliderUtils.forceLogin();
@@ -330,9 +330,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
client.createPath(zkPath, "", ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
return zkPath;
- } catch (InterruptedException e) {
- log.warn("Unable to create zk node {}", zkPath, e);
- } catch (KeeperException e) {
+ } catch (InterruptedException | KeeperException e) {
log.warn("Unable to create zk node {}", zkPath, e);
}
}
@@ -1008,8 +1006,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
commandLine.add(Arguments.ARG_RM_ADDR, rmAddr);
}
- if (serviceArgs.getFilesystemURL() != null) {
- commandLine.add(Arguments.ARG_FILESYSTEM, serviceArgs.getFilesystemURL());
+ if (serviceArgs.getFilesystemBinding() != null) {
+ commandLine.add(Arguments.ARG_FILESYSTEM, serviceArgs.getFilesystemBinding());
}
addConfOptionToCLI(commandLine, config, REGISTRY_PATH,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b52d4b0/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
index 44bc239..3a8cea4 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
@@ -25,7 +25,6 @@ import org.apache.slider.core.exceptions.ErrorStrings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.net.URI;
import java.util.ArrayList;
import java.util.List;
@@ -41,26 +40,23 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
}
/**
- * URI of the filesystem
+ * URI/binding to the filesystem
*/
@Parameter(names = {ARG_FILESYSTEM, ARG_FILESYSTEM_LONG},
- description = "Filesystem URI",
- converter = URIArgumentConverter.class)
- public URI filesystemURL;
+ description = "Filesystem Binding")
+ public String filesystemBinding;
@Parameter(names = {ARG_BASE_PATH},
description = "Slider base path on the filesystem",
converter = PathArgumentConverter.class)
public Path basePath;
-
/**
* This is the default parameter
*/
@Parameter
public final List<String> parameters = new ArrayList<>();
-
/**
* get the name: relies on arg 1 being the cluster name in all operations
* @return the name argument, null if there is none
@@ -92,7 +88,7 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
@Parameter(names = {ARG_MANAGER_SHORT, ARG_MANAGER},
- description = "hostname:port of the YARN resource manager")
+ description = "Binding (usually hostname:port) of the YARN resource manager")
public String manager;
@@ -110,7 +106,7 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
/**
* Get the name of the action
- * @return
+ * @return the action name
*/
public abstract String getActionName() ;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b52d4b0/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java b/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
index 0837dd2..b53ebd3 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
@@ -58,11 +58,12 @@ public class ArgOps {
return triple(msg, min, min);
}
- public static void applyFileSystemURL(URI filesystemURL, Configuration conf) {
- if (filesystemURL != null) {
+ public static void applyFileSystemBinding(String filesystemBinding,
+ Configuration conf) {
+ if (filesystemBinding != null) {
//filesystem argument was set -this overwrites any defaults in the
//configuration
- FileSystem.setDefaultUri(conf, filesystemURL);
+ FileSystem.setDefaultUri(conf, filesystemBinding);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b52d4b0/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
index 6e44402..ca854f1 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
@@ -103,9 +103,10 @@ public class ClientArgs extends CommonArgs {
log.debug("Setting RM to {}", getManager());
conf.set(YarnConfiguration.RM_ADDRESS, getManager());
}
- if ( getBasePath() != null ) {
+ if (getBasePath() != null) {
log.debug("Setting basePath to {}", getBasePath());
- conf.set(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH, getBasePath().toString());
+ conf.set(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH,
+ getBasePath().toString());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b52d4b0/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
index 4cee1d1..83e658d 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
@@ -30,7 +30,6 @@ import org.apache.slider.core.exceptions.SliderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.net.URI;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
@@ -215,12 +214,12 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
/**
- * If the Filesystem URL was provided, it overrides anything in
+ * If the Filesystem binding was provided, it overrides anything in
* the configuration
* @param conf configuration
*/
- public void applyFileSystemURL(Configuration conf) {
- ArgOps.applyFileSystemURL(getFilesystemURL(), conf);
+ public void applyFileSystemBinding(Configuration conf) {
+ ArgOps.applyFileSystemBinding(getFilesystemBinding(), conf);
}
public boolean isDebug() {
@@ -228,8 +227,8 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
}
- public URI getFilesystemURL() {
- return coreAction.filesystemURL;
+ public String getFilesystemBinding() {
+ return coreAction.filesystemBinding;
}
public Path getBasePath() { return coreAction.basePath; }
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b52d4b0/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 3b061d5..7e0ae5e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -340,7 +340,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
SliderAMCreateAction createAction = (SliderAMCreateAction) action;
//sort out the location of the AM
serviceArgs.applyDefinitions(conf);
- serviceArgs.applyFileSystemURL(conf);
+ serviceArgs.applyFileSystemBinding(conf);
String rmAddress = createAction.getRmAddress();
if (rmAddress != null) {
@@ -348,7 +348,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
SliderUtils.setRmSchedulerAddress(conf, rmAddress);
}
serviceArgs.applyDefinitions(conf);
- serviceArgs.applyFileSystemURL(conf);
+ serviceArgs.applyFileSystemBinding(conf);
//init security with our conf
if (SliderUtils.isHadoopClusterSecure(conf)) {
log.info("Secure mode with kerberos realm {}",
[45/50] [abbrv] git commit: SLIDER-288 Add role for HBase thrift2
gateway
Posted by st...@apache.org.
SLIDER-288 Add role for HBase thrift2 gateway
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6f912df2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6f912df2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6f912df2
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 6f912df214136a8cd903b0c5d17ee9d680e6cffc
Parents: a1bbf4d
Author: tedyu <yu...@gmail.com>
Authored: Fri Aug 8 11:24:28 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Fri Aug 8 11:24:28 2014 -0700
----------------------------------------------------------------------
app-packages/hbase/appConfig.json | 1 +
app-packages/hbase/links.py | 20 +++++++
app-packages/hbase/metainfo.xml | 15 +++++
.../hbase/package/scripts/hbase_service.py | 2 +
.../hbase/package/scripts/hbase_thrift2.py | 62 ++++++++++++++++++++
app-packages/hbase/package/scripts/params.py | 1 +
app-packages/hbase/resources.json | 9 ++-
.../providers/hbase/HBaseClientProvider.java | 4 ++
.../slider/providers/hbase/HBaseKeys.java | 5 ++
.../providers/hbase/HBaseProviderService.java | 5 ++
.../slider/providers/hbase/HBaseRoles.java | 2 +
11 files changed, 124 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json
index f0ab144..24d9a84 100644
--- a/app-packages/hbase/appConfig.json
+++ b/app-packages/hbase/appConfig.json
@@ -24,6 +24,7 @@
"site.global.ganglia_server_port": "8667",
"site.global.ganglia_server_id": "Application1",
"site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
+ "site.global.hbase_thrift2_port": "${HBASE_THRIFT2.ALLOCATED_PORT}",
"site.hbase-site.hbase.hstore.flush.retries.number": "120",
"site.hbase-site.hbase.client.keyvalue.maxsize": "10485760",
"site.hbase-site.hbase.hstore.compactionThreshold": "3",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/links.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/links.py b/app-packages/hbase/links.py
index c77e620..19b0d91 100644
--- a/app-packages/hbase/links.py
+++ b/app-packages/hbase/links.py
@@ -1,3 +1,23 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
from __future__ import print_function
import logging
import json
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase/metainfo.xml b/app-packages/hbase/metainfo.xml
index ee6ddea..a5dab0c 100644
--- a/app-packages/hbase/metainfo.xml
+++ b/app-packages/hbase/metainfo.xml
@@ -45,6 +45,10 @@
<value>http://${HBASE_REST_HOST}:${site.global.hbase_rest_port}</value>
</export>
<export>
+ <name>org.apache.slider.hbase.thrift2</name>
+ <value>http://${HBASE_THRIFT2_HOST}:${site.global.hbase_thrift2_port}</value>
+ </export>
+ <export>
<name>org.apache.slider.hbase.thrift</name>
<value>http://${HBASE_THRIFT_HOST}:${site.global.hbase_thrift_port}</value>
</export>
@@ -68,6 +72,7 @@
<command>HBASE_MASTER-START</command>
<requires>HBASE_REST-INSTALLED</requires>
<requires>HBASE_THRIFT-INSTALLED</requires>
+ <requires>HBASE_THRIFT2-INSTALLED</requires>
</commandOrder>
</commandOrders>
<components>
@@ -114,6 +119,16 @@
</component>
<component>
+ <name>HBASE_THRIFT2</name>
+ <category>SLAVE</category>
+ <minInstanceCount>1</minInstanceCount>
+ <commandScript>
+ <script>scripts/hbase_thrift2.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
+
+ <component>
<name>HBASE_CLIENT</name>
<category>CLIENT</category>
<minInstanceCount>0</minInstanceCount>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/package/scripts/hbase_service.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/scripts/hbase_service.py b/app-packages/hbase/package/scripts/hbase_service.py
index 01d38b3..96add84 100644
--- a/app-packages/hbase/package/scripts/hbase_service.py
+++ b/app-packages/hbase/package/scripts/hbase_service.py
@@ -39,6 +39,8 @@ def hbase_service(
daemon_cmd = format("{daemon_cmd} -p {rest_port}")
elif name == 'thrift':
daemon_cmd = format("{daemon_cmd} -p {thrift_port}")
+ elif name == 'thrift2':
+ daemon_cmd = format("{daemon_cmd} -p {thrift2_port}")
no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
elif action == 'stop':
daemon_cmd = format("{cmd} stop {role} && rm -f {pid_file}")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/package/scripts/hbase_thrift2.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/scripts/hbase_thrift2.py b/app-packages/hbase/package/scripts/hbase_thrift2.py
new file mode 100644
index 0000000..b72196c
--- /dev/null
+++ b/app-packages/hbase/package/scripts/hbase_thrift2.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+from hbase import hbase
+from hbase_service import hbase_service
+
+
+class HbaseThrift2(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ hbase(name='thrift2')
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env) # for security
+
+ hbase_service( 'thrift2',
+ action = 'start'
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ hbase_service( 'thrift2',
+ action = 'stop'
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ pid_file = format("{pid_dir}/hbase-{hbase_user}-thrift2.pid")
+ check_process_status(pid_file)
+
+if __name__ == "__main__":
+ HbaseThrift2().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/scripts/params.py b/app-packages/hbase/package/scripts/params.py
index 2d05289..1f25f68 100644
--- a/app-packages/hbase/package/scripts/params.py
+++ b/app-packages/hbase/package/scripts/params.py
@@ -59,6 +59,7 @@ ganglia_server_port = default('/configurations/global/ganglia_server_port', '866
rest_port = config['configurations']['global']['hbase_rest_port']
thrift_port = config['configurations']['global']['hbase_thrift_port']
+thrift2_port = config['configurations']['global']['hbase_thrift2_port']
if security_enabled:
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/app-packages/hbase/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/resources.json b/app-packages/hbase/resources.json
index 2d2b4f9..d2fdbd8 100644
--- a/app-packages/hbase/resources.json
+++ b/app-packages/hbase/resources.json
@@ -21,9 +21,14 @@
"yarn.role.priority": "3",
"yarn.component.instances": "1",
"yarn.memory": "256"
- }
+ },
"HBASE_THRIFT": {
- "yarn.role.priority": "3",
+ "yarn.role.priority": "4",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ },
+ "HBASE_THRIFT2": {
+ "yarn.role.priority": "5",
"yarn.component.instances": "1",
"yarn.memory": "256"
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
index fde2bf7..cd104a2 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
@@ -239,8 +239,12 @@ public class HBaseClientProvider extends AbstractClientProvider implements
providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_MASTER,
0, -1);
+ providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_REST_GATEWAY,
+ 0, -1);
providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_THRIFT_GATEWAY,
0, -1);
+ providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_THRIFT2_GATEWAY,
+ 0, -1);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
index 9c38fb2..2a20438 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
@@ -30,6 +30,8 @@ public interface HBaseKeys {
String ROLE_THRIFT_GATEWAY = "THRIFT";
+ String ROLE_THRIFT2_GATEWAY = "THRIFT2";
+
/** {@value */
String REGION_SERVER = "regionserver";
@@ -38,6 +40,9 @@ public interface HBaseKeys {
/** {@value */
String THRIFT_GATEWAY = "thrift";
+ /** {@value */
+ String THRIFT2_GATEWAY = "thrift2";
+ /**
/**
* What is the command for hbase to print a version: {@value}
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index 953ee79..f72967f 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -201,6 +201,11 @@ public class HBaseProviderService extends AbstractProviderService implements
logfile = "/thrift-gateway.txt";
break;
+ case ROLE_THRIFT2_GATEWAY:
+ roleCommand = THRIFT2_GATEWAY;
+
+ logfile = "/thrift2-gateway.txt";
+ break;
default:
throw new SliderInternalStateException("Cannot start role %s", role);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6f912df2/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
index 2cc457b..b2825b8 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
@@ -41,6 +41,7 @@ public class HBaseRoles {
public static final int KEY_THRIFT_GATEWAY = SliderKeys.ROLE_AM_PRIORITY_INDEX + 4;
+ public static final int KEY_THRIFT2_GATEWAY = SliderKeys.ROLE_AM_PRIORITY_INDEX + 5;
/**
* Initialize role list
*/
@@ -49,6 +50,7 @@ public class HBaseRoles {
ROLES.add(new ProviderRole(HBaseKeys.ROLE_MASTER, KEY_MASTER));
ROLES.add(new ProviderRole(HBaseKeys.ROLE_REST_GATEWAY, KEY_REST_GATEWAY));
ROLES.add(new ProviderRole(HBaseKeys.ROLE_THRIFT_GATEWAY, KEY_THRIFT_GATEWAY));
+ ROLES.add(new ProviderRole(HBaseKeys.ROLE_THRIFT_GATEWAY, KEY_THRIFT2_GATEWAY));
}
[17/50] [abbrv] git commit: Merge branch
'feature/SLIDER-270_Calling_flex_(down)_the_second_time_does_not_work' into
develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-270_Calling_flex_(down)_the_second_time_does_not_work' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d68be5aa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d68be5aa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d68be5aa
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: d68be5aa5eaa8c2e5a2d71097eb2907968c512bd
Parents: 405aa51 8d6c3fa
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 6 09:33:34 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 6 09:33:34 2014 +0100
----------------------------------------------------------------------
pom.xml | 2 +-
.../server/appmaster/SliderAppMaster.java | 16 +-
.../slider/server/appmaster/state/AppState.java | 142 +++++++++++-------
.../server/appmaster/state/RoleHistory.java | 6 +-
.../server/appmaster/state/RoleStatus.java | 4 +-
.../appstate/TestAppStateDynamicRoles.groovy | 4 +-
.../TestAppStateRebuildOnAMRestart.groovy | 4 +-
.../model/appstate/TestFlexDynamicRoles.groovy | 2 +
.../model/appstate/TestMockFlexing.groovy | 128 ++++++++++++++++
.../model/appstate/TestMockRMOperations.groovy | 37 -----
.../TestRoleHistoryContainerEvents.groovy | 2 +-
.../model/mock/BaseMockAppStateTest.groovy | 57 ++++++-
.../appmaster/model/mock/MockYarnEngine.groovy | 6 +-
.../slider/test/YarnMiniClusterTestBase.groovy | 7 +-
.../web/rest/agent/TestAMAgentWebServices.java | 3 +-
.../management/TestAMManagementWebServices.java | 1 +
.../minicluster/HBaseMiniClusterTestBase.groovy | 150 ++++++++++++-------
.../flexing/TestClusterFlex2To5.groovy | 2 +-
.../flexing/TestClusterFlexDownMultiple.groovy | 58 +++++++
.../TestClusterFlexHMasterFlex1To2.groovy | 45 ++++++
.../flexing/TestHMasterFlex1To2.groovy | 45 ------
.../src/test/resources/log4j.properties | 1 +
22 files changed, 497 insertions(+), 225 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d68be5aa/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
[49/50] [abbrv] git commit: SLIDER-302 add a renewing action which
executes then rechedules a nested action
Posted by st...@apache.org.
SLIDER-302 add a renewing action which executes then rechedules a nested action
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8ac67486
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8ac67486
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8ac67486
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 8ac67486f79a0e12a3047e9fadf440bc88c13d51
Parents: 323a80f
Author: Steve Loughran <st...@apache.org>
Authored: Sat Aug 9 13:46:05 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Sat Aug 9 13:46:05 2014 +0100
----------------------------------------------------------------------
.../server/appmaster/actions/ActionHalt.java | 9 +-
.../appmaster/actions/ActionStartContainer.java | 6 +-
.../appmaster/actions/ActionStopQueue.java | 9 +-
.../appmaster/actions/ActionStopSlider.java | 2 +-
.../server/appmaster/actions/AsyncAction.java | 55 ++++++++--
.../actions/ProviderReportedContainerLoss.java | 2 +-
.../actions/ProviderStartupCompleted.java | 2 +-
.../server/appmaster/actions/QueueAccess.java | 3 +
.../server/appmaster/actions/QueueExecutor.java | 2 +-
.../appmaster/actions/RenewingAction.java | 105 +++++++++++++++++++
.../server/appmaster/actions/TestActions.groovy | 37 ++++++-
11 files changed, 208 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ac67486/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
index 1468cd8..44337fc 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
@@ -21,6 +21,10 @@ package org.apache.slider.server.appmaster.actions;
import org.apache.hadoop.util.ExitUtil;
import org.apache.slider.server.appmaster.SliderAppMaster;
+/**
+ * Exit a JVM halt.
+ * @see ExitUtil#halt(int, String)
+ */
public class ActionHalt extends AsyncAction {
private final int status;
@@ -28,14 +32,15 @@ public class ActionHalt extends AsyncAction {
public ActionHalt(
int status,
- String text, int delay) {
+ String text,
+ int delay) {
super("Halt", delay, ActionAttributes.HALTS_CLUSTER);
this.status = status;
this.text = text;
}
@Override
- public void execute(SliderAppMaster appMaster) throws Exception {
+ public void execute(SliderAppMaster appMaster, QueueAccess queueService) throws Exception {
ExitUtil.halt(status, text);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ac67486/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
index 6179f35..1a86e5c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
@@ -23,6 +23,10 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.state.RoleInstance;
+/**
+ * Start a container
+ * @see SliderAppMaster#startContainer(Container, ContainerLaunchContext, RoleInstance)
+ */
public class ActionStartContainer extends AsyncAction {
private final Container container;
@@ -41,7 +45,7 @@ public class ActionStartContainer extends AsyncAction {
}
@Override
- public void execute(SliderAppMaster appMaster) throws Exception {
+ public void execute(SliderAppMaster appMaster, QueueAccess queueService) throws Exception {
appMaster.startContainer(container, ctx, instance);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ac67486/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
index a11bccb..7a31f12 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
@@ -20,6 +20,8 @@ package org.apache.slider.server.appmaster.actions;
import org.apache.slider.server.appmaster.SliderAppMaster;
+import java.util.concurrent.TimeUnit;
+
/**
* Action to tell a queue executor to stop -after handing this on/executing it
*/
@@ -29,8 +31,13 @@ public class ActionStopQueue extends AsyncAction {
super("stop queue", delay);
}
+ public ActionStopQueue(int delay,
+ TimeUnit timeUnit) {
+ super("stop queue", delay, timeUnit);
+ }
+
@Override
- public void execute(SliderAppMaster appMaster) throws Exception {
+ public void execute(SliderAppMaster appMaster, QueueAccess queueService) throws Exception {
// no-op
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ac67486/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
index 0e6dc25..24cad1c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
@@ -28,7 +28,7 @@ public class ActionStopSlider extends AsyncAction {
}
@Override
- public void execute(SliderAppMaster appMaster) throws Exception {
+ public void execute(SliderAppMaster appMaster, QueueAccess queueService) throws Exception {
String message = name;
SliderAppMaster.getLog().info("SliderAppMasterApi.stopCluster: {}",
message);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ac67486/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
index 5e325a0..fed28e4 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
@@ -33,7 +33,7 @@ public abstract class AsyncAction implements Delayed {
private static final AtomicLong sequencer = new AtomicLong(0);
public final String name;
- public final long nanos;
+ private long nanos;
private final EnumSet<ActionAttributes> attrs;
private final long sequenceNumber = sequencer.incrementAndGet();
@@ -44,36 +44,55 @@ public abstract class AsyncAction implements Delayed {
protected AsyncAction(String name,
int delayMillis) {
+ this(name, delayMillis, TimeUnit.MILLISECONDS);
+ }
+
+ protected AsyncAction(String name,
+ int delay,
+ TimeUnit timeUnit) {
this.name = name;
- this.nanos = convertAndOffset(delayMillis);
+ this.setNanos(convertAndOffset(delay, timeUnit));
attrs = EnumSet.noneOf(ActionAttributes.class);
}
protected AsyncAction(String name,
- int delayMillis, EnumSet<ActionAttributes> attrs) {
+ int delay,
+ TimeUnit timeUnit,
+ EnumSet<ActionAttributes> attrs) {
this.name = name;
- this.nanos = convertAndOffset(delayMillis);
+ this.setNanos(convertAndOffset(delay, timeUnit));
this.attrs = attrs;
}
protected AsyncAction(String name,
- int delayMillis,
+ int delay,
+ TimeUnit timeUnit,
ActionAttributes... attributes) {
- this(name, delayMillis);
+ this(name, delay, timeUnit);
Collections.addAll(attrs, attributes);
}
+
+ protected AsyncAction(String name,
+ int delayMillis,
+ ActionAttributes... attributes) {
+ this(name, delayMillis, TimeUnit.MILLISECONDS);
+ }
- private long convertAndOffset(int delay) {
- return now() + TimeUnit.NANOSECONDS.convert(delay, TimeUnit.MILLISECONDS);
+ protected long convertAndOffset(int delay, TimeUnit timeUnit) {
+ return now() + TimeUnit.NANOSECONDS.convert(delay, timeUnit);
}
+ /**
+ * The current time in nanos
+ * @return now
+ */
protected long now() {
return System.nanoTime();
}
@Override
public long getDelay(TimeUnit unit) {
- return unit.convert(nanos - now(), TimeUnit.NANOSECONDS);
+ return unit.convert(getNanos() - now(), TimeUnit.NANOSECONDS);
}
@Override
@@ -91,13 +110,17 @@ public abstract class AsyncAction implements Delayed {
final StringBuilder sb =
new StringBuilder(super.toString());
sb.append(" name='").append(name).append('\'');
- sb.append(", nanos=").append(nanos);
+ sb.append(", nanos=").append(getNanos());
sb.append(", attrs=").append(attrs);
sb.append(", sequenceNumber=").append(sequenceNumber);
sb.append('}');
return sb.toString();
}
+ protected EnumSet<ActionAttributes> getAttrs() {
+ return attrs;
+ }
+
/**
* Ask if an action has a specific attribute
* @param attr attribute
@@ -110,9 +133,19 @@ public abstract class AsyncAction implements Delayed {
/**
* Actual application
* @param appMaster
+ * @param queueService
* @throws IOException
*/
- public abstract void execute(SliderAppMaster appMaster) throws Exception;
+ public abstract void execute(SliderAppMaster appMaster,
+ QueueAccess queueService) throws Exception;
+
+ public long getNanos() {
+ return nanos;
+ }
+
+ public void setNanos(long nanos) {
+ this.nanos = nanos;
+ }
public enum ActionAttributes {
SHRINKS_CLUSTER,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ac67486/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
index 63b40d5..6a2cc6b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
@@ -41,7 +41,7 @@ public class ProviderReportedContainerLoss extends AsyncAction {
}
@Override
- public void execute(SliderAppMaster appMaster) throws Exception {
+ public void execute(SliderAppMaster appMaster, QueueAccess queueService) throws Exception {
appMaster.providerLostContainer(containerId);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ac67486/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
index 293492f..4e06f7a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
@@ -31,7 +31,7 @@ public class ProviderStartupCompleted extends AsyncAction {
}
@Override
- public void execute(SliderAppMaster appMaster) throws Exception {
+ public void execute(SliderAppMaster appMaster, QueueAccess queueService) throws Exception {
appMaster.eventCallbackEvent(null);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ac67486/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
index f717085..160333d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
@@ -18,6 +18,9 @@
package org.apache.slider.server.appmaster.actions;
+/**
+ * Access for queue operations
+ */
public interface QueueAccess {
/**
* Put an action on the immediate queue -to be executed when the queue
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ac67486/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
index 072f324..bc2b260 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
@@ -62,7 +62,7 @@ public class QueueExecutor implements Runnable {
do {
take = actionQueues.actionQueue.take();
log.debug("Executing {}", take);
- take.execute(appMaster);
+ take.execute(appMaster, actionQueues);
} while (!(take instanceof ActionStopQueue));
log.info("Queue Executor run() stopped");
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ac67486/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RenewingAction.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RenewingAction.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RenewingAction.java
new file mode 100644
index 0000000..30870bf
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RenewingAction.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * This action executes then reschedules an inner action; a limit
+ * can specify the number of times to run
+ */
+
+public class RenewingAction extends AsyncAction{
+ private static final Logger log =
+ LoggerFactory.getLogger(RenewingAction.class);
+ private final AsyncAction action;
+ private final int interval;
+ private final TimeUnit timeUnit;
+ public final AtomicInteger executionCount = new AtomicInteger();
+ public final int limit;
+
+
+ /**
+ * Rescheduling action
+ * @param action action to execute
+ * @param initialDelay initial delay
+ * @param interval interval for later delays
+ * @param timeUnit time unit for all times
+ * @param limit limit on the no. of executions. If 0 or less: no limit
+ */
+ public RenewingAction(AsyncAction action,
+ int initialDelay,
+ int interval, TimeUnit timeUnit,
+ int limit) {
+ super("renewing " + action.name, initialDelay, timeUnit, action.getAttrs());
+ this.action = action;
+ this.interval = interval;
+ this.timeUnit = timeUnit;
+ this.limit = limit;
+ }
+
+ /**
+ * Execute the inner action then reschedule ourselves
+ * @param appMaster
+ * @param queueService
+ * @throws Exception
+ */
+ @Override
+ public void execute(SliderAppMaster appMaster, QueueAccess queueService)
+ throws Exception {
+ long exCount = executionCount.incrementAndGet();
+ log.debug("{}: Executing inner action count # {}", this, exCount);
+ action.execute(appMaster, queueService);
+ boolean reschedule = true;
+ if (limit > 0) {
+ reschedule = limit > exCount;
+ }
+ if (reschedule) {
+ this.setNanos(convertAndOffset(interval, timeUnit));
+ log.debug("{}: rescheduling, new offset {} mS ", this,
+ getDelay(TimeUnit.MILLISECONDS));
+ queueService.putDelayed(this);
+ }
+ }
+
+ public AsyncAction getAction() {
+ return action;
+ }
+
+ public int getInterval() {
+ return interval;
+ }
+
+ public TimeUnit getTimeUnit() {
+ return timeUnit;
+ }
+
+ public int getExecutionCount() {
+ return executionCount.get();
+ }
+
+ public int getLimit() {
+ return limit;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ac67486/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
index 5dffea0..1e22acd 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
@@ -139,27 +139,52 @@ class TestActions {
queues.putDelayed(note2)
queues.putDelayed(stop)
// async to sync expected to run in order
+ runQueuesToCompletion()
+ assert note1.executed.get()
+ assert note2.executed.get()
+ }
+
+ public void runQueuesToCompletion() {
queues.run();
assert queues.delayedActions.empty
assert !queues.actionQueue.empty
QueueExecutor ex = new QueueExecutor(queues)
ex.run();
assert queues.actionQueue.empty
- assert note1.executed.get()
- assert note2.executed.get()
+ }
+
+ @Test
+ public void testRenewedActionFiresOnceAtLeast() throws Throwable {
+ ActionNoteExecuted note1 = new ActionNoteExecuted("note1", 500)
+ RenewingAction renewer = new RenewingAction(
+ note1,
+ 500,
+ 100,
+ TimeUnit.MILLISECONDS,
+ 3)
+ queues.putDelayed(renewer);
+ def stop = new ActionStopQueue(4, TimeUnit.SECONDS)
+ queues.putDelayed(stop);
+ // this runs all the delayed actions FIRST, so can't be used
+ // to play tricks of renewing actions ahead of the stop action
+ runQueuesToCompletion()
+ assert renewer.executionCount == 1
+ assert note1.executionCount == 1
+ // assert the renewed item is back in
+ assert queues.delayedActions.contains(renewer)
}
public class ActionNoteExecuted extends AsyncAction {
public final AtomicBoolean executed = new AtomicBoolean(false);
public final AtomicLong executionTimeNanos = new AtomicLong()
- public final AtomicLong executionCount = new AtomicLong()
+ private final AtomicLong executionCount = new AtomicLong()
public ActionNoteExecuted(String text, int delay) {
super(text, delay);
}
@Override
- public void execute(SliderAppMaster appMaster) throws Exception {
+ public void execute(SliderAppMaster appMaster, QueueAccess queueService) throws Exception {
log.info("Executing $name");
executed.set(true);
executionTimeNanos.set(System.nanoTime())
@@ -175,6 +200,8 @@ class TestActions {
" executed=${executed.get()}; count=${executionCount.get()};"
}
-
+ long getExecutionCount() {
+ return executionCount.get()
+ }
}
}
[40/50] [abbrv] git commit: SLIDER-303 Enhance get-hbase-site.sh to
obtain information about gateways
Posted by st...@apache.org.
SLIDER-303 Enhance get-hbase-site.sh to obtain information about gateways
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a1bbf4db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a1bbf4db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a1bbf4db
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: a1bbf4dba7613ca2d8b0942ed2b3df3a8bebe17a
Parents: 66cbd37
Author: tedyu <yu...@gmail.com>
Authored: Fri Aug 8 10:45:48 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Fri Aug 8 10:45:48 2014 -0700
----------------------------------------------------------------------
app-packages/hbase/get-hbase-site.sh | 26 +++++++++-----------------
app-packages/hbase/links.py | 15 +++++++++++++++
2 files changed, 24 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a1bbf4db/app-packages/hbase/get-hbase-site.sh
----------------------------------------------------------------------
diff --git a/app-packages/hbase/get-hbase-site.sh b/app-packages/hbase/get-hbase-site.sh
index 04f9d3d..5211d83 100755
--- a/app-packages/hbase/get-hbase-site.sh
+++ b/app-packages/hbase/get-hbase-site.sh
@@ -1,25 +1,17 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
tuple=`slider status $1 | grep "info.am.web.url"`
+echo $tuple
FS=":"
url=`echo $tuple | awk '{split($0,array,": ")} END{print array[2]}'`
url="${url%,}"
url="${url%\"}"
url="${url#\"}"
-url="${url}ws/v1/slider/publisher/slider/hbase-site.xml"
-curl -k -o hbase-site.dnld $url
+siteurl="${url}ws/v1/slider/publisher/slider/hbase-site.xml"
+curl -k -o hbase-site.dnld $siteurl
grep -v 'hbase.tmp.dir' hbase-site.dnld > hbase-site.xml
+
+linksurl="${url}ws/v1/slider/publisher/slider/quicklinks"
+curl -k -o links.json $linksurl
+python $DIR/links.py
+#| sed -e 's/\/\///g' | awk 'BEGIN { FS = ":" } ; { print $2 }'
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a1bbf4db/app-packages/hbase/links.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/links.py b/app-packages/hbase/links.py
new file mode 100644
index 0000000..c77e620
--- /dev/null
+++ b/app-packages/hbase/links.py
@@ -0,0 +1,15 @@
+from __future__ import print_function
+import logging
+import json
+
+file = open("links.json")
+links = json.load(file)
+file.close()
+if links.has_key("entries"):
+ entries = links["entries"]
+ if entries.has_key("org.apache.slider.hbase.rest"):
+ print("org.apache.slider.hbase.rest : %s" % entries["org.apache.slider.hbase.rest"])
+ if entries.has_key("org.apache.slider.hbase.thrift"):
+ print("org.apache.slider.hbase.thrift : %s" % entries["org.apache.slider.hbase.thrift"])
+ if entries.has_key("org.apache.slider.hbase.thrift2"):
+ print("org.apache.slider.hbase.thrift2 : %s" % entries["org.apache.slider.hbase.thrift2"])
[44/50] [abbrv] git commit: Merge branch 'develop' of
https://git-wip-us.apache.org/repos/asf/incubator-slider into develop
Posted by st...@apache.org.
Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c2150453
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c2150453
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c2150453
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: c2150453e0faa5ff4199a95e8885512b0f6a7b36
Parents: bfb4cbb a1bbf4d
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 8 19:17:32 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 8 19:17:32 2014 +0100
----------------------------------------------------------------------
app-packages/hbase/get-hbase-site.sh | 26 +++++++++-----------------
app-packages/hbase/links.py | 15 +++++++++++++++
2 files changed, 24 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
[38/50] [abbrv] git commit: SLIDER-302 role launch moves to async
model
Posted by st...@apache.org.
SLIDER-302 role launch moves to async model
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e75f605f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e75f605f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e75f605f
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: e75f605fb569eacab9d8e9e449e8f764e9fec13e
Parents: 3d02e1c
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 8 17:21:56 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 8 17:21:56 2014 +0100
----------------------------------------------------------------------
.../server/appmaster/RoleLaunchService.java | 18 +++++-----
.../server/appmaster/SliderAppMaster.java | 3 +-
.../server/appmaster/actions/QueueAccess.java | 35 ++++++++++++++++++++
.../server/appmaster/actions/QueueService.java | 4 ++-
4 files changed, 47 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e75f605f/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
index b83c905..e8b6802 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
@@ -30,6 +30,7 @@ import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.actions.ActionStartContainer;
import org.apache.slider.server.appmaster.actions.AsyncAction;
+import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.RoleStatus;
import org.apache.slider.server.services.workflow.WorkflowExecutorService;
@@ -52,13 +53,11 @@ public class RoleLaunchService
public static final String ROLE_LAUNCH_SERVICE = "RoleLaunchService";
+
/**
- * Callback to whatever has the task of actually running the container
- * start operation
+ * Queue submission API
*/
- private final ContainerStartOperation containerStarter;
-
- private final Queue<AsyncAction> actionQueue;
+ private final QueueAccess actionQueue;
/**
* Provider bulding up the command
@@ -93,15 +92,14 @@ public class RoleLaunchService
* @param envVars environment variables
* @param launcherTmpDirPath path for a temporary data in the launch process
*/
- public RoleLaunchService(ContainerStartOperation startOperation,
- Queue<AsyncAction> actionQueue, ProviderService provider,
+ public RoleLaunchService(QueueAccess queueAccess,
+ ProviderService provider,
SliderFileSystem fs,
Path generatedConfDirPath,
Map<String, String> envVars,
Path launcherTmpDirPath) {
super(ROLE_LAUNCH_SERVICE);
- containerStarter = startOperation;
- this.actionQueue = actionQueue;
+ this.actionQueue = queueAccess;
this.fs = fs;
this.generatedConfDirPath = generatedConfDirPath;
this.launcherTmpDirPath = launcherTmpDirPath;
@@ -216,7 +214,7 @@ public class RoleLaunchService
instance.role = containerRole;
instance.roleId = role.id;
instance.environment = envDescription;
- actionQueue.add(new ActionStartContainer("starting " + containerRole,
+ actionQueue.put(new ActionStartContainer("starting " + containerRole,
0, container,
containerLauncher.completeContainerLaunch(),
instance));
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e75f605f/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index b1ac6bb..b82967f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -750,8 +750,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
fs.getFileSystem().mkdirs(launcherTmpDirPath);
//launcher service
- launchService = new RoleLaunchService(this,
- actionQueues.actionQueue,
+ launchService = new RoleLaunchService(actionQueues,
providerService,
fs,
new Path(getGeneratedConfDir()),
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e75f605f/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
new file mode 100644
index 0000000..f717085
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueAccess.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+public interface QueueAccess {
+ /**
+ * Put an action on the immediate queue -to be executed when the queue
+ * reaches it.
+ * @param action action to queue
+ */
+ void put(AsyncAction action);
+
+ /**
+ * Put a delayed action: this will only be added to the main queue
+ * after its action time has been reached
+ * @param action action to queue
+ */
+ void putDelayed(AsyncAction action);
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e75f605f/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
index 69cb9f7..9e4e8d0 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
@@ -28,7 +28,7 @@ import java.util.concurrent.DelayQueue;
import java.util.concurrent.LinkedBlockingDeque;
public class QueueService extends AbstractService
- implements Runnable {
+ implements Runnable, QueueAccess {
private static final Logger log =
LoggerFactory.getLogger(QueueService.class);
@@ -41,11 +41,13 @@ public class QueueService extends AbstractService
super("action queue");
}
+ @Override
public void put(AsyncAction action) {
log.debug("Queueing {}", action);
actionQueue.add(action);
}
+ @Override
public void putDelayed(AsyncAction action) {
log.debug("Delayed Queueing {}", action);
delayedActions.add(action);
[43/50] [abbrv] git commit: Merge branch
'feature/SLIDER-302_scheduled_executor' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-302_scheduled_executor' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bfb4cbb0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bfb4cbb0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bfb4cbb0
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: bfb4cbb030a979edde16334342b72071e4d51e80
Parents: 66cbd37 cf2aa70
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 8 19:17:18 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 8 19:17:18 2014 +0100
----------------------------------------------------------------------
.../apache/slider/common/tools/SliderUtils.java | 17 +-
.../providers/AbstractProviderService.java | 12 +-
.../slider/providers/ProviderService.java | 16 +-
.../providers/agent/AgentProviderService.java | 15 +-
.../providers/agent/HeartbeatMonitor.java | 9 +-
.../server/appmaster/AMViewForProviders.java | 46 -----
.../appmaster/ContainerStartOperation.java | 40 -----
.../server/appmaster/RoleLaunchService.java | 40 +++--
.../server/appmaster/SliderAppMaster.java | 134 ++++++++++----
.../server/appmaster/actions/ActionHalt.java | 41 +++++
.../appmaster/actions/ActionStartContainer.java | 47 +++++
.../appmaster/actions/ActionStopQueue.java | 36 ++++
.../appmaster/actions/ActionStopSlider.java | 39 ++++
.../server/appmaster/actions/AsyncAction.java | 124 +++++++++++++
.../actions/ProviderReportedContainerLoss.java | 47 +++++
.../actions/ProviderStartupCompleted.java | 37 ++++
.../server/appmaster/actions/QueueAccess.java | 35 ++++
.../server/appmaster/actions/QueueExecutor.java | 75 ++++++++
.../server/appmaster/actions/QueueService.java | 76 ++++++++
.../appmaster/actions/StartContainerAction.java | 41 +++++
.../AbstractWorkflowExecutorService.java | 110 ------------
.../services/workflow/ForkedProcessService.java | 6 +-
.../workflow/WorkflowCallbackService.java | 3 +-
.../workflow/WorkflowExecutorService.java | 113 ++++++++++++
.../WorkflowScheduledExecutorService.java | 38 ++++
.../server/services/workflow/package-info.java | 2 +-
.../standalone/TestDestroyStandaloneAM.groovy | 156 ----------------
.../standalone/TestKillStandaloneAM.groovy | 73 --------
.../standalone/TestRestartStandaloneAM.groovy | 96 ----------
.../standalone/TestStandaloneAMDestroy.groovy | 154 ++++++++++++++++
.../standalone/TestStandaloneAMKill.groovy | 73 ++++++++
.../standalone/TestStandaloneAMRestart.groovy | 96 ++++++++++
.../server/appmaster/actions/TestActions.groovy | 180 +++++++++++++++++++
.../model/history/TestRoleHistoryRW.groovy | 2 +-
.../model/mock/MockProviderService.groovy | 4 +-
.../providers/agent/TestHeartbeatMonitor.java | 6 -
.../publisher/TestAgentProviderService.java | 6 +-
.../workflow/TestWorkflowExecutorService.java | 9 +-
38 files changed, 1428 insertions(+), 626 deletions(-)
----------------------------------------------------------------------
[36/50] [abbrv] git commit: SLIDER-302 queue and actions,
tests not quite working
Posted by st...@apache.org.
SLIDER-302 queue and actions, tests not quite working
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7ef046b6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7ef046b6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7ef046b6
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 7ef046b61ca0230b1485b24eb9a4f1dc55ba7e0e
Parents: 66cbd37
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 8 14:17:38 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 8 16:53:50 2014 +0100
----------------------------------------------------------------------
.../apache/slider/common/tools/SliderUtils.java | 17 +-
.../server/appmaster/RoleLaunchService.java | 32 +++-
.../server/appmaster/SliderAppMaster.java | 94 +++++++---
.../server/appmaster/actions/ActionHalt.java | 43 +++++
.../appmaster/actions/ActionStartContainer.java | 49 +++++
.../appmaster/actions/ActionStopQueue.java | 38 ++++
.../appmaster/actions/ActionStopSlider.java | 41 +++++
.../server/appmaster/actions/AsyncAction.java | 124 +++++++++++++
.../server/appmaster/actions/QueueExecutor.java | 75 ++++++++
.../server/appmaster/actions/QueueService.java | 70 ++++++++
.../appmaster/actions/StartContainerAction.java | 41 +++++
.../AbstractWorkflowExecutorService.java | 110 ------------
.../services/workflow/ForkedProcessService.java | 6 +-
.../workflow/WorkflowCallbackService.java | 3 +-
.../workflow/WorkflowExecutorService.java | 113 ++++++++++++
.../WorkflowScheduledExecutorService.java | 38 ++++
.../server/services/workflow/package-info.java | 2 +-
.../standalone/TestDestroyStandaloneAM.groovy | 156 ----------------
.../standalone/TestKillStandaloneAM.groovy | 73 --------
.../standalone/TestRestartStandaloneAM.groovy | 96 ----------
.../standalone/TestStandaloneAMDestroy.groovy | 154 ++++++++++++++++
.../standalone/TestStandaloneAMKill.groovy | 73 ++++++++
.../standalone/TestStandaloneAMRestart.groovy | 96 ++++++++++
.../server/appmaster/actions/TestActions.groovy | 180 +++++++++++++++++++
.../model/history/TestRoleHistoryRW.groovy | 2 +-
.../workflow/TestWorkflowExecutorService.java | 9 +-
26 files changed, 1262 insertions(+), 473 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 8e412ee..cda0cf5 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -1073,7 +1073,7 @@ public final class SliderUtils {
UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
return currentUser;
} catch (IOException e) {
- log.info("Failed to grt user info", e);
+ log.info("Failed to get user info", e);
throw e;
}
}
@@ -1415,6 +1415,21 @@ public final class SliderUtils {
}
/**
+ * A compareTo function that converts the result of a long
+ * comparision into the integer that <code>Comparable</code>
+ * expects.
+ * @param left left side
+ * @param right right side
+ * @return -1, 0, 1 depending on the diff
+ */
+ public static int compareTo(long left, long right) {
+ long diff = left - right;
+ if (diff < 0) return -1;
+ if (diff > 0) return 1;
+ return 0;
+ }
+
+ /**
* This wrapps ApplicationReports and generates a string version
* iff the toString() operator is invoked
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
index 5a5baaa..b83c905 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
@@ -28,20 +28,25 @@ import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderService;
+import org.apache.slider.server.appmaster.actions.ActionStartContainer;
+import org.apache.slider.server.appmaster.actions.AsyncAction;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.apache.slider.server.services.workflow.AbstractWorkflowExecutorService;
+import org.apache.slider.server.services.workflow.WorkflowExecutorService;
import org.apache.slider.server.services.workflow.ServiceThreadFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Map;
+import java.util.Queue;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* A service for launching containers
*/
-public class RoleLaunchService extends AbstractWorkflowExecutorService {
+public class RoleLaunchService
+ extends WorkflowExecutorService<ExecutorService> {
protected static final Logger log =
LoggerFactory.getLogger(RoleLaunchService.class);
@@ -53,7 +58,13 @@ public class RoleLaunchService extends AbstractWorkflowExecutorService {
*/
private final ContainerStartOperation containerStarter;
+ private final Queue<AsyncAction> actionQueue;
+
+ /**
+ * Provider bulding up the command
+ */
private final ProviderService provider;
+
/**
* Filesystem to use for the launch
*/
@@ -75,6 +86,7 @@ public class RoleLaunchService extends AbstractWorkflowExecutorService {
/**
* Construct an instance of the launcher
* @param startOperation the callback to start the opreation
+ * @param actionQueue
* @param provider the provider
* @param fs filesystem
* @param generatedConfDirPath path in the FS for the generated dir
@@ -82,13 +94,14 @@ public class RoleLaunchService extends AbstractWorkflowExecutorService {
* @param launcherTmpDirPath path for a temporary data in the launch process
*/
public RoleLaunchService(ContainerStartOperation startOperation,
- ProviderService provider,
- SliderFileSystem fs,
- Path generatedConfDirPath,
- Map<String, String> envVars,
+ Queue<AsyncAction> actionQueue, ProviderService provider,
+ SliderFileSystem fs,
+ Path generatedConfDirPath,
+ Map<String, String> envVars,
Path launcherTmpDirPath) {
super(ROLE_LAUNCH_SERVICE);
containerStarter = startOperation;
+ this.actionQueue = actionQueue;
this.fs = fs;
this.generatedConfDirPath = generatedConfDirPath;
this.launcherTmpDirPath = launcherTmpDirPath;
@@ -203,9 +216,10 @@ public class RoleLaunchService extends AbstractWorkflowExecutorService {
instance.role = containerRole;
instance.roleId = role.id;
instance.environment = envDescription;
- containerStarter.startContainer(container,
- containerLauncher.completeContainerLaunch(),
- instance);
+ actionQueue.add(new ActionStartContainer("starting " + containerRole,
+ 0, container,
+ containerLauncher.completeContainerLaunch(),
+ instance));
} catch (Exception e) {
log.error("Exception thrown while trying to start {}: {}",
containerRole, e);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index bc63351..b1ac6bb 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -94,6 +94,11 @@ import org.apache.slider.providers.ProviderService;
import org.apache.slider.providers.SliderProviderFactory;
import org.apache.slider.providers.slideram.SliderAMClientProvider;
import org.apache.slider.providers.slideram.SliderAMProviderService;
+import org.apache.slider.server.appmaster.actions.QueueExecutor;
+import org.apache.slider.server.appmaster.actions.ActionHalt;
+import org.apache.slider.server.appmaster.actions.QueueService;
+import org.apache.slider.server.appmaster.actions.ActionStopSlider;
+import org.apache.slider.server.appmaster.actions.AsyncAction;
import org.apache.slider.server.appmaster.operations.AsyncRMOperationHandler;
import org.apache.slider.server.appmaster.operations.ProviderNotifyingOperationHandler;
import org.apache.slider.server.appmaster.rpc.RpcBinder;
@@ -120,7 +125,10 @@ import org.apache.slider.server.services.registry.SliderRegistryService;
import org.apache.slider.server.services.security.CertificateManager;
import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
import org.apache.slider.server.services.utility.WebAppService;
+import org.apache.slider.server.services.workflow.ServiceThreadFactory;
+import org.apache.slider.server.services.workflow.WorkflowExecutorService;
import org.apache.slider.server.services.workflow.WorkflowRpcService;
+import org.apache.slider.server.services.workflow.WorkflowScheduledExecutorService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -133,12 +141,14 @@ import java.net.URLClassLoader;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
@@ -183,6 +193,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
public static final int HEARTBEAT_INTERVAL = 1000;
public static final int NUM_RPC_HANDLERS = 5;
+ public static final int SCHEDULED_EXECUTOR_POOL_SIZE = 1;
/** YARN RPC to communicate with the Resource Manager or Node Manager */
private YarnRPC yarnRPC;
@@ -299,7 +310,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
*/
private int containerMaxCores;
-
/**
* limit container memory
*/
@@ -321,6 +331,14 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
private String agentAccessUrl;
private CertificateManager certificateManager;
+ private WorkflowScheduledExecutorService<ScheduledExecutorService>
+ scheduledExecutors;
+ private WorkflowExecutorService<ExecutorService>
+ executorService;
+
+
+ private final QueueService actionQueues = new QueueService();
+
/**
* Service Constructor
*/
@@ -328,9 +346,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
super(SERVICE_CLASSNAME_SHORT);
}
-
-
- /* =================================================================== */
+/* =================================================================== */
/* service lifecycle methods */
/* =================================================================== */
@@ -374,11 +390,33 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//look at settings of Hadoop Auth, to pick up a problem seen once
checkAndWarnForAuthTokenProblems();
+ scheduledExecutors =
+ new WorkflowScheduledExecutorService<>(
+ "AmExecutor",
+ Executors.newScheduledThreadPool(
+ SCHEDULED_EXECUTOR_POOL_SIZE,
+ new ServiceThreadFactory("AmScheduledExecutor", true)));
+ addService(scheduledExecutors);
+
+ executorService = new WorkflowExecutorService<>("AmExecutor",
+ Executors.newCachedThreadPool(
+ new ServiceThreadFactory("AmExecutor", true)));
+ addService(executorService);
+
+
+ addService(actionQueues);
//init all child services
super.serviceInit(conf);
}
+
+ @Override
+ protected void serviceStart() throws Exception {
+ super.serviceStart();
+ executorService.execute(new QueueExecutor(this, actionQueues));
+ executorService.execute(actionQueues);
+ }
-/* =================================================================== */
+ /* =================================================================== */
/* RunService methods called from ServiceLauncher */
/* =================================================================== */
@@ -713,6 +751,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//launcher service
launchService = new RoleLaunchService(this,
+ actionQueues.actionQueue,
providerService,
fs,
new Path(getGeneratedConfDir()),
@@ -893,6 +932,14 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
/**
+ * Get the AM log
+ * @return the log of the AM
+ */
+ public static Logger getLog() {
+ return log;
+ }
+
+ /**
* Block until it is signalled that the AM is done
*/
private void waitForAMCompletionSignal() {
@@ -1208,8 +1255,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
YarnException {
SliderUtils.getCurrentUser();
String message = request.getMessage();
- log.info("SliderAppMasterApi.stopCluster: {}",message);
- signalAMComplete(EXIT_CLIENT_INITIATED_SHUTDOWN, message);
+ log.info("SliderAppMasterApi.stopCluster: {}", message);
+ queue(new ActionStopSlider(message, 1000));
return Messages.StopClusterResponseProto.getDefaultInstance();
}
@@ -1369,17 +1416,16 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
@Override
- public Messages.AMSuicideResponseProto amSuicide(Messages.AMSuicideRequestProto request) throws
- IOException,
- YarnException {
+ public Messages.AMSuicideResponseProto amSuicide(
+ Messages.AMSuicideRequestProto request)
+ throws IOException, YarnException {
int signal = request.getSignal();
String text = request.getText();
int delay = request.getDelay();
log.info("AM Suicide with signal {}, message {} delay = {}", signal, text, delay);
- SliderUtils.haltAM(signal, text, delay);
- Messages.AMSuicideResponseProto.Builder builder =
- Messages.AMSuicideResponseProto.newBuilder();
- return builder.build();
+ ActionHalt action = new ActionHalt(signal, text, delay);
+ schedule(action);
+ return Messages.AMSuicideResponseProto.getDefaultInstance();
}
/* =================================================================== */
@@ -1576,7 +1622,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
public void onContainerStatusReceived(ContainerId containerId,
ContainerStatus containerStatus) {
LOG_YARN.debug("Container Status: id={}, status={}", containerId,
- containerStatus);
+ containerStatus);
}
@Override // NMClientAsync.CallbackHandler
@@ -1616,11 +1662,19 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
/**
- * Get the username for the slider cluster as set in the environment
- * @return the username or null if none was set/it is a secure cluster
+ * Queue an action for immediate execution in the executor thread
+ * @param action action to execute
+ */
+ public void queue(AsyncAction action) {
+ actionQueues.actionQueue.add(action);
+ }
+
+ /**
+ * Schedule an action
+ * @param action for delayed execution
*/
- public String getHadoop_user_name() {
- return hadoop_user_name;
+ public void schedule(AsyncAction action) {
+ actionQueues.delayedActions.add(action);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
new file mode 100644
index 0000000..a16d207
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+
+import java.io.IOException;
+
+public class ActionHalt extends AsyncAction {
+
+ private final int status;
+ private final String text;
+
+ public ActionHalt(
+ int status,
+ String text, int delay) {
+ super("Halt", delay, ActionAttributes.HALTS_CLUSTER);
+ this.status = status;
+ this.text = text;
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster) throws IOException {
+ ExitUtil.halt(status, text);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
new file mode 100644
index 0000000..2d643ed
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+
+import java.io.IOException;
+
+public class ActionStartContainer extends AsyncAction {
+
+ private final Container container;
+ private final ContainerLaunchContext ctx;
+ private final RoleInstance instance;
+
+ public ActionStartContainer(String name,
+ int delay,
+ Container container,
+ ContainerLaunchContext ctx,
+ RoleInstance instance) {
+ super(name, delay);
+ this.container = container;
+ this.ctx = ctx;
+ this.instance = instance;
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster) throws IOException {
+ appMaster.startContainer(container, ctx, instance);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
new file mode 100644
index 0000000..b64fd73
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.slider.server.appmaster.SliderAppMaster;
+
+import java.io.IOException;
+
+/**
+ * Action to tell a queue executor to stop -after handing this on/executing it
+ */
+public class ActionStopQueue extends AsyncAction {
+
+ public ActionStopQueue(int delay) {
+ super("stop queue", delay);
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster) throws IOException {
+ // no-op
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
new file mode 100644
index 0000000..cffa3f2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.slider.core.main.LauncherExitCodes;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+
+import java.io.IOException;
+
+public class ActionStopSlider extends AsyncAction {
+ public ActionStopSlider(String message,
+ int delay) {
+ super(message, delay, ActionAttributes.HALTS_CLUSTER);
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster) throws IOException {
+ String message = name;
+ SliderAppMaster.getLog().info("SliderAppMasterApi.stopCluster: {}",
+ message);
+ appMaster.signalAMComplete(
+ LauncherExitCodes.EXIT_CLIENT_INITIATED_SHUTDOWN,
+ message);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
new file mode 100644
index 0000000..09bb517
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.concurrent.Delayed;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+public abstract class AsyncAction implements Delayed {
+
+ private static final AtomicLong sequencer = new AtomicLong(0);
+
+ public final String name;
+ public final long nanos;
+ private final EnumSet<ActionAttributes> attrs;
+ private final long sequenceNumber = sequencer.incrementAndGet();
+
+
+ protected AsyncAction(String name) {
+ this(name, 0);
+ }
+
+ protected AsyncAction(String name,
+ int delayMillis, EnumSet<ActionAttributes> attrs) {
+ this.name = name;
+ this.nanos = convertAndOffset(delayMillis);
+ this.attrs = attrs;
+ }
+
+ protected AsyncAction(String name,
+ int delayMillis,
+ ActionAttributes... attributes) {
+ this(name, delayMillis);
+ Collections.addAll(attrs, attributes);
+ }
+
+ protected AsyncAction(String name,
+ int delayMillis) {
+ this.name = name;
+ this.nanos = convertAndOffset(delayMillis);
+ attrs = EnumSet.noneOf(ActionAttributes.class);
+ }
+
+ private long convertAndOffset(int delay) {
+ return now() + TimeUnit.NANOSECONDS.convert(delay, TimeUnit.MILLISECONDS);
+ }
+
+ protected long now() {
+ return System.nanoTime();
+ }
+
+ @Override
+ public long getDelay(TimeUnit unit) {
+ return unit.convert(nanos - now(), TimeUnit.NANOSECONDS);
+ }
+
+ @Override
+ public int compareTo(Delayed that) {
+ if (this == that) {
+ return 0;
+ }
+ return SliderUtils.compareTo(
+ getDelay(TimeUnit.NANOSECONDS),
+ that.getDelay(TimeUnit.NANOSECONDS));
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder(super.toString());
+ sb.append(" name='").append(name).append('\'');
+ sb.append(", nanos=").append(nanos);
+ sb.append(", attrs=").append(attrs);
+ sb.append(", sequenceNumber=").append(sequenceNumber);
+ sb.append('}');
+ return sb.toString();
+ }
+
+ /**
+ * Ask if an action has a specific attribute
+ * @param attr attribute
+ * @return true iff the action has the specific attribute
+ */
+ public boolean hasAttr(ActionAttributes attr) {
+ return attrs.contains(attr);
+ }
+
+ /**
+ * Actual application
+ * @param appMaster
+ * @throws IOException
+ */
+ public abstract void execute(SliderAppMaster appMaster) throws IOException;
+
+ public enum ActionAttributes {
+ SHRINKS_CLUSTER,
+ EXPANDS_CLUSTER,
+ HALTS_CLUSTER,
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
new file mode 100644
index 0000000..7eeaddf
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ * Executor for async actions - hands them off to the AM as
+ * appropriate
+ */
+public class QueueExecutor implements Runnable {
+ private static final Logger log =
+ LoggerFactory.getLogger(QueueExecutor.class);
+
+ private final SliderAppMaster appMaster;
+ private final QueueService actionQueues;
+
+ public QueueExecutor(SliderAppMaster appMaster,
+ QueueService actionQueues) {
+ Preconditions.checkNotNull(appMaster);
+ Preconditions.checkNotNull(actionQueues);
+
+ this.appMaster = appMaster;
+ this.actionQueues = actionQueues;
+ }
+
+ @VisibleForTesting
+ QueueExecutor(QueueService actionQueues) {
+ Preconditions.checkNotNull(actionQueues);
+ this.appMaster = null;
+ this.actionQueues = actionQueues;
+ }
+
+ /**
+ * Run until the queue has been told to stop
+ */
+ @Override
+ public void run() {
+ AsyncAction take = null;
+ try {
+ log.info("Queue Executor run() started");
+ do {
+ take = actionQueues.actionQueue.take();
+ log.debug("Executing {}", take);
+ take.execute(appMaster);
+ } while (!(take instanceof ActionStopQueue));
+ log.info("Queue Executor run() stopped");
+ } catch (InterruptedException | IOException e) {
+ log.error("Exception processing {}: {}", take, e, e);
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
new file mode 100644
index 0000000..dd39de0
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+
+import org.apache.hadoop.service.AbstractService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.DelayQueue;
+import java.util.concurrent.LinkedBlockingDeque;
+
+public class QueueService extends AbstractService
+ implements Runnable {
+ private static final Logger log =
+ LoggerFactory.getLogger(QueueService.class);
+
+ public final DelayQueue<AsyncAction> delayedActions = new DelayQueue<>();
+
+ public final BlockingDeque<AsyncAction> actionQueue =
+ new LinkedBlockingDeque<>();
+
+ public QueueService() {
+ super("action queue");
+ }
+
+ public void putImmediately(AsyncAction action) {
+ actionQueue.add(action);
+ }
+
+ public void putDelayed(AsyncAction action) {
+ delayedActions.add(action);
+ }
+
+ /**
+ * Run until the queue has been told to stop
+ */
+ @Override
+ public void run() {
+ try {
+
+ log.info("ActionExecutor Queue terminated");
+
+ AsyncAction take;
+ do {
+ take = delayedActions.take();
+ actionQueue.push(take);
+ } while (!(take instanceof ActionStopQueue));
+ } catch (InterruptedException e) {
+ //game over
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/StartContainerAction.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/StartContainerAction.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/StartContainerAction.java
new file mode 100644
index 0000000..95ecbe8
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/StartContainerAction.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+
+/**
+ * Start a container
+ */
+public class StartContainerAction {
+
+ private final Container container;
+ private final ContainerLaunchContext ctx;
+ private final RoleInstance instance;
+
+ public StartContainerAction(Container container,
+ ContainerLaunchContext ctx,
+ RoleInstance instance) {
+ this.container = container;
+ this.ctx = ctx;
+ this.instance = instance;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/services/workflow/AbstractWorkflowExecutorService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/AbstractWorkflowExecutorService.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/AbstractWorkflowExecutorService.java
deleted file mode 100644
index c26e3c4..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/AbstractWorkflowExecutorService.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import org.apache.hadoop.service.AbstractService;
-
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-
-/**
- * A service that hosts an executor -when the service is stopped,
- * {@link ExecutorService#shutdownNow()} is invoked.
- */
-public abstract class AbstractWorkflowExecutorService extends AbstractService {
-
- private ExecutorService executor;
-
- /**
- * Construct an instance with the given name -but
- * no executor
- * @param name service name
- */
- public AbstractWorkflowExecutorService(String name) {
- this(name, null);
- }
-
- /**
- * Construct an instance with the given name and executor
- * @param name service name
- * @param executor exectuor
- */
- protected AbstractWorkflowExecutorService(String name,
- ExecutorService executor) {
- super(name);
- this.executor = executor;
- }
-
- /**
- * Get the executor
- * @return the executor
- */
- public synchronized ExecutorService getExecutor() {
- return executor;
- }
-
- /**
- * Set the executor. This is protected as it
- * is intended to be restricted to subclasses
- * @param executor executor
- */
- protected synchronized void setExecutor(ExecutorService executor) {
- this.executor = executor;
- }
-
- /**
- * Execute the runnable with the executor (which
- * must have been created already)
- * @param runnable runnable to execute
- */
- public void execute(Runnable runnable) {
- getExecutor().execute(runnable);
- }
-
- /**
- * Submit a callable
- * @param callable callable
- * @param <V> type of the final get
- * @return a future to wait on
- */
- public <V> Future<V> submit(Callable<V> callable) {
- return getExecutor().submit(callable);
- }
- /**
- * Stop the service: halt the executor.
- * @throws Exception exception.
- */
- @Override
- protected void serviceStop() throws Exception {
- super.serviceStop();
- stopExecutor();
- }
-
- /**
- * Stop the executor if it is not null.
- * This uses {@link ExecutorService#shutdownNow()}
- * and so does not block until they have completed.
- */
- protected synchronized void stopExecutor() {
- if (executor != null) {
- executor.shutdownNow();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
index ccce6cb..46c724c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ForkedProcessService.java
@@ -27,6 +27,7 @@ import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -65,8 +66,9 @@ import java.util.concurrent.atomic.AtomicInteger;
* it calls its {@link #stop()} method. If the error code was non-zero,
* the service is logged as having failed.
*/
-public class ForkedProcessService extends AbstractWorkflowExecutorService implements
- LongLivedProcessLifecycleEvent, Runnable {
+public class ForkedProcessService
+ extends WorkflowExecutorService<ExecutorService>
+ implements LongLivedProcessLifecycleEvent, Runnable {
/**
* Log for the forked master process
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java
index 6c50798..fae3a38 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java
@@ -36,7 +36,7 @@ import java.util.concurrent.TimeUnit;
* started in this service's <code>start()</code> operation.
*/
public class WorkflowCallbackService<V> extends
- AbstractWorkflowExecutorService {
+ WorkflowScheduledExecutorService<ScheduledExecutorService> {
protected static final Logger LOG =
LoggerFactory.getLogger(WorkflowCallbackService.class);
private final Callable<V> callback;
@@ -48,7 +48,6 @@ public class WorkflowCallbackService<V> extends
private Callable<V> callable;
private ScheduledFuture<V> scheduledFuture;
-
/**
* Create an instance of the service
* @param name service name
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowExecutorService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowExecutorService.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowExecutorService.java
new file mode 100644
index 0000000..85d5330
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowExecutorService.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.workflow;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.service.AbstractService;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+
+/**
+ * A service that hosts an executor -when the service is stopped,
+ * {@link ExecutorService#shutdownNow()} is invoked.
+ */
+public class WorkflowExecutorService<E extends ExecutorService> extends AbstractService {
+
+ private E executor;
+
+ /**
+ * Construct an instance with the given name -but
+ * no executor
+ * @param name service name
+ */
+ public WorkflowExecutorService(String name) {
+ this(name, null);
+ }
+
+ /**
+ * Construct an instance with the given name and executor
+ * @param name service name
+ * @param executor exectuor
+ */
+ public WorkflowExecutorService(String name,
+ E executor) {
+ super(name);
+ this.executor = executor;
+ }
+
+ /**
+ * Get the executor
+ * @return the executor
+ */
+ public synchronized E getExecutor() {
+ return executor;
+ }
+
+ /**
+ * Set the executor. Only valid if the current one is null
+ * @param executor executor
+ */
+ public synchronized void setExecutor(E executor) {
+ Preconditions.checkState(this.executor == null,
+ "Executor already set");
+ this.executor = executor;
+ }
+
+ /**
+ * Execute the runnable with the executor (which
+ * must have been created already)
+ * @param runnable runnable to execute
+ */
+ public void execute(Runnable runnable) {
+ getExecutor().execute(runnable);
+ }
+
+ /**
+ * Submit a callable
+ * @param callable callable
+ * @param <V> type of the final get
+ * @return a future to wait on
+ */
+ public <V> Future<V> submit(Callable<V> callable) {
+ return getExecutor().submit(callable);
+ }
+
+ /**
+ * Stop the service: halt the executor.
+ * @throws Exception exception.
+ */
+ @Override
+ protected void serviceStop() throws Exception {
+ super.serviceStop();
+ stopExecutor();
+ }
+
+ /**
+ * Stop the executor if it is not null.
+ * This uses {@link ExecutorService#shutdownNow()}
+ * and so does not block until they have completed.
+ */
+ protected synchronized void stopExecutor() {
+ if (executor != null) {
+ executor.shutdownNow();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowScheduledExecutorService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowScheduledExecutorService.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowScheduledExecutorService.java
new file mode 100644
index 0000000..e9f53ed
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowScheduledExecutorService.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.workflow;
+
+import java.util.concurrent.ScheduledExecutorService;
+
+/**
+ * Scheduled executor or subclass thereof
+ * @param <E> scheduled executor service type
+ */
+public class WorkflowScheduledExecutorService<E extends ScheduledExecutorService>
+ extends WorkflowExecutorService<E> {
+
+ public WorkflowScheduledExecutorService(String name) {
+ super(name);
+ }
+
+ public WorkflowScheduledExecutorService(String name,
+ E executor) {
+ super(name, executor);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/main/java/org/apache/slider/server/services/workflow/package-info.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/package-info.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/package-info.java
index fab1b9f..36d059a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/package-info.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/package-info.java
@@ -154,7 +154,7 @@ and simply in applications which are built from composite YARN services.
then signals its owning service to stop once that runnable is completed.
Any exception raised in the run is stored.
</li>
- <li>{@link org.apache.slider.server.services.workflow.AbstractWorkflowExecutorService}:
+ <li>{@link org.apache.slider.server.services.workflow.WorkflowExecutorService}:
A base class for services that wish to have a {@link java.util.concurrent.ExecutorService}
with a lifespan mapped to that of a service. When the service is stopped, the
{@link java.util.concurrent.ExecutorService#shutdownNow()} method is called to
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestDestroyStandaloneAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestDestroyStandaloneAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestDestroyStandaloneAM.groovy
deleted file mode 100644
index ed0604b..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestDestroyStandaloneAM.groovy
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.agent.standalone
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.slider.agent.AgentMiniClusterTestBase
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.params.ActionEchoArgs
-import org.apache.slider.common.params.Arguments
-import org.apache.slider.common.params.SliderActions
-import org.apache.slider.common.tools.SliderFileSystem
-import org.apache.slider.core.exceptions.ErrorStrings
-import org.apache.slider.core.exceptions.SliderException
-import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
-import org.apache.slider.core.main.ServiceLauncher
-import org.junit.Test
-
-/**
- * destroy a masterless AM
- */
-@CompileStatic
-@Slf4j
-
-class TestDestroyStandaloneAM extends AgentMiniClusterTestBase {
-
- @Test
- public void testDestroyStandaloneAM() throws Throwable {
- String clustername = createMiniCluster("", configuration, 1, false)
-
- describe "create a Standalone AM, stop it, try to create" +
- "a second cluster with the same name, destroy it, try a third time"
-
- ServiceLauncher<SliderClient> launcher1 = launchClientAgainstMiniMR(
- configuration,
- [
- SliderActions.ACTION_DESTROY,
- "no-cluster-of-this-name",
- Arguments.ARG_FILESYSTEM, fsDefaultName,
- ])
- assert launcher1.serviceExitCode == 0
-
-
-
- ServiceLauncher<SliderClient> launcher = createStandaloneAM(
- clustername,
- true,
- true)
- SliderClient sliderClient = launcher.service
- addToTeardown(sliderClient);
-
- SliderFileSystem sliderFileSystem = createSliderFileSystem()
- def fs = sliderFileSystem.fileSystem
- def instanceDir = sliderFileSystem.buildClusterDirPath(clustername)
-
- assertPathExists(
- fs,
- "cluster path not found",
- instanceDir)
-
- sliderFileSystem.locateInstanceDefinition(clustername)
- clusterActionFreeze(sliderClient, clustername,"stopping first cluster")
- waitForAppToFinish(sliderClient)
-
-
- describe "Warnings below are expected"
-
- //now try to create instance #2, and expect an in-use failure
- try {
- createStandaloneAM(clustername, false, false)
- fail("expected a failure, got an AM")
- } catch (SliderException e) {
- assertExceptionDetails(e,
- SliderExitCodes.EXIT_INSTANCE_EXISTS,
- ErrorStrings.E_ALREADY_EXISTS)
- }
-
- describe "END EXPECTED WARNINGS"
-
-
- describe "destroying $clustername"
- //now: destroy it
-
- int exitCode = sliderClient.actionDestroy(clustername);
- assert 0 == exitCode
-
- describe "post destroy checks"
- sliderFileSystem.verifyDirectoryNonexistent(instanceDir)
-
- describe "thaw expected to fail"
- //expect thaw to now fail
- def ex = launchExpectingException(SliderClient,
- configuration,
- "",
- [
- SliderActions.ACTION_THAW,
- clustername,
- Arguments.ARG_FILESYSTEM, fsDefaultName,
- Arguments.ARG_MANAGER, RMAddr,
- ])
- assert ex instanceof UnknownApplicationInstanceException
-
- describe "thaw completed, checking dir is still absent"
- sliderFileSystem.verifyDirectoryNonexistent(instanceDir)
-
-
- describe "recreating $clustername"
-
- //and create a new cluster
- launcher = createStandaloneAM(clustername, false, true)
- SliderClient cluster2 = launcher.service
-
- // do an echo here of a large string
- // Hadoop RPC couldn't handle strings > 32K chars, this
- // check here allows us to be confident that large JSON Reports are handled
- StringBuilder sb = new StringBuilder()
- for (int i = 0; i < 65536; i++) {
- sb.append(Integer.toString(i, 16))
- }
- ActionEchoArgs args = new ActionEchoArgs();
- args.message = sb.toString();
- def echoed = cluster2.actionEcho(clustername, args)
- assert echoed == args.message
- log.info(
- "Successful echo of a text document ${echoed.size()} characters long")
- //try to destroy it while live
- try {
- int ec = cluster2.actionDestroy(clustername)
- fail("expected a failure from the destroy, got error code $ec")
- } catch (SliderException e) {
- assertFailureClusterInUse(e);
- }
-
- //and try to destroy a completely different cluster just for the fun of it
- assert 0 == sliderClient.actionDestroy("no-cluster-of-this-name")
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestKillStandaloneAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestKillStandaloneAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestKillStandaloneAM.groovy
deleted file mode 100644
index fef98a7..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestKillStandaloneAM.groovy
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.agent.standalone
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.ApplicationReport
-import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.agent.AgentMiniClusterTestBase
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderXmlConfKeys
-import org.apache.slider.common.params.Arguments
-import org.apache.slider.core.main.ServiceLauncher
-import org.junit.Test
-
-/**
- * kill a masterless AM and verify it shuts down. This test
- * also sets the retry count to 1 to stop recreation attempts
- */
-@CompileStatic
-@Slf4j
-
-class TestKillStandaloneAM extends AgentMiniClusterTestBase {
-
-
- @Test
- public void testKillStandaloneAM() throws Throwable {
- String clustername = createMiniCluster("", configuration, 1, true)
-
- describe "kill a Standalone AM and verify that it shuts down"
- ServiceLauncher<SliderClient> launcher =
- createStandaloneAMWithArgs(clustername,
- [
- Arguments.ARG_OPTION, SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, "1"
- ],
- true,
- false)
- SliderClient sliderClient = launcher.service
- addToTeardown(sliderClient);
- ApplicationReport report = waitForClusterLive(sliderClient)
-
- describe("listing Java processes")
- lsJavaProcesses();
- describe("killing AM")
- killAM(SIGTERM);
- waitWhileClusterLive(sliderClient);
- //give yarn some time to notice
- sleep(10000)
- describe("final listing")
- lsJavaProcesses();
- report = sliderClient.applicationReport
- assert YarnApplicationState.FAILED == report.yarnApplicationState;
- clusterActionFreeze(sliderClient, clustername)
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestRestartStandaloneAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestRestartStandaloneAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestRestartStandaloneAM.groovy
deleted file mode 100644
index 62dd1f9..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestRestartStandaloneAM.groovy
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.agent.standalone
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.ApplicationReport
-import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.slider.agent.AgentMiniClusterTestBase
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderXmlConfKeys
-import org.apache.slider.common.params.ActionAMSuicideArgs
-import org.apache.slider.common.params.Arguments
-import org.apache.slider.core.main.ServiceLauncher
-import org.junit.Test
-
-/**
- * kill a masterless AM and verify it shuts down. This test
- * also sets the retry count to 1 to stop recreation attempts
- */
-@CompileStatic
-@Slf4j
-
-class TestRestartStandaloneAM extends AgentMiniClusterTestBase {
-
-
- @Test
- public void testRestartStandaloneAM() throws Throwable {
- describe "kill a Standalone AM and verify that it restarts"
- // patch the configuration for AM restart
- YarnConfiguration conf = getRestartableConfiguration(5)
-
- String clustername = createMiniCluster("", conf, 1, true)
- ServiceLauncher<SliderClient> launcher =
- createStandaloneAMWithArgs(clustername,
- [
- Arguments.ARG_OPTION, SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, "4"
- ],
- true,
- false)
- SliderClient sliderClient = launcher.service
- addToTeardown(sliderClient);
-
- ApplicationReport report = waitForClusterLive(sliderClient)
- logReport(report)
- waitUntilClusterLive(sliderClient, 30000)
-
-
- ActionAMSuicideArgs args = new ActionAMSuicideArgs()
- args.message = "test AM iteration"
- args.waittime = 100
- args.exitcode = 1
- sliderClient.actionAmSuicide(clustername, args)
- waitWhileClusterLive(sliderClient);
- //give yarn some time to notice
- sleep(20000)
- waitUntilClusterLive(sliderClient, 40000)
-
-
- // app should be running here
- assert 0 == sliderClient.actionExists(clustername, true)
-
-
- clusterActionFreeze(sliderClient, clustername)
- }
-
- /**
- * Get a restartable configuration
- * @param restarts
- * @return
- */
- public YarnConfiguration getRestartableConfiguration(int restarts) {
- def conf = new YarnConfiguration(configuration)
- conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, restarts)
- conf.setInt(SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, restarts)
- conf
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
new file mode 100644
index 0000000..17e4ff2
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.params.ActionEchoArgs
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.common.tools.SliderFileSystem
+import org.apache.slider.core.exceptions.ErrorStrings
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * destroy a masterless AM
+ */
+@CompileStatic
+@Slf4j
+
+class TestStandaloneAMDestroy extends AgentMiniClusterTestBase {
+
+ @Test
+ public void testDestroyStandaloneAM() throws Throwable {
+ String clustername = createMiniCluster("", configuration, 1, false)
+
+ describe "create a Standalone AM, stop it, try to create" +
+ "a second cluster with the same name, destroy it, try a third time"
+
+ ServiceLauncher<SliderClient> launcher1 = launchClientAgainstMiniMR(
+ configuration,
+ [
+ SliderActions.ACTION_DESTROY,
+ "no-cluster-of-this-name",
+ Arguments.ARG_FILESYSTEM, fsDefaultName,
+ ])
+ assert launcher1.serviceExitCode == 0
+
+ ServiceLauncher<SliderClient> launcher = createStandaloneAM(
+ clustername,
+ true,
+ true)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+ SliderFileSystem sliderFileSystem = createSliderFileSystem()
+ def fs = sliderFileSystem.fileSystem
+ def instanceDir = sliderFileSystem.buildClusterDirPath(clustername)
+
+ assertPathExists(
+ fs,
+ "cluster path not found",
+ instanceDir)
+
+ sliderFileSystem.locateInstanceDefinition(clustername)
+ clusterActionFreeze(sliderClient, clustername,"stopping first cluster")
+ waitForAppToFinish(sliderClient)
+
+
+ describe "Warnings below are expected"
+
+ //now try to create instance #2, and expect an in-use failure
+ try {
+ createStandaloneAM(clustername, false, false)
+ fail("expected a failure, got an AM")
+ } catch (SliderException e) {
+ assertExceptionDetails(e,
+ SliderExitCodes.EXIT_INSTANCE_EXISTS,
+ ErrorStrings.E_ALREADY_EXISTS)
+ }
+
+ describe "END EXPECTED WARNINGS"
+
+
+ describe "destroying $clustername"
+ //now: destroy it
+
+ int exitCode = sliderClient.actionDestroy(clustername);
+ assert 0 == exitCode
+
+ describe "post destroy checks"
+ sliderFileSystem.verifyDirectoryNonexistent(instanceDir)
+
+ describe "thaw expected to fail"
+ //expect thaw to now fail
+ def ex = launchExpectingException(SliderClient,
+ configuration,
+ "",
+ [
+ SliderActions.ACTION_THAW,
+ clustername,
+ Arguments.ARG_FILESYSTEM, fsDefaultName,
+ Arguments.ARG_MANAGER, RMAddr,
+ ])
+ assert ex instanceof UnknownApplicationInstanceException
+
+ describe "thaw completed, checking dir is still absent"
+ sliderFileSystem.verifyDirectoryNonexistent(instanceDir)
+
+
+ describe "recreating $clustername"
+
+ //and create a new cluster
+ launcher = createStandaloneAM(clustername, false, true)
+ SliderClient cluster2 = launcher.service
+
+ // do an echo here of a large string
+ // Hadoop RPC couldn't handle strings > 32K chars, this
+ // check here allows us to be confident that large JSON Reports are handled
+ StringBuilder sb = new StringBuilder()
+ for (int i = 0; i < 65536; i++) {
+ sb.append(Integer.toString(i, 16))
+ }
+ ActionEchoArgs args = new ActionEchoArgs();
+ args.message = sb.toString();
+ def echoed = cluster2.actionEcho(clustername, args)
+ assert echoed == args.message
+ log.info(
+ "Successful echo of a text document ${echoed.size()} characters long")
+ //try to destroy it while live
+ try {
+ int ec = cluster2.actionDestroy(clustername)
+ fail("expected a failure from the destroy, got error code $ec")
+ } catch (SliderException e) {
+ assertFailureClusterInUse(e);
+ }
+
+ //and try to destroy a completely different cluster just for the fun of it
+ assert 0 == sliderClient.actionDestroy("no-cluster-of-this-name")
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
new file mode 100644
index 0000000..75f9a2c
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMKill.groovy
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * kill a masterless AM and verify it shuts down. This test
+ * also sets the retry count to 1 to stop recreation attempts
+ */
+@CompileStatic
+@Slf4j
+
+class TestStandaloneAMKill extends AgentMiniClusterTestBase {
+
+
+ @Test
+ public void testKillStandaloneAM() throws Throwable {
+ String clustername = createMiniCluster("", configuration, 1, true)
+
+ describe "kill a Standalone AM and verify that it shuts down"
+ ServiceLauncher<SliderClient> launcher =
+ createStandaloneAMWithArgs(clustername,
+ [
+ Arguments.ARG_OPTION, SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, "1"
+ ],
+ true,
+ false)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+ ApplicationReport report = waitForClusterLive(sliderClient)
+
+ describe("listing Java processes")
+ lsJavaProcesses();
+ describe("killing AM")
+ killAM(SIGTERM);
+ waitWhileClusterLive(sliderClient);
+ //give yarn some time to notice
+ sleep(10000)
+ describe("final listing")
+ lsJavaProcesses();
+ report = sliderClient.applicationReport
+ assert YarnApplicationState.FAILED == report.yarnApplicationState;
+ clusterActionFreeze(sliderClient, clustername)
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMRestart.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMRestart.groovy
new file mode 100644
index 0000000..8d9318a
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMRestart.groovy
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.ActionAMSuicideArgs
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * kill a masterless AM and verify it shuts down. This test
+ * also sets the retry count to 1 to stop recreation attempts
+ */
+@CompileStatic
+@Slf4j
+
+class TestStandaloneAMRestart extends AgentMiniClusterTestBase {
+
+
+ @Test
+ public void testRestartStandaloneAM() throws Throwable {
+ describe "kill a Standalone AM and verify that it restarts"
+ // patch the configuration for AM restart
+ YarnConfiguration conf = getRestartableConfiguration(5)
+
+ String clustername = createMiniCluster("", conf, 1, true)
+ ServiceLauncher<SliderClient> launcher =
+ createStandaloneAMWithArgs(clustername,
+ [
+ Arguments.ARG_OPTION, SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, "4"
+ ],
+ true,
+ false)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+ ApplicationReport report = waitForClusterLive(sliderClient)
+ logReport(report)
+ waitUntilClusterLive(sliderClient, 30000)
+
+
+ ActionAMSuicideArgs args = new ActionAMSuicideArgs()
+ args.message = "test AM iteration"
+ args.waittime = 100
+ args.exitcode = 1
+ sliderClient.actionAmSuicide(clustername, args)
+ waitWhileClusterLive(sliderClient);
+ //give yarn some time to notice
+ sleep(20000)
+ waitUntilClusterLive(sliderClient, 40000)
+
+
+ // app should be running here
+ assert 0 == sliderClient.actionExists(clustername, true)
+
+
+ clusterActionFreeze(sliderClient, clustername)
+ }
+
+ /**
+ * Get a restartable configuration
+ * @param restarts
+ * @return
+ */
+ public YarnConfiguration getRestartableConfiguration(int restarts) {
+ def conf = new YarnConfiguration(configuration)
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, restarts)
+ conf.setInt(SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, restarts)
+ conf
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
new file mode 100644
index 0000000..24d4c40
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.service.ServiceOperations
+import org.apache.slider.server.appmaster.SliderAppMaster
+import org.apache.slider.server.services.workflow.ServiceThreadFactory
+import org.apache.slider.server.services.workflow.WorkflowExecutorService
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicBoolean
+import java.util.concurrent.atomic.AtomicLong
+
+@Slf4j
+//@CompileStatic
+class TestActions {
+
+ QueueService queues;
+ WorkflowExecutorService<ExecutorService> executorService;
+
+
+ @Before
+ void createService() {
+ queues = new QueueService();
+
+ def conf = new Configuration()
+ queues.init(conf)
+
+ queues.start();
+
+ executorService = new WorkflowExecutorService<>("AmExecutor",
+ Executors.newCachedThreadPool(
+ new ServiceThreadFactory("AmExecutor", true)));
+
+ executorService.init(conf)
+ executorService.start();
+ }
+
+ @After
+ void destroyService() {
+ ServiceOperations.stop(executorService);
+ ServiceOperations.stop(queues);
+ }
+
+ @Test
+ public void testBasicService() throws Throwable {
+ queues.start();
+ }
+
+ @Test
+ public void testDelayLogic() throws Throwable {
+ ActionNoteExecuted action = new ActionNoteExecuted("", 1000)
+ long now = System.currentTimeMillis();
+
+ def delay = action.getDelay(TimeUnit.MILLISECONDS)
+ assert delay >= 800
+ assert delay <= 1800
+
+ ActionNoteExecuted a2 = new ActionNoteExecuted("a2", 10000)
+ assert action.compareTo(a2) < 0
+ assert a2.compareTo(action) > 0
+ assert action.compareTo(action)== 0
+
+ }
+
+ @Test
+ public void testActionDelayedExecutorTermination() throws Throwable {
+ long start = System.currentTimeMillis()
+
+ ActionStopQueue stopAction = new ActionStopQueue(1000);
+ queues.delayedActions.add(stopAction);
+ queues.run();
+ AsyncAction take = queues.actionQueue.take();
+ assert take == stopAction
+ long stop = System.currentTimeMillis();
+ assert stop - start > 500
+ assert stop - start < 1500
+ }
+
+ @Test
+ public void testImmediateQueue() throws Throwable {
+ ActionNoteExecuted noteExecuted = new ActionNoteExecuted("executed", 0)
+ queues.putImmediately(noteExecuted)
+ queues.putImmediately(new ActionStopQueue(0))
+ QueueExecutor ex = new QueueExecutor(queues)
+ ex.run();
+ assert queues.actionQueue.empty
+ assert noteExecuted.executed.get()
+ }
+
+ @Test
+ public void testActionOrdering() throws Throwable {
+
+ ActionNoteExecuted note1 = new ActionNoteExecuted("note1", 500)
+ def stop = new ActionStopQueue(1500)
+ ActionNoteExecuted note2 = new ActionNoteExecuted("note2", 800)
+
+ List<AsyncAction> actions = [note1, stop, note2]
+ Collections.sort(actions)
+ assert actions.get(0) == note1
+ assert actions.get(1) == note2
+ assert actions.get(3) == stop
+ }
+
+ @Test
+ public void testDelayedQueueWithReschedule() throws Throwable {
+
+ ActionNoteExecuted note1 = new ActionNoteExecuted("note1", 500)
+ def stop = new ActionStopQueue(1500)
+ ActionNoteExecuted note2 = new ActionNoteExecuted("note2", 800)
+
+ assert note2.compareTo(stop) < 0
+ assert note1.nanos < note2.nanos
+ assert note2.nanos < stop.nanos
+ queues.putDelayed(note1)
+ queues.putDelayed(stop)
+ queues.putDelayed(note2)
+ // async to sync expected to run in order
+ queues.run();
+ assert queues.delayedActions.empty
+ assert !queues.actionQueue.empty
+ QueueExecutor ex = new QueueExecutor(queues)
+ ex.run();
+ assert queues.actionQueue.empty
+ assert note1.executed.get()
+ assert note2.executed.get()
+ }
+
+ public class ActionNoteExecuted extends AsyncAction {
+ public final AtomicBoolean executed = new AtomicBoolean(false);
+ public final AtomicLong executionTimeNanos = new AtomicLong()
+ public final AtomicLong executionCount = new AtomicLong()
+
+ public ActionNoteExecuted(String text, int delay) {
+ super(text, delay);
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster) throws IOException {
+ log.info("Executing $name");
+ executed.set(true);
+ executionTimeNanos.set(System.nanoTime())
+ executionCount.incrementAndGet()
+ synchronized (this) {
+ this.notify();
+ }
+ }
+
+ @Override
+ String toString() {
+ return super.toString() +
+ " executed=${executed.get()}; count=${executionCount.get()};"
+ }
+
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
index 0f4a4ca..4242ba1 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
@@ -31,7 +31,7 @@ import org.apache.slider.server.avro.RoleHistoryWriter
import org.junit.Test
@Slf4j
-@CompileStatic
+//@CompileStatic
class TestRoleHistoryRW extends BaseMockAppStateTest {
static long time = System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7ef046b6/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowExecutorService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowExecutorService.java b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowExecutorService.java
index 9514f47..dc160d9 100644
--- a/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowExecutorService.java
+++ b/slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowExecutorService.java
@@ -20,9 +20,13 @@ package org.apache.slider.server.services.workflow;
import org.junit.Test;
+import java.util.concurrent.ExecutorService;
-public class TestWorkflowExecutorService extends WorkflowServiceTestBase {
+/**
+ * Basic tests for executor service
+ */
+public class TestWorkflowExecutorService extends WorkflowServiceTestBase {
@Test
public void testAsyncRun() throws Throwable {
@@ -51,7 +55,8 @@ public class TestWorkflowExecutorService extends WorkflowServiceTestBase {
assertNotNull(runnable.getException());
}
- private static class ExecutorSvc extends AbstractWorkflowExecutorService {
+ private static class ExecutorSvc
+ extends WorkflowExecutorService<ExecutorService> {
private ExecutorSvc() {
super("ExecutorService",
ServiceThreadFactory.singleThreadExecutor("test", true));
[47/50] [abbrv] git commit: SLIDER-240 Add log after verification
passes
Posted by st...@apache.org.
SLIDER-240 Add log after verification passes
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e464bbad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e464bbad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e464bbad
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: e464bbadf0c9a395c3d0c1a610838f6e6269f94e
Parents: 05bf45d
Author: tedyu <yu...@gmail.com>
Authored: Fri Aug 8 11:41:51 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Fri Aug 8 11:41:51 2014 -0700
----------------------------------------------------------------------
.../main/java/org/apache/slider/common/tools/CoreFileSystem.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e464bbad/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index 93de4d9..9718be9 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -335,6 +335,7 @@ public class CoreFileSystem {
}
dst.delete();
if (!found) throw new FileNotFoundException("file: " + file + " not found in " + path);
+ log.info("Verification of " + path + " passed");
}
/**
* Create the application-instance specific temporary directory
[06/50] [abbrv] git commit: SLIDER-270 improving diags and failure
log generation in AppState release operations
Posted by st...@apache.org.
SLIDER-270 improving diags and failure log generation in AppState release operations
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/268253c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/268253c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/268253c1
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 268253c1bb1582d4d1879c4798064e59bb447da0
Parents: ae4d073
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 5 21:07:23 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 5 21:08:05 2014 +0100
----------------------------------------------------------------------
.../slider/server/appmaster/state/AppState.java | 69 +++++++++++++-------
.../server/appmaster/state/RoleStatus.java | 4 +-
2 files changed, 46 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/268253c1/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 9f8a39d..74e91a0 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -696,7 +696,7 @@ public class AppState {
roleStatusMap.get(priority));
}
roleStatusMap.put(priority,
- new RoleStatus(providerRole));
+ new RoleStatus(providerRole));
roles.put(providerRole.name, providerRole);
}
@@ -1175,10 +1175,14 @@ public class AppState {
log.info("Container was queued for release");
Container container = containersBeingReleased.remove(containerId);
RoleStatus roleStatus = lookupRoleStatus(container);
- log.info("decrementing role count for role {}", roleStatus.getName());
- roleStatus.decReleasing();
- roleStatus.decActual();
- roleStatus.incCompleted();
+ int releasing = roleStatus.decReleasing();
+ int actual = roleStatus.decActual();
+ int completedCount = roleStatus.incCompleted();
+ log.info("decrementing role count for role {} to {}; releasing={}, completed={}",
+ roleStatus.getName(),
+ actual,
+ releasing,
+ completedCount);
roleHistory.onReleaseCompleted(container);
} else if (surplusNodes.remove(containerId)) {
@@ -1205,25 +1209,17 @@ public class AppState {
roleStatus.decActual();
boolean shortLived = isShortLived(roleInstance);
String message;
- if (roleInstance.container != null) {
- String user = null;
- try {
- user = SliderUtils.getCurrentUser().getShortUserName();
- } catch (IOException ignored) {
- }
- String completedLogsUrl = null;
- Container c = roleInstance.container;
- String url = logServerURL;
- if (user != null && SliderUtils.isSet(url)) {
- completedLogsUrl = url
- + "/" + c.getNodeId() + "/" + roleInstance.getContainerId() + "/ctx/" + user;
- }
- message = String.format("Failure %s on host %s" +
- (completedLogsUrl != null ? ", see %s" : ""), roleInstance.getContainerId(),
- c.getNodeId().getHost(), completedLogsUrl);
+ Container failedContainer = roleInstance.container;
+
+ //build the failure message
+ if (failedContainer != null) {
+ String completedLogsUrl = getLogsURLForContainer(failedContainer);
+ message = String.format("Failure %s on host %s: %s" +
+ roleInstance.getContainerId(),
+ failedContainer.getNodeId().getHost(),
+ completedLogsUrl);
} else {
- message = String.format("Failure %s",
- containerId.toString());
+ message = String.format("Failure %s", containerId);
}
roleStatus.noteFailed(message);
//have a look to see if it short lived
@@ -1231,8 +1227,8 @@ public class AppState {
roleStatus.incStartFailed();
}
- if (roleInstance.container != null) {
- roleHistory.onFailedContainer(roleInstance.container, shortLived);
+ if (failedContainer!= null) {
+ roleHistory.onFailedContainer(failedContainer, shortLived);
}
} catch (YarnRuntimeException e1) {
@@ -1270,6 +1266,29 @@ public class AppState {
return result;
}
+ /**
+ * Get the URL log for a container
+ * @param c container
+ * @return the URL or "" if it cannot be determined
+ */
+ protected String getLogsURLForContainer(Container c) {
+ if (c==null) {
+ return null;
+ }
+ String user = null;
+ try {
+ user = SliderUtils.getCurrentUser().getShortUserName();
+ } catch (IOException ignored) {
+ }
+ String completedLogsUrl = "";
+ String url = logServerURL;
+ if (user != null && SliderUtils.isSet(url)) {
+ completedLogsUrl = url
+ + "/" + c.getNodeId() + "/" + c.getId() + "/ctx/" + user;
+ }
+ return completedLogsUrl;
+ }
+
/**
* Return the percentage done that Slider is to have YARN display in its
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/268253c1/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index 04d8b37..b3dddf6 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -175,8 +175,8 @@ public final class RoleStatus implements Cloneable {
this.completed = completed;
}
- public void incCompleted() {
- completed ++;
+ public int incCompleted() {
+ return completed ++;
}
public int getStarted() {
return started;
[35/50] [abbrv] git commit: SLIDER-240 Detect discrepancy between
file content of {application.def} and {package_list} value
Posted by st...@apache.org.
SLIDER-240 Detect discrepancy between file content of {application.def} and {package_list} value
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/66cbd376
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/66cbd376
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/66cbd376
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 66cbd3761ead0fa39132b2a07833c9e6b2f4642d
Parents: bd69a4c
Author: tedyu <yu...@gmail.com>
Authored: Thu Aug 7 11:41:23 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Thu Aug 7 11:41:23 2014 -0700
----------------------------------------------------------------------
.../slider/common/tools/CoreFileSystem.java | 33 ++++++++++++++++++++
.../providers/agent/AgentClientProvider.java | 6 ++++
2 files changed, 39 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66cbd376/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index cada732..93de4d9 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -48,8 +48,11 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.charset.Charset;
+import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
import static org.apache.slider.common.SliderXmlConfKeys.CLUSTER_DIRECTORY_PERMISSIONS;
import static org.apache.slider.common.SliderXmlConfKeys.DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS;
@@ -304,6 +307,36 @@ public class CoreFileSystem {
}
/**
+ * Verify that a file exists in the zip file given by path
+ * @param path path to zip file
+ * @param file file expected to be in zip
+ * @throws FileNotFoundException file not found or is not a zip file
+ * @throws IOException trouble with FS
+ */
+ public void verifyFileExistsInZip(Path path, String file) throws IOException {
+ fileSystem.copyToLocalFile(path, new Path("/tmp"));
+ File dst = new File((new Path("/tmp", path.getName())).toString());
+ Enumeration<? extends ZipEntry> entries;
+ ZipFile zipFile = new ZipFile(dst);
+ boolean found = false;
+
+ try {
+ entries = zipFile.entries();
+ while (entries.hasMoreElements()) {
+ ZipEntry entry = entries.nextElement();
+ String nm = entry.getName();
+ if (nm.endsWith(file)) {
+ found = true;
+ break;
+ }
+ }
+ } finally {
+ zipFile.close();
+ }
+ dst.delete();
+ if (!found) throw new FileNotFoundException("file: " + file + " not found in " + path);
+ }
+ /**
* Create the application-instance specific temporary directory
* in the DFS
*
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66cbd376/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 3835df6..2ce16c7 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -92,6 +92,12 @@ public class AgentClientProvider extends AbstractClientProvider
getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
Path appDefPath = new Path(appDef);
sliderFileSystem.verifyFileExists(appDefPath);
+
+ String pkgList = instanceDefinition.getAppConfOperations().
+ getGlobalOptions().getOption(AgentKeys.PACKAGE_LIST, null);
+ if (pkgList != null) {
+ sliderFileSystem.verifyFileExistsInZip(appDefPath, pkgList);
+ }
String agentConf = instanceDefinition.getAppConfOperations().
getGlobalOptions().getOption(AgentKeys.AGENT_CONF, "");
[31/50] [abbrv] git commit: SLIDER-286 bug fix in waitUntilClusterLive
Posted by st...@apache.org.
SLIDER-286 bug fix in waitUntilClusterLive
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/56083ecc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/56083ecc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/56083ecc
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 56083eccf49c4caf78f69d52e33c36bbe3754ba4
Parents: 1cbd890
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 7 17:27:13 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 7 17:27:13 2014 +0100
----------------------------------------------------------------------
.../src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/56083ecc/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index a250e55..c5a8b36 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -205,7 +205,7 @@ class SliderTestUtils extends Assert {
public static void waitUntilClusterLive(SliderClient client, int timeout) {
Duration duration = new Duration(timeout);
duration.start()
- while (!client.actionExists(client.deployedClusterName, true) &&
+ while (0 != client.actionExists(client.deployedClusterName, true) &&
!duration.limitExceeded) {
sleep(1000);
}
[15/50] [abbrv] git commit: SLIDER-219. Multiple dynamic ports may be
asked as part of the same config value
Posted by st...@apache.org.
SLIDER-219. Multiple dynamic ports may be asked as part of the same config value
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ad2c798d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ad2c798d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ad2c798d
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: ad2c798d309aded2ef18dea7aa1f90acdd367b14
Parents: 1894a0c
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Jul 30 23:44:22 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Aug 5 18:51:58 2014 -0700
----------------------------------------------------------------------
.../memcached/package/scripts/memcached.py | 2 +-
.../memcached/package/scripts/params.py | 1 +
.../python/agent/CustomServiceOrchestrator.py | 2 +-
.../agent/TestCustomServiceOrchestrator.py | 6 +-
slider-agent/src/test/python/agent/TestMain.py | 4 +-
.../providers/agent/AgentProviderService.java | 70 ++++++++++++++++++--
.../agent/TestAgentProviderService.java | 4 +-
7 files changed, 76 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad2c798d/app-packages/memcached/package/scripts/memcached.py
----------------------------------------------------------------------
diff --git a/app-packages/memcached/package/scripts/memcached.py b/app-packages/memcached/package/scripts/memcached.py
index b2df89a..cf04cf1 100644
--- a/app-packages/memcached/package/scripts/memcached.py
+++ b/app-packages/memcached/package/scripts/memcached.py
@@ -33,7 +33,7 @@ class Memcached(Script):
import params
env.set_params(params)
self.configure(env)
- process_cmd = format("{java64_home}/bin/java -Xmx{xmx_val} -Xms{xms_val} -classpath {app_root}/*:{additional_cp} com.thimbleware.jmemcached.Main --memory={memory_val}")
+ process_cmd = format("{java64_home}/bin/java -Xmx{xmx_val} -Xms{xms_val} -classpath {app_root}/*:{additional_cp} com.thimbleware.jmemcached.Main --memory={memory_val} --port={port}")
Execute(process_cmd,
user=params.app_user,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad2c798d/app-packages/memcached/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/memcached/package/scripts/params.py b/app-packages/memcached/package/scripts/params.py
index a2fba8b..f8e64da 100644
--- a/app-packages/memcached/package/scripts/params.py
+++ b/app-packages/memcached/package/scripts/params.py
@@ -30,3 +30,4 @@ additional_cp = config['configurations']['global']['additional_cp']
xmx_val = config['configurations']['global']['xmx_val']
xms_val = config['configurations']['global']['xms_val']
memory_val = config['configurations']['global']['memory_val']
+port = config['configurations']['global']['listen_port']
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad2c798d/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 7d740c2..f6cdfb2 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -250,7 +250,7 @@ class CustomServiceOrchestrator():
self.config.getLogPath())
if port_allocation_req in value:
value = self.allocate_ports(value, port_allocation_req)
- allocated_ports[k] = value
+ allocated_ports[key + "." + k] = value
command['configurations'][key][k] = value
pass
pass
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad2c798d/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index 3b41bd7..3343a8b 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -226,7 +226,7 @@ class TestCustomServiceOrchestrator(TestCase):
}
ret = orchestrator.runCommand(command, "out.txt", "err.txt")
self.assertEqual(ret['exitcode'], 0)
- self.assertEqual(ret['allocated_ports'], {'a.port': '10233'})
+ self.assertEqual(ret['allocated_ports'], {'a.a.port': '10233'})
self.assertTrue(run_file_mock.called)
self.assertEqual(run_file_mock.call_count, 1)
@@ -480,8 +480,8 @@ class TestCustomServiceOrchestrator(TestCase):
self.assertEqual(command['configurations']['oozie-site']['a_port'], "10023")
self.assertEqual(orchestrator.applied_configs, {})
self.assertEqual(len(allocated_ports), 1)
- self.assertTrue('a_port' in allocated_ports)
- self.assertEqual(allocated_ports['a_port'], '10023')
+ self.assertTrue('oozie-site.a_port' in allocated_ports)
+ self.assertEqual(allocated_ports['oozie-site.a_port'], '10023')
command['configurations']['hbase-site']['work_root'] = "${AGENT_WORK_ROOT}"
command['configurations']['hbase-site']['log_root'] = "${AGENT_LOG_ROOT}/log"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad2c798d/slider-agent/src/test/python/agent/TestMain.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestMain.py b/slider-agent/src/test/python/agent/TestMain.py
index bc8d2a3..4ad7a8f 100644
--- a/slider-agent/src/test/python/agent/TestMain.py
+++ b/slider-agent/src/test/python/agent/TestMain.py
@@ -20,10 +20,10 @@ limitations under the License.
import StringIO
import sys
-from agent import NetUtil, security
+import NetUtil, security
from mock.mock import MagicMock, patch, ANY
import unittest
-from agent import ProcessHelper, main
+import ProcessHelper, main
import logging
import signal
from AgentConfig import AgentConfig
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad2c798d/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index f878b36..b39601a 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -109,7 +109,7 @@ public class AgentProviderService extends AbstractProviderService implements
private static final String CONTAINER_ID = "container_id";
private static final String GLOBAL_CONFIG_TAG = "global";
private static final String LOG_FOLDERS_TAG = "LogFolders";
- private static final String COMPONENT_DATA_TAG = "ComponentData";
+ private static final String COMPONENT_DATA_TAG = "ComponentInstanceData";
private static final int MAX_LOG_ENTRIES = 20;
private static final int DEFAULT_HEARTBEAT_MONITOR_INTERVAL = 60 * 1000;
private final Object syncLock = new Object();
@@ -127,7 +127,7 @@ public class AgentProviderService extends AbstractProviderService implements
return size() > MAX_LOG_ENTRIES;
}
});
- private Map<String, Map<String, String>> componentData = new ConcurrentHashMap();
+ private Map<String, Map<String, String>> componentInstanceData = new ConcurrentHashMap();
private Boolean canAnyMasterPublish = null;
private AgentLaunchParameter agentLaunchParameter = null;
@@ -501,6 +501,9 @@ public class AgentProviderService extends AbstractProviderService implements
log.info("Recording allocated port for {} as {}", port.getKey(), port.getValue());
this.allocatedPorts.put(port.getKey(), port.getValue());
}
+
+ // component specific publishes
+ processComponentSpecificPublishes(ports, containerId, heartBeat.getFqdn(), roleName);
}
CommandResult result = CommandResult.getCommandResult(report.getStatus());
Command command = Command.getCommand(report.getRoleCommand());
@@ -635,6 +638,64 @@ public class AgentProviderService extends AbstractProviderService implements
}
}
+ /** Publish component instance specific data if the component demands it */
+ protected void processComponentSpecificPublishes(Map<String, String> ports,
+ String containerId,
+ String hostFqdn,
+ String roleName) {
+ String portVarFormat = "${site.%s}";
+ String hostNamePattern = "${THIS_HOST}";
+ Map<String, String> toPublish = new HashMap<>();
+
+ Application application = getMetainfo().getApplication();
+ for (Component component : application.getComponents()) {
+ if (component.getName().equals(roleName)) {
+ if(component.getExports().size() > 0) {
+
+ for(Export export : component.getExports()) {
+ String templateToExport = export.getValue();
+ for(String portName : ports.keySet()) {
+ boolean publishData = false;
+ String portValPattern = String.format(portVarFormat, portName);
+ if(templateToExport.contains(portValPattern)) {
+ templateToExport.replace(portValPattern, ports.get(portName));
+ publishData = true;
+ }
+ if(templateToExport.contains(hostNamePattern)) {
+ templateToExport.replace(hostNamePattern, hostFqdn);
+ publishData = true;
+ }
+ if(publishData) {
+ toPublish.put(export.getName(), templateToExport);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if(toPublish.size() > 0) {
+ Map<String, String> perContainerData = null;
+ if(!this.componentInstanceData.containsKey(containerId)) {
+ perContainerData = new ConcurrentHashMap<>();
+ } else {
+ perContainerData = this.componentInstanceData.get(containerId);
+ }
+ perContainerData.putAll(toPublish);
+ this.componentInstanceData.put(containerId, perContainerData);
+ Map<String, String> dataToPublish = new HashMap<>();
+ synchronized (this.componentInstanceData) {
+ for(String container : this.componentInstanceData.keySet()) {
+ for(String prop : this.componentInstanceData.get(container).keySet()) {
+ dataToPublish.put(
+ container + "." + prop, this.componentInstanceData.get(container).get(prop));
+ }
+ }
+ }
+ publishComponentConfiguration(COMPONENT_DATA_TAG, COMPONENT_DATA_TAG, dataToPublish.entrySet());
+ }
+ }
+
/**
* Extract script path from the application metainfo
*
@@ -971,8 +1032,9 @@ public class AgentProviderService extends AbstractProviderService implements
//apply any port updates
if (!this.getAllocatedPorts().isEmpty()) {
for (String key : config.keySet()) {
- if (this.getAllocatedPorts().containsKey(key)) {
- config.put(key, getAllocatedPorts().get(key));
+ String lookupKey = configName + "." + key;
+ if (this.getAllocatedPorts().containsKey(lookupKey)) {
+ config.put(key, getAllocatedPorts().get(lookupKey));
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad2c798d/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index c34d0ea..4d63263 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -843,8 +843,8 @@ public class TestAgentProviderService {
roleClusterNodeMap.put("HBASE_MASTER", container);
doReturn(roleClusterNodeMap).when(mockAps).getRoleClusterNodeMapping();
Map<String, String> allocatedPorts = new HashMap<>();
- allocatedPorts.put("a.port", "10023");
- allocatedPorts.put("b.port", "10024");
+ allocatedPorts.put("hbase-site.a.port", "10023");
+ allocatedPorts.put("hbase-site.b.port", "10024");
doReturn(allocatedPorts).when(mockAps).getAllocatedPorts();
replay(access);
[24/50] [abbrv] git commit: IDE-indicated import cleanup
Posted by st...@apache.org.
IDE-indicated import cleanup
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/868f7e2c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/868f7e2c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/868f7e2c
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 868f7e2c6319040d4f9edea202672f18b7939f1a
Parents: 3bb202f
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 6 17:29:18 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 6 17:29:18 2014 +0100
----------------------------------------------------------------------
.../providers/agent/TestAgentAMManagementWS.groovy | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/868f7e2c/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
index d3a0be7..e248ec3 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy
@@ -27,7 +27,6 @@ import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderKeys
import org.apache.slider.core.conf.MapOperations
import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.server.appmaster.web.SliderAMWebApp
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse
import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus
import org.apache.slider.server.services.security.CertificateManager
@@ -37,13 +36,15 @@ import org.junit.Test
import org.slf4j.Logger
import org.slf4j.LoggerFactory
+import javax.net.ssl.HostnameVerifier
+import javax.net.ssl.HttpsURLConnection
+import javax.net.ssl.SSLSession
import javax.ws.rs.core.MediaType
import static org.apache.slider.common.params.Arguments.ARG_OPTION
import static org.apache.slider.providers.agent.AgentKeys.*
import static org.apache.slider.providers.agent.AgentTestUtils.createDummyJSONRegister
import static org.apache.slider.providers.agent.AgentTestUtils.createTestClient
-import static org.apache.slider.test.SliderTestUtils.log
@CompileStatic
@Slf4j
@@ -53,10 +54,10 @@ class TestAgentAMManagementWS extends AgentTestBase {
final static Logger logger = LoggerFactory.getLogger(TestAgentAMManagementWS.class)
static {
//for localhost testing only
- javax.net.ssl.HttpsURLConnection.setDefaultHostnameVerifier(
- new javax.net.ssl.HostnameVerifier(){
+ HttpsURLConnection.setDefaultHostnameVerifier(
+ new HostnameVerifier(){
public boolean verify(String hostname,
- javax.net.ssl.SSLSession sslSession) {
+ SSLSession sslSession) {
logger.info("verifying hostname ${hostname}")
InetAddress[] addresses =
InetAddress.getAllByName(hostname);
[16/50] [abbrv] git commit: SLIDER-270: one-liner fix to actually
resolve the problem
Posted by st...@apache.org.
SLIDER-270: one-liner fix to actually resolve the problem
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8d6c3fad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8d6c3fad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8d6c3fad
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 8d6c3fad0e1edc9c482f43305f82c976e3d53aa9
Parents: df963f2
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 6 09:32:36 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 6 09:33:07 2014 +0100
----------------------------------------------------------------------
.../apache/slider/server/appmaster/state/AppState.java | 13 ++++++++++---
.../server/appmaster/model/mock/MockYarnEngine.groovy | 2 +-
2 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8d6c3fad/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 2b17c3a..db74ea8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -786,7 +786,10 @@ public class AppState {
public RoleInstance getActiveContainer(ContainerId id) {
return activeContainers.get(id);
}
-
+
+ private RoleInstance removeActiveContainer(ContainerId id) {
+ return activeContainers.remove(id);
+ }
public synchronized List<RoleInstance> cloneLiveContainerInfoList() {
List<RoleInstance> allRoleInstances;
@@ -1265,11 +1268,15 @@ public class AppState {
completionOfNodeNotInLiveListEvent.incrementAndGet();
}
+
+ // and the active node list if present
+ removeActiveContainer(containerId);
+
// finally, verify the node doesn't exist any more
assert !containersBeingReleased.containsKey(
- containerId) : "container in release queue";
+ containerId) : "container still in release queue";
assert !getLiveNodes().containsKey(
- containerId) : " container in live nodes";
+ containerId) : " container still in live nodes";
assert getActiveContainer(containerId) ==
null : "Container still in active container list";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8d6c3fad/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
index 763f653..b57a79a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
@@ -104,7 +104,7 @@ class MockYarnEngine {
if (op instanceof ContainerReleaseOperation) {
ContainerReleaseOperation cro = (ContainerReleaseOperation) op
ContainerId cid = cro.containerId
- releaseContainer(cid);
+ assert releaseContainer(cid);
released.add(cid)
} else {
ContainerRequestOperation req = (ContainerRequestOperation) op
[08/50] [abbrv] git commit: SLIDER-270 more work on the double flex
test case
Posted by st...@apache.org.
SLIDER-270 more work on the double flex test case
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/291a82ef
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/291a82ef
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/291a82ef
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 291a82ef1e99a6590e245ebce7c0d1d861eac263
Parents: 040ef79
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 5 21:09:00 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 5 21:09:00 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/test/YarnMiniClusterTestBase.groovy | 7 ++++---
.../hbase/minicluster/HBaseMiniClusterTestBase.groovy | 1 +
.../minicluster/flexing/TestClusterFlexDownMultiple.groovy | 4 ++--
3 files changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/291a82ef/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index 49dc02e..ed66190 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -136,6 +136,9 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
}
}
+ protected String buildClustername(String clustername) {
+ return clustername ?: createClusterName()
+ }
/**
* Create the cluster name from the method name and an auto-incrementing
@@ -240,9 +243,7 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
conf.set(YarnConfiguration.RM_SCHEDULER, FIFO_SCHEDULER);
SliderUtils.patchConfiguration(conf)
- if (!name) {
- name = createClusterName()
- }
+ name = buildClustername(name)
miniCluster = new MiniYARNCluster(name, noOfNodeManagers, numLocalDirs, numLogDirs)
miniCluster.init(conf)
miniCluster.start();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/291a82ef/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
index c9f0a4f..6a69e17 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
@@ -284,6 +284,7 @@ public abstract class HBaseMiniClusterTestBase extends YarnZKMiniClusterTestBase
int workers,
int workerFlexTarget,
boolean testHBaseAfter) {
+ clustername = buildClustername(clustername);
SliderClient sliderClient = startHBaseCluster(clustername, masters, workers)
//now flex
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/291a82ef/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
index 43ade73..97a9b35 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
@@ -34,13 +34,13 @@ class TestClusterFlexDownMultiple extends HBaseMiniClusterTestBase {
@Test
public void testClusterFlexDownMultiple() throws Throwable {
def clusterName = createClusterName();
- SliderClient sliderClient = startHBaseCluster(clusterName, 1, 4)
+ SliderClient sliderClient = startHBaseCluster(clusterName, 1, 3)
assert flexCluster(
sliderClient,
clusterName,
1,
- 3,
+ 2,
true)
assert flexCluster(
[11/50] [abbrv] git commit: SLIDER-272. Application should specify
the config types to publish
Posted by st...@apache.org.
SLIDER-272. Application should specify the config types to publish
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/640233dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/640233dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/640233dd
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 640233dd355f8ac3fe917a307ca3251caf6593e7
Parents: 50dc76c
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sun Aug 3 21:48:32 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Aug 5 18:51:58 2014 -0700
----------------------------------------------------------------------
app-packages/memcached-win/metainfo.xml | 1 +
app-packages/memcached/metainfo.xml | 1 +
.../providers/agent/AgentProviderService.java | 19 +++++++++++++++++--
.../agent/application/metadata/Application.java | 9 +++++++++
.../application/metadata/MetainfoParser.java | 1 +
.../agent/TestAgentProviderService.java | 2 ++
6 files changed, 31 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/640233dd/app-packages/memcached-win/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/metainfo.xml b/app-packages/memcached-win/metainfo.xml
index a0de8c2..dc2d67b 100644
--- a/app-packages/memcached-win/metainfo.xml
+++ b/app-packages/memcached-win/metainfo.xml
@@ -22,6 +22,7 @@
<name>MEMCACHED</name>
<comment>Memcache is a network accessible key/value storage system, often used as a distributed cache.</comment>
<version>1.0.0</version>
+ <exportedConfigs>None</exportedConfigs>
<components>
<component>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/640233dd/app-packages/memcached/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/memcached/metainfo.xml b/app-packages/memcached/metainfo.xml
index 0550b7d..08a504e 100644
--- a/app-packages/memcached/metainfo.xml
+++ b/app-packages/memcached/metainfo.xml
@@ -22,6 +22,7 @@
<name>MEMCACHED</name>
<comment>Memcache is a network accessible key/value storage system, often used as a distributed cache.</comment>
<version>1.0.0</version>
+ <exportedConfigs>None</exportedConfigs>
<components>
<component>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/640233dd/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 28c874b..682efba 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -88,6 +88,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
@@ -637,12 +638,26 @@ public class AgentProviderService extends AbstractProviderService implements
Application application = getMetainfo().getApplication();
List<ExportGroup> exportGroups = application.getExportGroups();
boolean hasExportGroups = exportGroups != null && !exportGroups.isEmpty();
+ Set<String> exportedConfigs = new HashSet();
+ String exportedConfigsStr = application.getExportedConfigs();
+ boolean exportedConfigSpecified = exportedConfigsStr != null;
+ if (application.getExportedConfigs() != null && application.getExportedConfigs().length() > 0) {
+ for (String exportedConfig : application.getExportedConfigs().split(",")) {
+ if (exportedConfig.trim().length() > 0) {
+ exportedConfigs.add(exportedConfig.trim());
+ }
+ }
+ }
+
for (ComponentStatus status : statuses) {
log.info("Status report: " + status.toString());
if (status.getConfigs() != null) {
for (String key : status.getConfigs().keySet()) {
- Map<String, String> configs = status.getConfigs().get(key);
- publishApplicationInstanceData(key, key, configs.entrySet());
+ if ((exportedConfigSpecified && exportedConfigs.contains(key)) ||
+ !exportedConfigSpecified) {
+ Map<String, String> configs = status.getConfigs().get(key);
+ publishApplicationInstanceData(key, key, configs.entrySet());
+ }
}
if (hasExportGroups) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/640233dd/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
index b007313..016b4d4 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
@@ -26,6 +26,7 @@ public class Application {
String name;
String comment;
String version;
+ String exportedConfigs;
List<Component> components;
List<ExportGroup> exportGroups;
List<OSSpecific> osSpecifics;
@@ -63,6 +64,14 @@ public class Application {
this.version = version;
}
+ public String getExportedConfigs() {
+ return exportedConfigs;
+ }
+
+ public void setExportedConfigs(String exportedConfigs) {
+ this.exportedConfigs = exportedConfigs;
+ }
+
public ConfigurationDependencies getConfigDependencies() {
return configDependencies;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/640233dd/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
index d2801d4..deb7a6a 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
@@ -38,6 +38,7 @@ public class MetainfoParser {
digester.addBeanPropertySetter("*/application/name");
digester.addBeanPropertySetter("*/application/comment");
digester.addBeanPropertySetter("*/application/version");
+ digester.addBeanPropertySetter("*/application/exportedConfigs");
digester.addObjectCreate("*/commandOrder", CommandOrder.class);
digester.addBeanPropertySetter("*/commandOrder/command");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/640233dd/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 10b1eb7..4b2e253 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -106,6 +106,7 @@ public class TestAgentProviderService {
+ " <version>0.96.0.2.1.1</version>\n"
+ " <type>YARN-APP</type>\n"
+ " <minHadoopVersion>2.1.0</minHadoopVersion>\n"
+ + " <exportedConfigs>hbase-site,global</exportedConfigs>\n"
+ " <exportGroups>\n"
+ " <exportGroup>\n"
+ " <name>QuickLinks</name>\n"
@@ -454,6 +455,7 @@ public class TestAgentProviderService {
Application application = metainfo.getApplication();
log.info("Service: " + application.toString());
Assert.assertEquals(application.getName(), "HBASE");
+ Assert.assertEquals(application.getExportedConfigs(), "hbase-site,global");
Assert.assertEquals(application.getComponents().size(), 2);
List<Component> components = application.getComponents();
int found = 0;
[22/50] [abbrv] git commit: SLIDER-296 removing use of
RoleHistory/nodemap target release selection logic
Posted by st...@apache.org.
SLIDER-296 removing use of RoleHistory/nodemap target release selection logic
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8cecce18
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8cecce18
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8cecce18
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 8cecce1821733c33de3cf9723f9a96aee411305d
Parents: 5ce953c
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 6 17:11:39 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 6 17:11:39 2014 +0100
----------------------------------------------------------------------
.../server/appmaster/state/NodeInstance.java | 12 +-
.../server/appmaster/state/RoleHistory.java | 14 +
.../history/TestFindNodesForNewInstances.groovy | 128 ---------
.../history/TestFindNodesForRelease.groovy | 146 -----------
.../model/history/TestHistoryRW.groovy | 258 -------------------
.../model/history/TestHistoryRWOrdering.groovy | 145 -----------
.../model/history/TestNIComparators.groovy | 77 ------
.../TestOutstandingRequestTracker.groovy | 73 ------
.../TestRoleHistoryContainerEvents.groovy | 17 +-
...stRoleHistoryFindNodesForNewInstances.groovy | 128 +++++++++
.../TestRoleHistoryFindNodesForRelease.groovy | 146 +++++++++++
.../history/TestRoleHistoryNIComparators.groovy | 77 ++++++
...tRoleHistoryOutstandingRequestTracker.groovy | 73 ++++++
.../model/history/TestRoleHistoryRW.groovy | 258 +++++++++++++++++++
.../history/TestRoleHistoryRWOrdering.groovy | 145 +++++++++++
15 files changed, 861 insertions(+), 836 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
index 06375fb..fad60f2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
@@ -75,7 +75,7 @@ public class NodeInstance {
}
/**
- * Cout the number of active role instances on this node
+ * Count the number of active role instances on this node
* @param role role index
* @return 0 if there are none, otherwise the #of nodes that are running and
* not being released already.
@@ -84,6 +84,16 @@ public class NodeInstance {
NodeEntry nodeEntry = get(role);
return (nodeEntry != null ) ? nodeEntry.getActive() : 0;
}
+
+ /**
+ * Count the number of live role instances on this node
+ * @param role role index
+ * @return 0 if there are none, otherwise the #of nodes that are running
+ */
+ public int getLiveRoleInstances(int role) {
+ NodeEntry nodeEntry = get(role);
+ return (nodeEntry != null ) ? nodeEntry.getLive() : 0;
+ }
/**
* Get the entry for a role -and remove it if present
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index de7e9d8..dbb104c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -529,6 +529,20 @@ public class RoleHistory {
return nodemap.findNodesForRelease(role, count);
}
+
+ /**
+ * Get the list of active nodes ... walks the node map so
+ * is O(nodes)
+ * @param role role index
+ * @return a possibly empty list of nodes with an instance of that node
+ */
+ public synchronized List<NodeInstance> listActiveNodes(int role) {
+ return nodemap.listActiveNodes(role);
+ }
+
+
+
+
/**
* Get the node entry of a container
* @param container container to look up
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForNewInstances.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForNewInstances.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForNewInstances.groovy
deleted file mode 100644
index dab03f5..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForNewInstances.groovy
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.slider.providers.ProviderRole
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.state.NodeInstance
-import org.apache.slider.server.appmaster.state.RoleHistory
-import org.apache.slider.server.appmaster.state.RoleStatus
-import org.junit.Before
-import org.junit.Test
-
-/**
- * Testing finding nodes for new instances.
- * These tests validate the (currently) suboptimal
- * behavior of not listing any known nodes when there
- * are none in the available list -even if there are nodes
- * known to be not running live instances in the cluster.
- */
-@Slf4j
-@CompileStatic
-class TestFindNodesForNewInstances extends BaseMockAppStateTest {
-
- @Override
- String getTestName() {
- return "TestFindNodesForNewInstances"
- }
-
- NodeInstance age1Active4 = nodeInstance(1, 4, 0, 0)
- NodeInstance age2Active2 = nodeInstance(2, 2, 0, 1)
- NodeInstance age3Active0 = nodeInstance(3, 0, 0, 0)
- NodeInstance age4Active1 = nodeInstance(4, 1, 0, 0)
- NodeInstance age2Active0 = nodeInstance(2, 0, 0, 0)
- NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
-
- List<NodeInstance> nodes = [age2Active2, age2Active0, age4Active1, age1Active4, age3Active0]
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
-
- String roleName = "test"
- RoleStatus roleStat = new RoleStatus(new ProviderRole(roleName, 0))
- RoleStatus roleStat2 = new RoleStatus(new ProviderRole(roleName, 2))
-
- @Before
- public void setupNodeMap() {
- roleHistory.insert(nodes)
- roleHistory.buildAvailableNodeLists();
- }
-
-
- public List<NodeInstance> findNodes(int count, RoleStatus roleStatus = roleStat) {
- List < NodeInstance > found = [];
- for (int i = 0; i < count; i++) {
- NodeInstance f = roleHistory.findNodeForNewInstance(roleStatus)
- if (f) {
- found << f
- };
- }
- return found
- }
-
- @Test
- public void testFind1NodeR0() throws Throwable {
- NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
- log.info("found: $found")
- assert [age3Active0].contains(found)
- }
-
- @Test
- public void testFind2NodeR0() throws Throwable {
- NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
- log.info("found: $found")
- assert [age2Active0, age3Active0].contains(found)
- NodeInstance found2 = roleHistory.findNodeForNewInstance(roleStat)
- log.info("found: $found2")
- assert [age2Active0, age3Active0].contains(found2)
- assert found != found2;
- }
- @Test
- public void testFind3NodeR0ReturnsNull() throws Throwable {
- assert 2== findNodes(2).size()
- NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
- assert found == null;
- }
-
- @Test
- public void testFindNodesOneEntry() throws Throwable {
- List<NodeInstance> nodes = findNodes(4, roleStat2)
- assert 0 == nodes.size()
- }
-
- @Test
- public void testFindNodesIndependent() throws Throwable {
- assert 2 == findNodes(2).size()
- roleHistory.dump()
- assert 0 == findNodes(3, roleStat2).size()
- }
-
- @Test
- public void testFindNodesFallsBackWhenUsed() throws Throwable {
- // mark age2 and active 0 as busy, expect a null back
- age2Active0.get(0).onStartCompleted()
- assert age2Active0.getActiveRoleInstances(0) != 0
- age3Active0.get(0).onStartCompleted()
- assert age3Active0.getActiveRoleInstances(0) != 0
- NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
- log.info(found ?.toFullString())
- assert found == null
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForRelease.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForRelease.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForRelease.groovy
deleted file mode 100644
index 92915dd..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestFindNodesForRelease.groovy
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.state.NodeInstance
-import org.apache.slider.server.appmaster.state.NodeMap
-import org.junit.Before
-import org.junit.Test
-
-@Slf4j
-@CompileStatic
-class TestFindNodesForRelease extends BaseMockAppStateTest {
-
-
- @Override
- String getTestName() {
- return "TestFindNodesForRelease"
- }
- NodeInstance age1Active4 = nodeInstance(1, 4, 0, 0)
- NodeInstance age2Active2 = nodeInstance(2, 2, 0, 0)
- NodeInstance age3Active0 = nodeInstance(3, 0, 0, 0)
- NodeInstance age4Active1 = nodeInstance(4, 1, 0, 0)
- NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
-
- List<NodeInstance> nodes = [age2Active2, age4Active1, age1Active4, age3Active0]
- NodeMap nodeMap = new NodeMap(MockFactory.ROLE_COUNT);
-
-
- @Before
- public void setupNodeMap() {
- nodeMap.insert(nodes)
- }
-
- private void assertReleased(
- int count,
- List<NodeInstance> expected,
- int role = 0) {
- List<NodeInstance> released = nodeMap.findNodesForRelease(role, count)
- assertListEquals(released, expected)
- }
- private void assertReleased(
- List<NodeInstance> expected,
- int role = 0) {
- List<NodeInstance> released = nodeMap.findNodesForRelease(role, expected.size())
- assertListEquals(released, expected)
- }
-
- @Test
- public void testListActiveNodes() throws Throwable {
- assertListEquals(nodeMap.listActiveNodes(0),
- [age1Active4,age2Active2, age4Active1])
- }
-
- @Test
- public void testReleaseMinus1() throws Throwable {
- try {
- nodeMap.findNodesForRelease(0, -1)
- fail("Expected an exception")
- } catch (IllegalArgumentException e) {
- }
- }
- @Test
- public void testReleaseO() throws Throwable {
- assertReleased(0, [])
- }
-
- @Test
- public void testRelease1() throws Throwable {
- assertReleased(1, [age1Active4])
- }
-
- @Test
- public void testRelease2() throws Throwable {
- assertReleased(2, [age1Active4, age1Active4])
- }
-
- @Test
- public void testRelease3() throws Throwable {
- assertReleased(3, [age1Active4, age1Active4, age1Active4 ])
- }
-
- @Test
- public void testRelease4() throws Throwable {
- assertReleased(4, [age1Active4, age1Active4, age1Active4 , age2Active2])
- }
-
- @Test
- public void testRelease5() throws Throwable {
- assertReleased([age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1])
- }
-
- @Test
- public void testRelease6() throws Throwable {
- assertReleased(
- [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1, age1Active4])
- }
-
- @Test
- public void testRelease7() throws Throwable {
- assertReleased(
- [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1,
- age1Active4, age2Active2])
- }
-
- @Test
- public void testRelease8() throws Throwable {
- assertReleased(8,
- [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1,
- age1Active4, age2Active2])
- }
-
- @Test
- public void testPurgeInactiveTime3() throws Throwable {
- assert nodeMap.purgeUnusedEntries(3) == 0;
- }
-
- @Test
- public void testPurgeInactiveTime4() throws Throwable {
- assert nodeMap.purgeUnusedEntries(4) == 1;
- }
- @Test
- public void testPurgeInactiveTime5() throws Throwable {
- assert nodeMap.purgeUnusedEntries(5) == 1;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRW.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRW.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRW.groovy
deleted file mode 100644
index b646661..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRW.groovy
+++ /dev/null
@@ -1,258 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.fs.FSDataOutputStream
-import org.apache.hadoop.fs.Path
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.state.NodeEntry
-import org.apache.slider.server.appmaster.state.NodeInstance
-import org.apache.slider.server.appmaster.state.RoleHistory
-import org.apache.slider.server.avro.RoleHistoryWriter
-import org.junit.Test
-
-@Slf4j
-@CompileStatic
-class TestHistoryRW extends BaseMockAppStateTest {
-
- static long time = System.currentTimeMillis();
-
- @Override
- String getTestName() {
- return "TestHistoryRW"
- }
-
- @Test
- public void testWriteReadEmpty() throws Throwable {
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
- roleHistory.onStart(fs, historyPath)
- Path history = roleHistory.saveHistory(time++)
- assert fs.isFile(history)
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- historyWriter.read(fs, history, roleHistory)
- }
-
- @Test
- public void testWriteReadData() throws Throwable {
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
- assert !roleHistory.onStart(fs, historyPath)
- String addr = "localhost"
- NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
- NodeEntry ne1 = instance.getOrCreate(0)
- ne1.lastUsed = 0xf00d
-
- Path history = roleHistory.saveHistory(time++)
- assert fs.isFile(history)
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
-
- assert 0 < historyWriter.read(fs, history, rh2)
- NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
- assert ni2 != null
- NodeEntry ne2 = ni2.get(0)
- assert ne2 !=null
- assert ne2.lastUsed == ne1.lastUsed
- }
-
- @Test
- public void testWriteReadActiveData() throws Throwable {
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
- roleHistory.onStart(fs, historyPath)
- String addr = "localhost"
- String addr2 = "rack1server5"
- NodeInstance localhost = roleHistory.getOrCreateNodeInstance(addr)
- NodeEntry orig1 = localhost.getOrCreate(0)
- orig1.lastUsed = 0x10
- NodeInstance rack1server5 = roleHistory.getOrCreateNodeInstance(addr2)
- NodeEntry orig2 = rack1server5.getOrCreate(1)
- orig2.live = 3
- assert !orig2.available
- NodeEntry orig3 = localhost.getOrCreate(1)
- orig3.lastUsed = 0x20
- orig3.live = 1
- assert !orig3.available
- orig3.release()
- assert orig3.available
- roleHistory.dump()
-
- long savetime = 0x0001000
- Path history = roleHistory.saveHistory(savetime)
- assert fs.isFile(history)
- describe("Loaded")
- log.info("testWriteReadActiveData in $history")
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
-
- assert 3 == historyWriter.read(fs, history, rh2)
- rh2.dump()
-
- assert rh2.clusterSize == 2;
- NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
- assert ni2 != null
- NodeEntry loadedNE = ni2.get(0)
- assert loadedNE.lastUsed == orig1.lastUsed
- NodeInstance ni2b = rh2.getExistingNodeInstance(addr2)
- assert ni2b != null
- NodeEntry loadedNE2 = ni2b.get(1)
- assert loadedNE2 != null
- assert loadedNE2.lastUsed == savetime
- assert rh2.thawedDataTime == savetime
-
- // now thaw it
- rh2.buildAvailableNodeLists();
- describe("thawing")
- rh2.dump();
- List<NodeInstance> available0 = rh2.cloneAvailableList(0)
- assert available0.size() == 1
-
- NodeInstance entry = available0.get(0)
- assert entry.hostname == "localhost"
- assert entry == localhost
- List<NodeInstance> available1 = rh2.cloneAvailableList(1)
- assert available1.size() == 2
- //and verify that even if last used was set, the save time is picked up
- assert entry.get(1).lastUsed == roleHistory.saveTime
-
- }
-
- @Test
- public void testWriteThaw() throws Throwable {
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
- assert !roleHistory.onStart(fs, historyPath)
- String addr = "localhost"
- NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
- NodeEntry ne1 = instance.getOrCreate(0)
- ne1.lastUsed = 0xf00d
-
- Path history = roleHistory.saveHistory(time++)
- long savetime =roleHistory.saveTime;
- assert fs.isFile(history)
- RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
- assert rh2.onStart(fs, historyPath)
- NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
- assert ni2 != null
- NodeEntry ne2 = ni2.get(0)
- assert ne2 != null
- assert ne2.lastUsed == ne1.lastUsed
- assert rh2.thawedDataTime == savetime
- }
-
-
- @Test
- public void testPurgeOlderEntries() throws Throwable {
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- time = 1;
- Path file1 = touch(historyWriter, time++)
- Path file2 = touch(historyWriter, time++)
- Path file3 = touch(historyWriter, time++)
- Path file4 = touch(historyWriter, time++)
- Path file5 = touch(historyWriter, time++)
- Path file6 = touch(historyWriter, time++)
-
- assert historyWriter.purgeOlderHistoryEntries(fs, file1) == 0
- assert historyWriter.purgeOlderHistoryEntries(fs, file2) == 1
- assert historyWriter.purgeOlderHistoryEntries(fs, file2) == 0
- assert historyWriter.purgeOlderHistoryEntries(fs, file5) == 3
- assert historyWriter.purgeOlderHistoryEntries(fs, file6) == 1
- try {
- // make an impossible assertion that will fail if the method
- // actually completes
- assert -1 == historyWriter.purgeOlderHistoryEntries(fs, file1)
- } catch (FileNotFoundException ignored) {
- // expected
- }
-
- }
-
- public Path touch(RoleHistoryWriter historyWriter, long time){
- Path path = historyWriter.createHistoryFilename(historyPath, time);
- FSDataOutputStream out = fs.create(path);
- out.close()
- return path
- }
-
- @Test
- public void testSkipEmptyFileOnRead() throws Throwable {
- describe "verify that empty histories are skipped on read; old histories purged"
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
- roleHistory.onStart(fs, historyPath)
- time = 0
- Path oldhistory = roleHistory.saveHistory(time++)
-
- String addr = "localhost"
- NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
- NodeEntry ne1 = instance.getOrCreate(0)
- ne1.lastUsed = 0xf00d
-
- Path goodhistory = roleHistory.saveHistory(time++)
-
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- Path touched = touch(historyWriter, time++)
-
- RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
- assert rh2.onStart(fs, historyPath)
- NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
- assert ni2 != null
-
- //and assert the older file got purged
- assert !fs.exists(oldhistory)
- assert fs.exists(goodhistory)
- assert fs.exists(touched )
- }
-
- @Test
- public void testSkipBrokenFileOnRead() throws Throwable {
- describe "verify that empty histories are skipped on read; old histories purged"
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
- roleHistory.onStart(fs, historyPath)
- time = 0
- Path oldhistory = roleHistory.saveHistory(time++)
-
- String addr = "localhost"
- NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
- NodeEntry ne1 = instance.getOrCreate(0)
- ne1.lastUsed = 0xf00d
-
- Path goodhistory = roleHistory.saveHistory(time++)
-
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- Path badfile = historyWriter.createHistoryFilename(historyPath, time++)
- FSDataOutputStream out = fs.create(badfile)
- out.writeBytes("{broken:true}")
- out.close()
-
- RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
- describe("IGNORE STACK TRACE BELOW")
-
- assert rh2.onStart(fs, historyPath)
-
- describe( "IGNORE STACK TRACE ABOVE")
- NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
- assert ni2 != null
-
- //and assert the older file got purged
- assert !fs.exists(oldhistory)
- assert fs.exists(goodhistory)
- assert fs.exists(badfile )
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRWOrdering.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRWOrdering.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRWOrdering.groovy
deleted file mode 100644
index 6ec046c..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestHistoryRWOrdering.groovy
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history
-
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.fs.Path
-import org.apache.slider.common.SliderKeys
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.state.NodeEntry
-import org.apache.slider.server.appmaster.state.NodeInstance
-import org.apache.slider.server.appmaster.state.RoleHistory
-import org.apache.slider.server.avro.RoleHistoryWriter
-import org.junit.Test
-
-import java.util.regex.Matcher
-import java.util.regex.Pattern
-
-@Slf4j
-class TestHistoryRWOrdering extends BaseMockAppStateTest {
-
- def paths = pathlist(
- [
- "hdfs://localhost/history-0406c.json",
- "hdfs://localhost/history-5fffa.json",
- "hdfs://localhost/history-0001a.json",
- "hdfs://localhost/history-0001f.json",
- ]
- )
- Path h_0406c = paths[0]
- Path h_5fffa = paths[1]
- Path h_0001a = paths[3]
-
-
- List<Path> pathlist(List<String> pathnames) {
- def result = []
- pathnames.each { result << new Path(new URI(it as String)) }
- result
- }
-
- @Override
- String getTestName() {
- return "TestHistoryRWOrdering"
- }
-
-
- /**
- * This tests regexp pattern matching. It uses the current time so isn't
- * repeatable -but it does test a wider range of values in the process
- * @throws Throwable
- */
- @Test
- public void testPatternRoundTrip() throws Throwable {
- describe "test pattern matching of names"
- long value=System.currentTimeMillis()
- String name = String.format(SliderKeys.HISTORY_FILENAME_CREATION_PATTERN,value)
- String matchpattern = SliderKeys.HISTORY_FILENAME_MATCH_PATTERN
- Pattern pattern = Pattern.compile(matchpattern)
- Matcher matcher = pattern.matcher(name);
- if (!matcher.find()) {
- throw new Exception("No match for pattern $matchpattern in $name")
- }
- }
-
-
- @Test
- public void testWriteSequenceReadData() throws Throwable {
- describe "test that if multiple entries are written, the newest is picked up"
- long time = System.currentTimeMillis();
-
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
- assert !roleHistory.onStart(fs, historyPath)
- String addr = "localhost"
- NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
- NodeEntry ne1 = instance.getOrCreate(0)
- ne1.lastUsed = 0xf00d
-
- Path history1 = roleHistory.saveHistory(time++)
- Path history2 = roleHistory.saveHistory(time++)
- Path history3 = roleHistory.saveHistory(time++)
-
- //inject a later file with a different name
- sliderFileSystem.cat(new Path(historyPath, "file.json"), true, "hello, world")
-
-
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
-
- List<Path> entries = historyWriter.findAllHistoryEntries(
- fs,
- historyPath,
- false)
- assert entries.size() == 3
- assert entries[0] == history3
- assert entries[1] == history2
- assert entries[2] == history1
- }
-
- @Test
- public void testPathStructure() throws Throwable {
- assert h_5fffa.getName() == "history-5fffa.json"
- }
-
- @Test
- public void testPathnameComparator() throws Throwable {
-
- def newerName = new RoleHistoryWriter.NewerFilesFirst()
-
- log.info("$h_5fffa name is ${h_5fffa.getName()}")
- log.info("$h_0406c name is ${h_0406c.getName()}")
- assert newerName.compare(h_5fffa, h_5fffa) == 0
- assert newerName.compare(h_5fffa, h_0406c) < 0
- assert newerName.compare(h_5fffa, h_0001a) < 0
- assert newerName.compare(h_0001a, h_5fffa) > 0
-
- }
-
- @Test
- public void testPathSort() throws Throwable {
- def paths2 = new ArrayList(paths)
- RoleHistoryWriter.sortHistoryPaths(paths2)
- assertListEquals(paths2,
- [
- paths[1],
- paths[0],
- paths[3],
- paths[2]
- ])
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestNIComparators.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestNIComparators.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestNIComparators.groovy
deleted file mode 100644
index 77119d5..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestNIComparators.groovy
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history
-
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.state.NodeInstance
-import org.junit.Test
-
-/**
- * Unit test to verify the comparators sort as expected
- */
-class TestNIComparators extends BaseMockAppStateTest {
-
- NodeInstance age1Active4 = nodeInstance(1000, 4, 0, 0)
- NodeInstance age2Active2 = nodeInstance(1001, 2, 0, 0)
- NodeInstance age3Active0 = nodeInstance(1002, 0, 0, 0)
- NodeInstance age4Active1 = nodeInstance(1005, 0, 0, 0)
- NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
-
- List<NodeInstance> nodes = [age2Active2, age4Active1, age1Active4, age3Active0]
-
- @Override
- String getTestName() {
- return "TestNIComparators"
- }
-
- @Test
- public void testNewerThan() throws Throwable {
-
- Collections.sort(nodes, new NodeInstance.newerThan(0))
- assertListEquals(nodes,
- [age4Active1, age3Active0, age2Active2, age1Active4])
- }
-
- @Test
- public void testNewerThanNoRole() throws Throwable {
-
- nodes << empty
- Collections.sort(nodes, new NodeInstance.newerThan(0))
- assertListEquals(nodes,
- [age4Active1, age3Active0, age2Active2, age1Active4, empty])
- }
-
- @Test
- public void testMoreActiveThan() throws Throwable {
-
- Collections.sort(nodes, new NodeInstance.moreActiveThan(0))
- assertListEquals(nodes,
- [age1Active4, age2Active2, age4Active1, age3Active0],)
- }
-
- @Test
- public void testMoreActiveThanEmpty() throws Throwable {
- nodes << empty
- Collections.sort(nodes, new NodeInstance.moreActiveThan(0))
- assertListEquals(nodes,
- [age1Active4, age2Active2, age4Active1, age3Active0, empty])
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestOutstandingRequestTracker.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestOutstandingRequestTracker.groovy
deleted file mode 100644
index 8d1f4b0..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestOutstandingRequestTracker.groovy
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history
-
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.state.NodeInstance
-import org.apache.slider.server.appmaster.state.OutstandingRequest
-import org.apache.slider.server.appmaster.state.OutstandingRequestTracker
-import org.junit.Test
-
-class TestOutstandingRequestTracker extends BaseMockAppStateTest {
-
- NodeInstance host1 = new NodeInstance("host1", 3)
- NodeInstance host2 = new NodeInstance("host2", 3)
-
- OutstandingRequestTracker tracker = new OutstandingRequestTracker()
-
- @Override
- String getTestName() {
- return "TestOutstandingRequestTracker"
- }
-
- @Test
- public void testAddRetrieveEntry() throws Throwable {
- OutstandingRequest request = tracker.addRequest(host1, 0)
- assert tracker.lookup(0, "host1").equals(request)
- assert tracker.remove(request).equals(request)
- assert !tracker.lookup(0, "host1")
- }
-
- @Test
- public void testAddCompleteEntry() throws Throwable {
- tracker.addRequest(host1, 0)
- tracker.addRequest(host2, 0)
- tracker.addRequest(host1, 1)
- assert tracker.onContainerAllocated(1, "host1")
- assert !tracker.lookup(1, "host1")
- assert tracker.lookup(0, "host1")
- }
-
- @Test
- public void testCancelEntries() throws Throwable {
- OutstandingRequest r1 = tracker.addRequest(host1, 0)
- OutstandingRequest r2 = tracker.addRequest(host2, 0)
- OutstandingRequest r3 = tracker.addRequest(host1, 1)
- List<NodeInstance> canceled = tracker.cancelOutstandingRequests(0)
- assert canceled.size() == 2
- assert canceled.contains(host1)
- assert canceled.contains(host2)
- assert tracker.lookup(1, "host1")
- assert !tracker.lookup(0, "host1")
- canceled = tracker.cancelOutstandingRequests(0)
- assert canceled.size() == 0
- assert tracker.cancelOutstandingRequests(1).size() == 1
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
index 1a71aed..340e72d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
@@ -114,12 +114,12 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
RoleStatus roleStatus = new RoleStatus(provRole)
//verify it is empty
- assert roleHistory.findNodesForRelease(role, 1).isEmpty()
+ assert roleHistory.listActiveNodes(role).empty
AMRMClient.ContainerRequest request =
roleHistory.requestNode(roleStatus, resource);
- List<String> nodes = request.getNodes()
+ List<String> nodes = request.nodes
assert nodes == null
//pick an idle host
@@ -128,7 +128,7 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
//build a container
MockContainer container = factory.newContainer()
container.nodeId = new MockNodeId(hostname, 0)
- container.priority = request.getPriority()
+ container.priority = request.priority
roleHistory.onContainerAssigned(container);
NodeMap nodemap = roleHistory.cloneNodemap();
@@ -147,10 +147,11 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
assert roleEntry.live == 1
// now pick that instance to destroy
+ List<NodeInstance> activeNodes = roleHistory.listActiveNodes(role)
+
- List<NodeInstance> forRelease = roleHistory.findNodesForRelease(role, 1)
- assert forRelease.size() == 1
- NodeInstance target = forRelease[0]
+ assert activeNodes.size() == 1
+ NodeInstance target = activeNodes[0]
assert target == allocated
roleHistory.onContainerReleaseSubmitted(container);
assert roleEntry.releasing == 1
@@ -164,13 +165,13 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
assert roleEntry.active == 0
// verify it is empty
- assert roleHistory.findNodesForRelease(role, 1).isEmpty()
+ assert roleHistory.listActiveNodes(role).empty
// ask for a container and expect to get the recently released one
AMRMClient.ContainerRequest request2 =
roleHistory.requestNode(roleStatus, resource);
- List<String> nodes2 = request2.getNodes()
+ List<String> nodes2 = request2.nodes
assert nodes2 != null
String hostname2 = nodes2[0]
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy
new file mode 100644
index 0000000..79cd348
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.providers.ProviderRole
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.RoleHistory
+import org.apache.slider.server.appmaster.state.RoleStatus
+import org.junit.Before
+import org.junit.Test
+
+/**
+ * Testing finding nodes for new instances.
+ * These tests validate the (currently) suboptimal
+ * behavior of not listing any known nodes when there
+ * are none in the available list -even if there are nodes
+ * known to be not running live instances in the cluster.
+ */
+@Slf4j
+@CompileStatic
+class TestRoleHistoryFindNodesForNewInstances extends BaseMockAppStateTest {
+
+ @Override
+ String getTestName() {
+ return "TestFindNodesForNewInstances"
+ }
+
+ NodeInstance age1Active4 = nodeInstance(1, 4, 0, 0)
+ NodeInstance age2Active2 = nodeInstance(2, 2, 0, 1)
+ NodeInstance age3Active0 = nodeInstance(3, 0, 0, 0)
+ NodeInstance age4Active1 = nodeInstance(4, 1, 0, 0)
+ NodeInstance age2Active0 = nodeInstance(2, 0, 0, 0)
+ NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
+
+ List<NodeInstance> nodes = [age2Active2, age2Active0, age4Active1, age1Active4, age3Active0]
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+
+ String roleName = "test"
+ RoleStatus roleStat = new RoleStatus(new ProviderRole(roleName, 0))
+ RoleStatus roleStat2 = new RoleStatus(new ProviderRole(roleName, 2))
+
+ @Before
+ public void setupNodeMap() {
+ roleHistory.insert(nodes)
+ roleHistory.buildAvailableNodeLists();
+ }
+
+
+ public List<NodeInstance> findNodes(int count, RoleStatus roleStatus = roleStat) {
+ List < NodeInstance > found = [];
+ for (int i = 0; i < count; i++) {
+ NodeInstance f = roleHistory.findNodeForNewInstance(roleStatus)
+ if (f) {
+ found << f
+ };
+ }
+ return found
+ }
+
+ @Test
+ public void testFind1NodeR0() throws Throwable {
+ NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
+ log.info("found: $found")
+ assert [age3Active0].contains(found)
+ }
+
+ @Test
+ public void testFind2NodeR0() throws Throwable {
+ NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
+ log.info("found: $found")
+ assert [age2Active0, age3Active0].contains(found)
+ NodeInstance found2 = roleHistory.findNodeForNewInstance(roleStat)
+ log.info("found: $found2")
+ assert [age2Active0, age3Active0].contains(found2)
+ assert found != found2;
+ }
+ @Test
+ public void testFind3NodeR0ReturnsNull() throws Throwable {
+ assert 2== findNodes(2).size()
+ NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
+ assert found == null;
+ }
+
+ @Test
+ public void testFindNodesOneEntry() throws Throwable {
+ List<NodeInstance> nodes = findNodes(4, roleStat2)
+ assert 0 == nodes.size()
+ }
+
+ @Test
+ public void testFindNodesIndependent() throws Throwable {
+ assert 2 == findNodes(2).size()
+ roleHistory.dump()
+ assert 0 == findNodes(3, roleStat2).size()
+ }
+
+ @Test
+ public void testFindNodesFallsBackWhenUsed() throws Throwable {
+ // mark age2 and active 0 as busy, expect a null back
+ age2Active0.get(0).onStartCompleted()
+ assert age2Active0.getActiveRoleInstances(0) != 0
+ age3Active0.get(0).onStartCompleted()
+ assert age3Active0.getActiveRoleInstances(0) != 0
+ NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
+ log.info(found ?.toFullString())
+ assert found == null
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForRelease.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForRelease.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForRelease.groovy
new file mode 100644
index 0000000..64db899
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForRelease.groovy
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.NodeMap
+import org.junit.Before
+import org.junit.Test
+
+@Slf4j
+@CompileStatic
+class TestRoleHistoryFindNodesForRelease extends BaseMockAppStateTest {
+
+
+ @Override
+ String getTestName() {
+ return "TestFindNodesForRelease"
+ }
+ NodeInstance age1Active4 = nodeInstance(1, 4, 0, 0)
+ NodeInstance age2Active2 = nodeInstance(2, 2, 0, 0)
+ NodeInstance age3Active0 = nodeInstance(3, 0, 0, 0)
+ NodeInstance age4Active1 = nodeInstance(4, 1, 0, 0)
+ NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
+
+ List<NodeInstance> nodes = [age2Active2, age4Active1, age1Active4, age3Active0]
+ NodeMap nodeMap = new NodeMap(MockFactory.ROLE_COUNT);
+
+
+ @Before
+ public void setupNodeMap() {
+ nodeMap.insert(nodes)
+ }
+
+ private void assertReleased(
+ int count,
+ List<NodeInstance> expected,
+ int role = 0) {
+ List<NodeInstance> released = nodeMap.findNodesForRelease(role, count)
+ assertListEquals(released, expected)
+ }
+ private void assertReleased(
+ List<NodeInstance> expected,
+ int role = 0) {
+ List<NodeInstance> released = nodeMap.findNodesForRelease(role, expected.size())
+ assertListEquals(released, expected)
+ }
+
+ @Test
+ public void testListActiveNodes() throws Throwable {
+ assertListEquals(nodeMap.listActiveNodes(0),
+ [age1Active4,age2Active2, age4Active1])
+ }
+
+ @Test
+ public void testReleaseMinus1() throws Throwable {
+ try {
+ nodeMap.findNodesForRelease(0, -1)
+ fail("Expected an exception")
+ } catch (IllegalArgumentException e) {
+ }
+ }
+ @Test
+ public void testReleaseO() throws Throwable {
+ assertReleased(0, [])
+ }
+
+ @Test
+ public void testRelease1() throws Throwable {
+ assertReleased(1, [age1Active4])
+ }
+
+ @Test
+ public void testRelease2() throws Throwable {
+ assertReleased(2, [age1Active4, age1Active4])
+ }
+
+ @Test
+ public void testRelease3() throws Throwable {
+ assertReleased(3, [age1Active4, age1Active4, age1Active4 ])
+ }
+
+ @Test
+ public void testRelease4() throws Throwable {
+ assertReleased(4, [age1Active4, age1Active4, age1Active4 , age2Active2])
+ }
+
+ @Test
+ public void testRelease5() throws Throwable {
+ assertReleased([age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1])
+ }
+
+ @Test
+ public void testRelease6() throws Throwable {
+ assertReleased(
+ [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1, age1Active4])
+ }
+
+ @Test
+ public void testRelease7() throws Throwable {
+ assertReleased(
+ [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1,
+ age1Active4, age2Active2])
+ }
+
+ @Test
+ public void testRelease8() throws Throwable {
+ assertReleased(8,
+ [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1,
+ age1Active4, age2Active2])
+ }
+
+ @Test
+ public void testPurgeInactiveTime3() throws Throwable {
+ assert nodeMap.purgeUnusedEntries(3) == 0;
+ }
+
+ @Test
+ public void testPurgeInactiveTime4() throws Throwable {
+ assert nodeMap.purgeUnusedEntries(4) == 1;
+ }
+ @Test
+ public void testPurgeInactiveTime5() throws Throwable {
+ assert nodeMap.purgeUnusedEntries(5) == 1;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy
new file mode 100644
index 0000000..612cce8
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history
+
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.junit.Test
+
+/**
+ * Unit test to verify the comparators sort as expected
+ */
+class TestRoleHistoryNIComparators extends BaseMockAppStateTest {
+
+ NodeInstance age1Active4 = nodeInstance(1000, 4, 0, 0)
+ NodeInstance age2Active2 = nodeInstance(1001, 2, 0, 0)
+ NodeInstance age3Active0 = nodeInstance(1002, 0, 0, 0)
+ NodeInstance age4Active1 = nodeInstance(1005, 0, 0, 0)
+ NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
+
+ List<NodeInstance> nodes = [age2Active2, age4Active1, age1Active4, age3Active0]
+
+ @Override
+ String getTestName() {
+ return "TestNIComparators"
+ }
+
+ @Test
+ public void testNewerThan() throws Throwable {
+
+ Collections.sort(nodes, new NodeInstance.newerThan(0))
+ assertListEquals(nodes,
+ [age4Active1, age3Active0, age2Active2, age1Active4])
+ }
+
+ @Test
+ public void testNewerThanNoRole() throws Throwable {
+
+ nodes << empty
+ Collections.sort(nodes, new NodeInstance.newerThan(0))
+ assertListEquals(nodes,
+ [age4Active1, age3Active0, age2Active2, age1Active4, empty])
+ }
+
+ @Test
+ public void testMoreActiveThan() throws Throwable {
+
+ Collections.sort(nodes, new NodeInstance.moreActiveThan(0))
+ assertListEquals(nodes,
+ [age1Active4, age2Active2, age4Active1, age3Active0],)
+ }
+
+ @Test
+ public void testMoreActiveThanEmpty() throws Throwable {
+ nodes << empty
+ Collections.sort(nodes, new NodeInstance.moreActiveThan(0))
+ assertListEquals(nodes,
+ [age1Active4, age2Active2, age4Active1, age3Active0, empty])
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
new file mode 100644
index 0000000..7085678
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history
+
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.OutstandingRequest
+import org.apache.slider.server.appmaster.state.OutstandingRequestTracker
+import org.junit.Test
+
+class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
+
+ NodeInstance host1 = new NodeInstance("host1", 3)
+ NodeInstance host2 = new NodeInstance("host2", 3)
+
+ OutstandingRequestTracker tracker = new OutstandingRequestTracker()
+
+ @Override
+ String getTestName() {
+ return "TestOutstandingRequestTracker"
+ }
+
+ @Test
+ public void testAddRetrieveEntry() throws Throwable {
+ OutstandingRequest request = tracker.addRequest(host1, 0)
+ assert tracker.lookup(0, "host1").equals(request)
+ assert tracker.remove(request).equals(request)
+ assert !tracker.lookup(0, "host1")
+ }
+
+ @Test
+ public void testAddCompleteEntry() throws Throwable {
+ tracker.addRequest(host1, 0)
+ tracker.addRequest(host2, 0)
+ tracker.addRequest(host1, 1)
+ assert tracker.onContainerAllocated(1, "host1")
+ assert !tracker.lookup(1, "host1")
+ assert tracker.lookup(0, "host1")
+ }
+
+ @Test
+ public void testCancelEntries() throws Throwable {
+ OutstandingRequest r1 = tracker.addRequest(host1, 0)
+ OutstandingRequest r2 = tracker.addRequest(host2, 0)
+ OutstandingRequest r3 = tracker.addRequest(host1, 1)
+ List<NodeInstance> canceled = tracker.cancelOutstandingRequests(0)
+ assert canceled.size() == 2
+ assert canceled.contains(host1)
+ assert canceled.contains(host2)
+ assert tracker.lookup(1, "host1")
+ assert !tracker.lookup(0, "host1")
+ canceled = tracker.cancelOutstandingRequests(0)
+ assert canceled.size() == 0
+ assert tracker.cancelOutstandingRequests(1).size() == 1
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
new file mode 100644
index 0000000..0f4a4ca
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.FSDataOutputStream
+import org.apache.hadoop.fs.Path
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.state.NodeEntry
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.RoleHistory
+import org.apache.slider.server.avro.RoleHistoryWriter
+import org.junit.Test
+
+@Slf4j
+@CompileStatic
+class TestRoleHistoryRW extends BaseMockAppStateTest {
+
+ static long time = System.currentTimeMillis();
+
+ @Override
+ String getTestName() {
+ return "TestHistoryRW"
+ }
+
+ @Test
+ public void testWriteReadEmpty() throws Throwable {
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ roleHistory.onStart(fs, historyPath)
+ Path history = roleHistory.saveHistory(time++)
+ assert fs.isFile(history)
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ historyWriter.read(fs, history, roleHistory)
+ }
+
+ @Test
+ public void testWriteReadData() throws Throwable {
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ assert !roleHistory.onStart(fs, historyPath)
+ String addr = "localhost"
+ NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
+ NodeEntry ne1 = instance.getOrCreate(0)
+ ne1.lastUsed = 0xf00d
+
+ Path history = roleHistory.saveHistory(time++)
+ assert fs.isFile(history)
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+
+ assert 0 < historyWriter.read(fs, history, rh2)
+ NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
+ assert ni2 != null
+ NodeEntry ne2 = ni2.get(0)
+ assert ne2 !=null
+ assert ne2.lastUsed == ne1.lastUsed
+ }
+
+ @Test
+ public void testWriteReadActiveData() throws Throwable {
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ roleHistory.onStart(fs, historyPath)
+ String addr = "localhost"
+ String addr2 = "rack1server5"
+ NodeInstance localhost = roleHistory.getOrCreateNodeInstance(addr)
+ NodeEntry orig1 = localhost.getOrCreate(0)
+ orig1.lastUsed = 0x10
+ NodeInstance rack1server5 = roleHistory.getOrCreateNodeInstance(addr2)
+ NodeEntry orig2 = rack1server5.getOrCreate(1)
+ orig2.live = 3
+ assert !orig2.available
+ NodeEntry orig3 = localhost.getOrCreate(1)
+ orig3.lastUsed = 0x20
+ orig3.live = 1
+ assert !orig3.available
+ orig3.release()
+ assert orig3.available
+ roleHistory.dump()
+
+ long savetime = 0x0001000
+ Path history = roleHistory.saveHistory(savetime)
+ assert fs.isFile(history)
+ describe("Loaded")
+ log.info("testWriteReadActiveData in $history")
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+
+ assert 3 == historyWriter.read(fs, history, rh2)
+ rh2.dump()
+
+ assert rh2.clusterSize == 2;
+ NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
+ assert ni2 != null
+ NodeEntry loadedNE = ni2.get(0)
+ assert loadedNE.lastUsed == orig1.lastUsed
+ NodeInstance ni2b = rh2.getExistingNodeInstance(addr2)
+ assert ni2b != null
+ NodeEntry loadedNE2 = ni2b.get(1)
+ assert loadedNE2 != null
+ assert loadedNE2.lastUsed == savetime
+ assert rh2.thawedDataTime == savetime
+
+ // now thaw it
+ rh2.buildAvailableNodeLists();
+ describe("thawing")
+ rh2.dump();
+ List<NodeInstance> available0 = rh2.cloneAvailableList(0)
+ assert available0.size() == 1
+
+ NodeInstance entry = available0.get(0)
+ assert entry.hostname == "localhost"
+ assert entry == localhost
+ List<NodeInstance> available1 = rh2.cloneAvailableList(1)
+ assert available1.size() == 2
+ //and verify that even if last used was set, the save time is picked up
+ assert entry.get(1).lastUsed == roleHistory.saveTime
+
+ }
+
+ @Test
+ public void testWriteThaw() throws Throwable {
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ assert !roleHistory.onStart(fs, historyPath)
+ String addr = "localhost"
+ NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
+ NodeEntry ne1 = instance.getOrCreate(0)
+ ne1.lastUsed = 0xf00d
+
+ Path history = roleHistory.saveHistory(time++)
+ long savetime =roleHistory.saveTime;
+ assert fs.isFile(history)
+ RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+ assert rh2.onStart(fs, historyPath)
+ NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
+ assert ni2 != null
+ NodeEntry ne2 = ni2.get(0)
+ assert ne2 != null
+ assert ne2.lastUsed == ne1.lastUsed
+ assert rh2.thawedDataTime == savetime
+ }
+
+
+ @Test
+ public void testPurgeOlderEntries() throws Throwable {
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ time = 1;
+ Path file1 = touch(historyWriter, time++)
+ Path file2 = touch(historyWriter, time++)
+ Path file3 = touch(historyWriter, time++)
+ Path file4 = touch(historyWriter, time++)
+ Path file5 = touch(historyWriter, time++)
+ Path file6 = touch(historyWriter, time++)
+
+ assert historyWriter.purgeOlderHistoryEntries(fs, file1) == 0
+ assert historyWriter.purgeOlderHistoryEntries(fs, file2) == 1
+ assert historyWriter.purgeOlderHistoryEntries(fs, file2) == 0
+ assert historyWriter.purgeOlderHistoryEntries(fs, file5) == 3
+ assert historyWriter.purgeOlderHistoryEntries(fs, file6) == 1
+ try {
+ // make an impossible assertion that will fail if the method
+ // actually completes
+ assert -1 == historyWriter.purgeOlderHistoryEntries(fs, file1)
+ } catch (FileNotFoundException ignored) {
+ // expected
+ }
+
+ }
+
+ public Path touch(RoleHistoryWriter historyWriter, long time){
+ Path path = historyWriter.createHistoryFilename(historyPath, time);
+ FSDataOutputStream out = fs.create(path);
+ out.close()
+ return path
+ }
+
+ @Test
+ public void testSkipEmptyFileOnRead() throws Throwable {
+ describe "verify that empty histories are skipped on read; old histories purged"
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ roleHistory.onStart(fs, historyPath)
+ time = 0
+ Path oldhistory = roleHistory.saveHistory(time++)
+
+ String addr = "localhost"
+ NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
+ NodeEntry ne1 = instance.getOrCreate(0)
+ ne1.lastUsed = 0xf00d
+
+ Path goodhistory = roleHistory.saveHistory(time++)
+
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ Path touched = touch(historyWriter, time++)
+
+ RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+ assert rh2.onStart(fs, historyPath)
+ NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
+ assert ni2 != null
+
+ //and assert the older file got purged
+ assert !fs.exists(oldhistory)
+ assert fs.exists(goodhistory)
+ assert fs.exists(touched )
+ }
+
+ @Test
+ public void testSkipBrokenFileOnRead() throws Throwable {
+ describe "verify that empty histories are skipped on read; old histories purged"
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ roleHistory.onStart(fs, historyPath)
+ time = 0
+ Path oldhistory = roleHistory.saveHistory(time++)
+
+ String addr = "localhost"
+ NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
+ NodeEntry ne1 = instance.getOrCreate(0)
+ ne1.lastUsed = 0xf00d
+
+ Path goodhistory = roleHistory.saveHistory(time++)
+
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ Path badfile = historyWriter.createHistoryFilename(historyPath, time++)
+ FSDataOutputStream out = fs.create(badfile)
+ out.writeBytes("{broken:true}")
+ out.close()
+
+ RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+ describe("IGNORE STACK TRACE BELOW")
+
+ assert rh2.onStart(fs, historyPath)
+
+ describe( "IGNORE STACK TRACE ABOVE")
+ NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
+ assert ni2 != null
+
+ //and assert the older file got purged
+ assert !fs.exists(oldhistory)
+ assert fs.exists(goodhistory)
+ assert fs.exists(badfile )
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8cecce18/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy
new file mode 100644
index 0000000..a0663e8
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.Path
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.state.NodeEntry
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.RoleHistory
+import org.apache.slider.server.avro.RoleHistoryWriter
+import org.junit.Test
+
+import java.util.regex.Matcher
+import java.util.regex.Pattern
+
+@Slf4j
+class TestRoleHistoryRWOrdering extends BaseMockAppStateTest {
+
+ def paths = pathlist(
+ [
+ "hdfs://localhost/history-0406c.json",
+ "hdfs://localhost/history-5fffa.json",
+ "hdfs://localhost/history-0001a.json",
+ "hdfs://localhost/history-0001f.json",
+ ]
+ )
+ Path h_0406c = paths[0]
+ Path h_5fffa = paths[1]
+ Path h_0001a = paths[3]
+
+
+ List<Path> pathlist(List<String> pathnames) {
+ def result = []
+ pathnames.each { result << new Path(new URI(it as String)) }
+ result
+ }
+
+ @Override
+ String getTestName() {
+ return "TestHistoryRWOrdering"
+ }
+
+
+ /**
+ * This tests regexp pattern matching. It uses the current time so isn't
+ * repeatable -but it does test a wider range of values in the process
+ * @throws Throwable
+ */
+ @Test
+ public void testPatternRoundTrip() throws Throwable {
+ describe "test pattern matching of names"
+ long value=System.currentTimeMillis()
+ String name = String.format(SliderKeys.HISTORY_FILENAME_CREATION_PATTERN,value)
+ String matchpattern = SliderKeys.HISTORY_FILENAME_MATCH_PATTERN
+ Pattern pattern = Pattern.compile(matchpattern)
+ Matcher matcher = pattern.matcher(name);
+ if (!matcher.find()) {
+ throw new Exception("No match for pattern $matchpattern in $name")
+ }
+ }
+
+
+ @Test
+ public void testWriteSequenceReadData() throws Throwable {
+ describe "test that if multiple entries are written, the newest is picked up"
+ long time = System.currentTimeMillis();
+
+ RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ assert !roleHistory.onStart(fs, historyPath)
+ String addr = "localhost"
+ NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
+ NodeEntry ne1 = instance.getOrCreate(0)
+ ne1.lastUsed = 0xf00d
+
+ Path history1 = roleHistory.saveHistory(time++)
+ Path history2 = roleHistory.saveHistory(time++)
+ Path history3 = roleHistory.saveHistory(time++)
+
+ //inject a later file with a different name
+ sliderFileSystem.cat(new Path(historyPath, "file.json"), true, "hello, world")
+
+
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+
+ List<Path> entries = historyWriter.findAllHistoryEntries(
+ fs,
+ historyPath,
+ false)
+ assert entries.size() == 3
+ assert entries[0] == history3
+ assert entries[1] == history2
+ assert entries[2] == history1
+ }
+
+ @Test
+ public void testPathStructure() throws Throwable {
+ assert h_5fffa.getName() == "history-5fffa.json"
+ }
+
+ @Test
+ public void testPathnameComparator() throws Throwable {
+
+ def newerName = new RoleHistoryWriter.NewerFilesFirst()
+
+ log.info("$h_5fffa name is ${h_5fffa.getName()}")
+ log.info("$h_0406c name is ${h_0406c.getName()}")
+ assert newerName.compare(h_5fffa, h_5fffa) == 0
+ assert newerName.compare(h_5fffa, h_0406c) < 0
+ assert newerName.compare(h_5fffa, h_0001a) < 0
+ assert newerName.compare(h_0001a, h_5fffa) > 0
+
+ }
+
+ @Test
+ public void testPathSort() throws Throwable {
+ def paths2 = new ArrayList(paths)
+ RoleHistoryWriter.sortHistoryPaths(paths2)
+ assertListEquals(paths2,
+ [
+ paths[1],
+ paths[0],
+ paths[3],
+ paths[2]
+ ])
+ }
+}
[41/50] [abbrv] git commit: SLIDER-302 Add a provider startup
event...need to think through how best to use this
Posted by st...@apache.org.
SLIDER-302 Add a provider startup event...need to think through how best to use this
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b7b9a158
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b7b9a158
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b7b9a158
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: b7b9a158b7f7da3c4deb5009eb48c3a8a301838b
Parents: 8f777f3
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 8 19:15:58 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 8 19:16:30 2014 +0100
----------------------------------------------------------------------
.../actions/ProviderStartupCompleted.java | 37 ++++++++++++++++++++
1 file changed, 37 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b7b9a158/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
new file mode 100644
index 0000000..293492f
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderStartupCompleted.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.slider.server.appmaster.SliderAppMaster;
+
+public class ProviderStartupCompleted extends AsyncAction {
+
+ public ProviderStartupCompleted() {
+ super("ProviderStartupCompleted");
+ }
+
+ public ProviderStartupCompleted(int delayMillis) {
+ super("ProviderStartupCompleted", delayMillis);
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster) throws Exception {
+ appMaster.eventCallbackEvent(null);
+ }
+}
[28/50] [abbrv] git commit: SLIDER-298. Add an application package
template
Posted by st...@apache.org.
SLIDER-298. Add an application package template
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/609fcd32
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/609fcd32
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/609fcd32
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 609fcd327c4ee814e4f33b0357bdef9e77caf0fc
Parents: ec76068
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Aug 6 13:55:14 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Aug 6 13:55:14 2014 -0700
----------------------------------------------------------------------
app-packages/app-pkg-template/README.txt | 34 +++++++++++
app-packages/app-pkg-template/appConfig.json | 21 +++++++
app-packages/app-pkg-template/metainfo.xml | 57 +++++++++++++++++++
app-packages/app-pkg-template/myapp-1.0.0.zip | Bin 0 -> 5455 bytes
.../package/files/myapp-1.0.0.tar.REPLACE | 16 ++++++
.../package/scripts/myapp_master.py | 46 +++++++++++++++
.../app-pkg-template/package/scripts/params.py | 30 ++++++++++
app-packages/app-pkg-template/resources.json | 16 ++++++
8 files changed, 220 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/609fcd32/app-packages/app-pkg-template/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/app-pkg-template/README.txt b/app-packages/app-pkg-template/README.txt
new file mode 100644
index 0000000..00dfdbc
--- /dev/null
+++ b/app-packages/app-pkg-template/README.txt
@@ -0,0 +1,34 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+How to create a Slider app package for myapp?
+
+To create the app package you will need the application tarball copied to a specific location.
+
+E.g.
+ cp ~/Downloads/myapp-1.0.0.tar package/files/
+
+Create a zip package at the root of the package (<slider enlistment>/app-packages/myapp/)
+ zip -r myapp-1.0.0.zip .
+
+Verify the content using
+ zip -Tv myapp-1.0.0.zip
+
+While appConfig.json and resources.json are not required for the package they work
+well as the default configuration for Slider apps. So its advisable that when you
+create an application package for Slider, include sample/default resources.json and
+appConfig.json for a one-node Yarn cluster.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/609fcd32/app-packages/app-pkg-template/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/app-pkg-template/appConfig.json b/app-packages/app-pkg-template/appConfig.json
new file mode 100644
index 0000000..a6f61f9
--- /dev/null
+++ b/app-packages/app-pkg-template/appConfig.json
@@ -0,0 +1,21 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "application.def": "package/myapp-1.0.0.zip",
+ "java_home": "/usr/jdk64/jdk1.7.0_45",
+
+ "site.global.app_user": "yarn",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/install/myapp-1.0.0",
+
+ "site.global.listen_port": "${MYAPP_COMPONENT.ALLOCATED_PORT}"
+ },
+ "components": {
+ "slider-appmaster": {
+ "jvm.heapsize": "256M"
+ },
+ "MYAPP_COMPONENT": {
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/609fcd32/app-packages/app-pkg-template/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/app-pkg-template/metainfo.xml b/app-packages/app-pkg-template/metainfo.xml
new file mode 100644
index 0000000..c6e1485
--- /dev/null
+++ b/app-packages/app-pkg-template/metainfo.xml
@@ -0,0 +1,57 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <application>
+ <name>MyApp</name>
+ <comment>MyApp is an app that can run on YARN.</comment>
+ <version>1.0.0</version>
+ <exportedConfigs>None</exportedConfigs>
+
+ <components>
+ <component>
+ <name>MYAPP_COMPONENT</name>
+ <category>MASTER</category>
+ <exports>
+ <export>
+ <name>host_port</name>
+ <value>${THIS_HOST}:${site.global.listen_port}</value>
+ </export>
+ </exports>
+ <commandScript>
+ <script>scripts/myapp_component.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osType>any</osType>
+ <packages>
+ <package>
+ <type>tarball</type>
+ <name>files/myapp-1.0.0.tar</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ </application>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/609fcd32/app-packages/app-pkg-template/myapp-1.0.0.zip
----------------------------------------------------------------------
diff --git a/app-packages/app-pkg-template/myapp-1.0.0.zip b/app-packages/app-pkg-template/myapp-1.0.0.zip
new file mode 100644
index 0000000..80dbbb7
Binary files /dev/null and b/app-packages/app-pkg-template/myapp-1.0.0.zip differ
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/609fcd32/app-packages/app-pkg-template/package/files/myapp-1.0.0.tar.REPLACE
----------------------------------------------------------------------
diff --git a/app-packages/app-pkg-template/package/files/myapp-1.0.0.tar.REPLACE b/app-packages/app-pkg-template/package/files/myapp-1.0.0.tar.REPLACE
new file mode 100644
index 0000000..2114587
--- /dev/null
+++ b/app-packages/app-pkg-template/package/files/myapp-1.0.0.tar.REPLACE
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+Replace this file with a tarball of the myapp version you want to package.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/609fcd32/app-packages/app-pkg-template/package/scripts/myapp_master.py
----------------------------------------------------------------------
diff --git a/app-packages/app-pkg-template/package/scripts/myapp_master.py b/app-packages/app-pkg-template/package/scripts/myapp_master.py
new file mode 100644
index 0000000..e6bc867
--- /dev/null
+++ b/app-packages/app-pkg-template/package/scripts/myapp_master.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+class MyApp_Component(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ def status(self, env):
+ import params
+ env.set_params(params)
+
+if __name__ == "__main__":
+ MyApp_Component().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/609fcd32/app-packages/app-pkg-template/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/app-pkg-template/package/scripts/params.py b/app-packages/app-pkg-template/package/scripts/params.py
new file mode 100644
index 0000000..e81bda0
--- /dev/null
+++ b/app-packages/app-pkg-template/package/scripts/params.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+# server configurations
+config = Script.get_config()
+
+app_root = config['configurations']['global']['app_root']
+java64_home = config['hostLevelParams']['java_home']
+app_user = config['configurations']['global']['app_user']
+port = config['configurations']['global']['listen_port']
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/609fcd32/app-packages/app-pkg-template/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/app-pkg-template/resources.json b/app-packages/app-pkg-template/resources.json
new file mode 100644
index 0000000..0e95879
--- /dev/null
+++ b/app-packages/app-pkg-template/resources.json
@@ -0,0 +1,16 @@
+{
+ "schema" : "http://example.org/specification/v2.0.0",
+ "metadata" : {
+ },
+ "global" : {
+ },
+ "components": {
+ "slider-appmaster": {
+ },
+ "MYAPP_COMPONENT": {
+ "yarn.role.priority": "1",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ }
+ }
+}
\ No newline at end of file
[02/50] [abbrv] git commit: SLIDER-270: flex down twice not working.
This isn't showing up in a (new) mock test
Posted by st...@apache.org.
SLIDER-270: flex down twice not working. This isn't showing up in a (new) mock test
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/28babb19
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/28babb19
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/28babb19
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 28babb1923bbfd23cc9bc0148d12497b9e848f29
Parents: 0b52d4b
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 5 16:05:03 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 5 16:05:03 2014 +0100
----------------------------------------------------------------------
.../server/appmaster/SliderAppMaster.java | 16 +--
.../slider/server/appmaster/state/AppState.java | 43 +++----
.../appstate/TestAppStateDynamicRoles.groovy | 4 +-
.../TestAppStateRebuildOnAMRestart.groovy | 4 +-
.../model/appstate/TestFlexDynamicRoles.groovy | 2 +
.../model/appstate/TestMockFlexing.groovy | 128 +++++++++++++++++++
.../model/appstate/TestMockRMOperations.groovy | 37 ------
.../model/mock/BaseMockAppStateTest.groovy | 57 ++++++++-
.../appmaster/model/mock/MockYarnEngine.groovy | 4 +-
.../web/rest/agent/TestAMAgentWebServices.java | 3 +-
.../management/TestAMManagementWebServices.java | 1 +
11 files changed, 219 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 7e0ae5e..1ad7fa7 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -671,12 +671,13 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//build the instance
appState.buildInstance(instanceDefinition,
- providerConf,
- providerRoles,
- fs.getFileSystem(),
- historyDir,
- liveContainers,
- appInformation);
+ serviceConf,
+ providerConf,
+ providerRoles,
+ fs.getFileSystem(),
+ historyDir,
+ liveContainers,
+ appInformation);
// add the AM to the list of nodes in the cluster
@@ -1055,8 +1056,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// non complete containers should not be here
assert (status.getState() == ContainerState.COMPLETE);
- AppState.NodeCompletionResult result = appState.onCompletedNode(
- getConfig(), status);
+ AppState.NodeCompletionResult result = appState.onCompletedNode(status);
if (result.containerFailed) {
RoleInstance ri = result.roleInstance;
log.error("Role instance {} failed ", ri);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index cc238ff..9f8a39d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -54,7 +54,6 @@ import org.apache.slider.core.exceptions.ErrorStrings;
import org.apache.slider.core.exceptions.NoSuchNodeException;
import org.apache.slider.core.exceptions.SliderInternalStateException;
import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
-import org.apache.slider.core.registry.docstore.PublishedConfigSet;
import org.apache.slider.providers.ProviderRole;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -250,6 +249,8 @@ public class AppState {
private long startTimeThreshold;
private int failureThreshold = 10;
+
+ private String logServerURL = "";
public AppState(AbstractRecordFactory recordFactory) {
this.recordFactory = recordFactory;
@@ -432,6 +433,7 @@ public class AppState {
/**
* Build up the application state
* @param instanceDefinition definition of the applicatin instance
+ * @param appmasterConfig
* @param publishedProviderConf any configuration info to be published by a provider
* @param providerRoles roles offered by a provider
* @param fs filesystem
@@ -440,12 +442,12 @@ public class AppState {
* @param applicationInfo
*/
public synchronized void buildInstance(AggregateConf instanceDefinition,
- Configuration publishedProviderConf,
- List<ProviderRole> providerRoles,
- FileSystem fs,
- Path historyDir,
- List<Container> liveContainers,
- Map<String, String> applicationInfo) throws
+ Configuration appmasterConfig, Configuration publishedProviderConf,
+ List<ProviderRole> providerRoles,
+ FileSystem fs,
+ Path historyDir,
+ List<Container> liveContainers,
+ Map<String, String> applicationInfo) throws
BadClusterStateException,
BadConfigException,
IOException {
@@ -514,6 +516,10 @@ public class AppState {
//rebuild any live containers
rebuildModelFromRestart(liveContainers);
+ // any am config options to pick up
+
+ logServerURL = appmasterConfig.get(YarnConfiguration.YARN_LOG_SERVER_URL,
+ "");
//mark as live
applicationLive = true;
}
@@ -1161,18 +1167,6 @@ public class AppState {
* @return NodeCompletionResult
*/
public synchronized NodeCompletionResult onCompletedNode(ContainerStatus status) {
- return onCompletedNode(null, status);
- }
-
- /**
- * handle completed node in the CD -move something from the live
- * server list to the completed server list
- * @param amConf YarnConfiguration
- * @param status the node that has just completed
- * @return NodeCompletionResult
- */
- public synchronized NodeCompletionResult onCompletedNode(Configuration amConf,
- ContainerStatus status) {
ContainerId containerId = status.getContainerId();
NodeCompletionResult result = new NodeCompletionResult();
RoleInstance roleInstance;
@@ -1219,11 +1213,8 @@ public class AppState {
}
String completedLogsUrl = null;
Container c = roleInstance.container;
- String url = null;
- if (amConf != null) {
- url = amConf.get(YarnConfiguration.YARN_LOG_SERVER_URL);
- }
- if (user != null && url != null) {
+ String url = logServerURL;
+ if (user != null && SliderUtils.isSet(url)) {
completedLogsUrl = url
+ "/" + c.getNodeId() + "/" + roleInstance.getContainerId() + "/ctx/" + user;
}
@@ -1255,12 +1246,12 @@ public class AppState {
completionOfUnknownContainerEvent.incrementAndGet();
}
}
-
+
if (result.surplusNode) {
//a surplus node
return result;
}
-
+
//record the complete node's details; this pulls it from the livenode set
//remove the node
ContainerId id = status.getContainerId();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy
index 6e387d8..6d70885 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy
@@ -72,11 +72,13 @@ class TestAppStateDynamicRoles extends BaseMockAppStateTest
appState.buildInstance(
instance,
+ new Configuration(),
new Configuration(false),
factory.ROLES,
fs,
historyPath,
- null, null)
+ null,
+ null)
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy
index 190e927..8d7ff4f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy
@@ -71,11 +71,13 @@ class TestAppStateRebuildOnAMRestart extends BaseMockAppStateTest
//and rebuild
appState.buildInstance(
factory.newInstanceDefinition(r0, r1, r2),
+ new Configuration(),
new Configuration(false),
factory.ROLES,
fs,
historyPath,
- containers, null)
+ containers,
+ null)
assert appState.getStartedCountainerCount() == clusterSize
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy
index 1693365..911880a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy
@@ -73,6 +73,7 @@ class TestFlexDynamicRoles extends BaseMockAppStateTest
appState.buildInstance(instance,
+ new Configuration(),
new Configuration(false),
factory.ROLES,
fs,
@@ -176,6 +177,7 @@ class TestFlexDynamicRoles extends BaseMockAppStateTest
appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
appState.buildInstance(
factory.newInstanceDefinition(0, 0, 0),
+ new Configuration(),
new Configuration(false),
factory.ROLES,
fs,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockFlexing.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockFlexing.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockFlexing.groovy
new file mode 100644
index 0000000..d2dc028
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockFlexing.groovy
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.state.AbstractRMOperation
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.ContainerAssignment
+import org.apache.slider.server.appmaster.state.ContainerReleaseOperation
+import org.apache.slider.server.appmaster.state.ContainerRequestOperation
+import org.apache.slider.server.appmaster.state.RMOperationHandler
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.junit.Test
+
+import static org.apache.slider.server.appmaster.state.ContainerPriority.buildPriority
+import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole
+
+@Slf4j
+class TestMockFlexing extends BaseMockAppStateTest implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestMockFlexing"
+ }
+
+
+
+ @Test
+ public void testFlexDuringLaunchPhase() throws Throwable {
+ role0Status.desired = 1
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ List<Container> allocations = engine.execute(ops)
+ List<ContainerAssignment> assignments = [];
+ List<AbstractRMOperation> releases = []
+ appState.onContainersAllocated(allocations, assignments, releases)
+ assert assignments.size() == 1
+ ContainerAssignment assigned = assignments[0]
+ Container target = assigned.container
+ RoleInstance ri = roleInstance(assigned)
+
+ ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.empty
+
+ //now this is the start point.
+ appState.containerStartSubmitted(target, ri);
+
+ ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.empty
+
+ RoleInstance ri2 = appState.innerOnNodeManagerContainerStarted(target.id)
+ }
+
+ @Test
+ public void testFlexBeforeAllocationPhase() throws Throwable {
+ role0Status.desired = 1
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ assert !ops.empty
+ List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes()
+ assert ops2.empty
+ }
+
+
+ @Test
+ public void testFlexDownTwice() throws Throwable {
+ int r0 = 6
+ int r1 = 0
+ int r2 = 0
+ role0Status.desired = r0
+ role1Status.desired = r1
+ role2Status.desired = r2
+ List<RoleInstance> instances = createAndStartNodes()
+
+ int clusterSize = r0 + r1 + r2
+ assert instances.size() == clusterSize
+ log.info("shrinking cluster")
+ r0 = 4
+ role0Status.desired = r0
+ List<AppState.NodeCompletionResult> completionResults = []
+ instances = createStartAndStopNodes(completionResults)
+ assert instances.size() == 0
+ // assert two nodes were released
+ assert completionResults.size() == 2
+
+ // no-op review
+ completionResults = []
+ instances = createStartAndStopNodes(completionResults)
+ assert instances.size() == 0
+ // assert two nodes were released
+ assert completionResults.size() == 0
+
+
+ // now shrink again
+ role0Status.desired = r0 = 1
+ completionResults = []
+ instances = createStartAndStopNodes(completionResults)
+ assert instances.size() == 0
+ // assert two nodes were released
+ assert completionResults.size() == 3
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy
index 168ac9f..e27827b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy
@@ -173,41 +173,4 @@ class TestMockRMOperations extends BaseMockAppStateTest implements MockRoles {
assert ri3 == null
}
- @Test
- public void testFlexDuringLaunchPhase() throws Throwable {
- role0Status.desired = 1
-
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- List<Container> allocations = engine.execute(
- ops)
- List<ContainerAssignment> assignments = [];
- List<AbstractRMOperation> releases = []
- appState.onContainersAllocated(allocations, assignments, releases)
- assert assignments.size() == 1
- ContainerAssignment assigned = assignments[0]
- Container target = assigned.container
- RoleInstance ri = roleInstance(assigned)
-
- ops = appState.reviewRequestAndReleaseNodes()
- assert ops.empty
-
- //now this is the start point.
- appState.containerStartSubmitted(target, ri);
-
- ops = appState.reviewRequestAndReleaseNodes()
- assert ops.empty
-
- RoleInstance ri2 = appState.innerOnNodeManagerContainerStarted(target.id)
- }
-
- @Test
- public void testFlexBeforeAllocationPhase() throws Throwable {
- role0Status.desired = 1
-
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- assert !ops.empty
- List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes()
- assert ops2.empty
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
index 628c729..4a34e7f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -82,6 +82,7 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
appState.buildInstance(
factory.newInstanceDefinition(0, 0, 0),
+ new Configuration(),
new Configuration(false),
factory.ROLES,
fs,
@@ -166,20 +167,72 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
* @return a list of roles
*/
protected List<RoleInstance> createAndStartNodes() {
- List<RoleInstance> instances = createAndSubmitNodes()
+ return createStartAndStopNodes([])
+ }
+
+ /**
+ * Create, Start and stop nodes
+ * @param completionResults List filled in with the status on all completed nodes
+ * @return the nodes
+ */
+ public List<RoleInstance> createStartAndStopNodes(
+ List<AppState.NodeCompletionResult> completionResults) {
+ List<ContainerId> released = []
+ List<RoleInstance> instances = createAndSubmitNodes(released)
for (RoleInstance instance : instances) {
assert appState.onNodeManagerContainerStarted(instance.containerId)
}
+ releaseContainers(completionResults,
+ released,
+ ContainerState.COMPLETE,
+ "released",
+ 0
+ )
return instances
}
/**
+ * Release a list of containers, updating the completion results
+ * @param completionResults
+ * @param containerIds
+ * @param containerState
+ * @param exitText
+ * @param containerExitCode
+ * @return
+ */
+ public def releaseContainers(
+ List<AppState.NodeCompletionResult> completionResults,
+ List<ContainerId> containerIds,
+ ContainerState containerState,
+ String exitText,
+ int containerExitCode) {
+ containerIds.each { ContainerId id ->
+ ContainerStatus status = ContainerStatus.newInstance(id,
+ containerState,
+ exitText,
+ containerExitCode)
+ completionResults << appState.onCompletedNode(status)
+
+ }
+ }
+
+ /**
* Create nodes and submit them
* @return a list of roles
*/
public List<RoleInstance> createAndSubmitNodes() {
+ return createAndSubmitNodes([])
+ }
+
+ /**
+ * Create nodes and submit them
+ * @param released a list that is built up of all released nodes
+ * @return a list of roles allocated
+ */
+ public List<RoleInstance> createAndSubmitNodes(
+ List<ContainerId> released) {
List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- List<Container> allocatedContainers = engine.execute(ops)
+ List<Container> allocatedContainers = engine.execute(ops, released)
List<ContainerAssignment> assignments = [];
List<AbstractRMOperation> operations = []
appState.onContainersAllocated(allocatedContainers, assignments, operations)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
index 7ebdf52..763f653 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
@@ -86,9 +86,7 @@ class MockYarnEngine {
* @param ops
* @return
*/
- List<Container> execute(
- List<AbstractRMOperation> ops
- ) {
+ List<Container> execute(List<AbstractRMOperation> ops) {
return execute(ops, [])
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
index 17fbe2b..96a4510 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
@@ -24,7 +24,6 @@ import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.api.json.JSONConfiguration;
-import com.sun.jersey.test.framework.JerseyTest;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -43,7 +42,6 @@ import org.apache.slider.server.appmaster.web.WebAppApiImpl;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
import org.apache.slider.server.services.security.CertificateManager;
import org.apache.slider.server.services.security.SecurityUtils;
-import org.apache.slider.test.SliderTestBase;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -125,6 +123,7 @@ public class TestAMAgentWebServices {
appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES);
appState.buildInstance(
factory.newInstanceDefinition(0, 0, 0),
+ new Configuration(),
new Configuration(false),
factory.ROLES,
fs,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
index 134f0bc..96e196f 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
@@ -170,6 +170,7 @@ public class TestAMManagementWebServices extends JerseyTest {
appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES);
appState.buildInstance(
factory.newInstanceDefinition(0, 0, 0),
+ new Configuration(),
new Configuration(false),
factory.ROLES,
fs,
[07/50] [abbrv] git commit: SLIDER-284 switch branch-2 Hadoop version
to 2.6.0-SNAPSHOT
Posted by st...@apache.org.
SLIDER-284 switch branch-2 Hadoop version to 2.6.0-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/040ef798
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/040ef798
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/040ef798
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 040ef79802d0f9c3a77a4b64a5fbea0d05f67fb0
Parents: 268253c
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 5 21:08:38 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 5 21:08:38 2014 +0100
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/040ef798/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7cca66c..253d40f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1264,7 +1264,7 @@
<!-- hadoop branch-2 builds -->
<id>branch-2</id>
<properties>
- <hadoop.version>2.5.0-SNAPSHOT</hadoop.version>
+ <hadoop.version>2.6.0-SNAPSHOT</hadoop.version>
</properties>
</profile>
[25/50] [abbrv] git commit: SLIDER-296 providers now get notified of
container create/release requests
Posted by st...@apache.org.
SLIDER-296 providers now get notified of container create/release requests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0646c234
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0646c234
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0646c234
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 0646c234ff20a98c4620f2dbd69a4a36414ca27d
Parents: 868f7e2
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 6 19:24:55 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 6 19:24:55 2014 +0100
----------------------------------------------------------------------
.../providers/AbstractProviderService.java | 11 +++++
.../slider/providers/ProviderService.java | 8 +--
.../appmaster/AsyncRMOperationHandler.java | 51 --------------------
.../server/appmaster/SliderAppMaster.java | 15 ++++--
.../operations/AbstractRMOperation.java | 29 +++++++++++
.../operations/AsyncRMOperationHandler.java | 51 ++++++++++++++++++++
.../operations/ContainerReleaseOperation.java | 44 +++++++++++++++++
.../operations/ContainerRequestOperation.java | 44 +++++++++++++++++
.../ProviderNotifyingOperationHandler.java | 43 +++++++++++++++++
.../operations/RMOperationHandler.java | 35 ++++++++++++++
.../operations/RMOperationHandlerActions.java | 28 +++++++++++
.../appmaster/state/AbstractRMOperation.java | 31 ------------
.../slider/server/appmaster/state/AppState.java | 5 +-
.../state/ContainerReleaseOperation.java | 44 -----------------
.../state/ContainerRequestOperation.java | 44 -----------------
.../appmaster/state/RMOperationHandler.java | 43 -----------------
.../TestMockAppStateDynamicRoles.groovy | 2 +-
.../appstate/TestMockAppStateFlexing.groovy | 12 +----
.../TestMockAppStateRMOperations.groovy | 4 ++
.../TestMockAppStateRebuildOnAMRestart.groovy | 1 +
.../TestMockAppStateRolePlacement.groovy | 3 ++
.../appstate/TestMockAppStateRoleRelease.groovy | 2 +-
.../TestMockContainerResourceAllocations.groovy | 4 +-
.../model/mock/BaseMockAppStateTest.groovy | 1 +
.../model/mock/MockProviderService.groovy | 11 +++++
.../model/mock/MockRMOperationHandler.groovy | 8 +--
.../appmaster/model/mock/MockYarnEngine.groovy | 6 +--
27 files changed, 337 insertions(+), 243 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index 7483a8d..158640d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -21,6 +21,7 @@ package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.ConfigHelper;
@@ -337,4 +338,14 @@ public abstract class AbstractProviderService
public ContainerReleaseSelector createContainerReleaseSelector() {
return new MostRecentContainerReleaseSelector();
}
+
+ @Override
+ public void releaseAssignedContainer(ContainerId containerId) {
+ // no-op
+ }
+
+ @Override
+ public void addContainerRequest(AMRMClient.ContainerRequest req) {
+ // no-op
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index 4278b09..c0a75c1 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -34,18 +34,20 @@ import org.apache.slider.core.main.ExitCodeProvider;
import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.server.appmaster.AMViewForProviders;
import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
+import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
import org.apache.slider.server.services.registry.RegistryViewForProviders;
import java.io.File;
import java.io.IOException;
-import java.net.MalformedURLException;
import java.net.URL;
import java.util.Map;
-public interface ProviderService extends ProviderCore, Service,
- ExitCodeProvider {
+public interface ProviderService extends ProviderCore,
+ Service,
+ RMOperationHandlerActions,
+ ExitCodeProvider {
/**
* Set up the entire container launch context
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/main/java/org/apache/slider/server/appmaster/AsyncRMOperationHandler.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/AsyncRMOperationHandler.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/AsyncRMOperationHandler.java
deleted file mode 100644
index 171c021..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/AsyncRMOperationHandler.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
-import org.apache.slider.server.appmaster.state.RMOperationHandler;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Hands off RM operations to the Resource Manager
- */
-public class AsyncRMOperationHandler extends RMOperationHandler {
- protected static final Logger log =
- LoggerFactory.getLogger(AsyncRMOperationHandler.class);
- private final AMRMClientAsync client;
-
- public AsyncRMOperationHandler(AMRMClientAsync client) {
- this.client = client;
- }
-
- @Override
- public void releaseAssignedContainer(ContainerId containerId) {
- log.debug("Releasing container {}", containerId);
-
- client.releaseAssignedContainer(containerId);
- }
-
- @Override
- public void addContainerRequest(AMRMClient.ContainerRequest req) {
- client.addContainerRequest(req);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index dbf08e1..e84103c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -94,15 +94,17 @@ import org.apache.slider.providers.ProviderService;
import org.apache.slider.providers.SliderProviderFactory;
import org.apache.slider.providers.slideram.SliderAMClientProvider;
import org.apache.slider.providers.slideram.SliderAMProviderService;
+import org.apache.slider.server.appmaster.operations.AsyncRMOperationHandler;
+import org.apache.slider.server.appmaster.operations.ProviderNotifyingOperationHandler;
import org.apache.slider.server.appmaster.rpc.RpcBinder;
import org.apache.slider.server.appmaster.rpc.SliderAMPolicyProvider;
import org.apache.slider.server.appmaster.rpc.SliderClusterProtocolPBImpl;
-import org.apache.slider.server.appmaster.state.AbstractRMOperation;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
import org.apache.slider.server.appmaster.state.AppState;
import org.apache.slider.server.appmaster.state.ContainerAssignment;
-import org.apache.slider.server.appmaster.state.ContainerReleaseOperation;
+import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
import org.apache.slider.server.appmaster.state.ProviderAppState;
-import org.apache.slider.server.appmaster.state.RMOperationHandler;
+import org.apache.slider.server.appmaster.operations.RMOperationHandler;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.RoleStatus;
import org.apache.slider.server.appmaster.state.SimpleReleaseSelector;
@@ -191,6 +193,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
private RMOperationHandler rmOperationHandler;
+
+ private RMOperationHandler providerRMOperationHandler;
/** Handle to communicate with the Node Manager*/
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
@@ -495,6 +499,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
providerService = factory.createServerProvider();
// init the provider BUT DO NOT START IT YET
initAndAddService(providerService);
+ providerRMOperationHandler =
+ new ProviderNotifyingOperationHandler(providerService);
+
// create a slider AM provider
sliderAMProvider = new SliderAMProviderService();
initAndAddService(sliderAMProvider);
@@ -1109,6 +1116,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
try {
List<AbstractRMOperation> allOperations = appState.reviewRequestAndReleaseNodes();
+ // tell the provider
+ providerRMOperationHandler.execute(allOperations);
//now apply the operations
rmOperationHandler.execute(allOperations);
return !allOperations.isEmpty();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AbstractRMOperation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AbstractRMOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AbstractRMOperation.java
new file mode 100644
index 0000000..2c55215
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AbstractRMOperation.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.operations;
+
+public abstract class AbstractRMOperation {
+
+ /**
+ * Execute the operation
+ * @param asyncRMClient client
+ */
+ public abstract void execute(RMOperationHandler handler);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java
new file mode 100644
index 0000000..f7a95a7
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.operations;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Hands off RM operations to the Resource Manager
+ */
+public class AsyncRMOperationHandler extends RMOperationHandler {
+ protected static final Logger log =
+ LoggerFactory.getLogger(AsyncRMOperationHandler.class);
+ private final AMRMClientAsync client;
+
+ public AsyncRMOperationHandler(AMRMClientAsync client) {
+ this.client = client;
+ }
+
+ @Override
+ public void releaseAssignedContainer(ContainerId containerId) {
+ log.debug("Releasing container {}", containerId);
+
+ client.releaseAssignedContainer(containerId);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void addContainerRequest(AMRMClient.ContainerRequest req) {
+ client.addContainerRequest(req);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerReleaseOperation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerReleaseOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerReleaseOperation.java
new file mode 100644
index 0000000..3d2016b
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerReleaseOperation.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.operations;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+
+public class ContainerReleaseOperation extends AbstractRMOperation {
+
+ private final ContainerId containerId;
+
+ public ContainerReleaseOperation(ContainerId containerId) {
+ this.containerId = containerId;
+ }
+
+ public ContainerId getContainerId() {
+ return containerId;
+ }
+
+ @Override
+ public void execute(RMOperationHandler handler) {
+ handler.releaseAssignedContainer(containerId);
+ }
+
+ @Override
+ public String toString() {
+ return "release container " + containerId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java
new file mode 100644
index 0000000..711bb98
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.operations;
+
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+
+public class ContainerRequestOperation extends AbstractRMOperation {
+
+ private final AMRMClient.ContainerRequest request;
+
+ public ContainerRequestOperation(AMRMClient.ContainerRequest request) {
+ this.request = request;
+ }
+
+ public AMRMClient.ContainerRequest getRequest() {
+ return request;
+ }
+
+ @Override
+ public void execute(RMOperationHandler handler) {
+ handler.addContainerRequest(request);
+ }
+
+ @Override
+ public String toString() {
+ return "request container ";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java
new file mode 100644
index 0000000..a24d9e5
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ProviderNotifyingOperationHandler.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.operations;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.slider.providers.ProviderService;
+
+public class ProviderNotifyingOperationHandler extends RMOperationHandler {
+
+ final ProviderService providerService;
+
+ public ProviderNotifyingOperationHandler(ProviderService providerService) {
+ this.providerService = providerService;
+ }
+
+ @Override
+ public void releaseAssignedContainer(ContainerId containerId) {
+ providerService.releaseAssignedContainer(containerId);
+ }
+
+ @Override
+ public void addContainerRequest(AMRMClient.ContainerRequest req) {
+ providerService.addContainerRequest(req);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandler.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandler.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandler.java
new file mode 100644
index 0000000..2b6e9e2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandler.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.operations;
+
+import java.util.List;
+
+public abstract class RMOperationHandler implements RMOperationHandlerActions {
+
+
+ /**
+ * Execute an entire list of operations
+ * @param operations ops
+ */
+ public void execute(List<AbstractRMOperation> operations) {
+ for (AbstractRMOperation operation : operations) {
+ operation.execute(this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java
new file mode 100644
index 0000000..6659cc9
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.operations;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+
+public interface RMOperationHandlerActions {
+ void releaseAssignedContainer(ContainerId containerId);
+
+ void addContainerRequest(AMRMClient.ContainerRequest req);
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractRMOperation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractRMOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractRMOperation.java
deleted file mode 100644
index e3e595f..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractRMOperation.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-public class AbstractRMOperation {
-
- /**
- * Execute the operation
- * @param asyncRMClient client
- */
- public void execute(RMOperationHandler handler) {
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index c20c924..cbe7535 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -18,7 +18,6 @@
package org.apache.slider.server.appmaster.state;
-import com.beust.jcommander.internal.Lists;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
@@ -57,6 +56,9 @@ import org.apache.slider.core.exceptions.NoSuchNodeException;
import org.apache.slider.core.exceptions.SliderInternalStateException;
import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
+import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,7 +67,6 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseOperation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseOperation.java
deleted file mode 100644
index 8e73f19..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseOperation.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-
-public class ContainerReleaseOperation extends AbstractRMOperation {
-
- private final ContainerId containerId;
-
- public ContainerReleaseOperation(ContainerId containerId) {
- this.containerId = containerId;
- }
-
- public ContainerId getContainerId() {
- return containerId;
- }
-
- @Override
- public void execute(RMOperationHandler handler) {
- handler.releaseAssignedContainer(containerId);
- }
-
- @Override
- public String toString() {
- return "release container " + containerId;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerRequestOperation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerRequestOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerRequestOperation.java
deleted file mode 100644
index 25c3d60..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerRequestOperation.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-
-public class ContainerRequestOperation extends AbstractRMOperation {
-
- private final AMRMClient.ContainerRequest request;
-
- public ContainerRequestOperation(AMRMClient.ContainerRequest request) {
- this.request = request;
- }
-
- public AMRMClient.ContainerRequest getRequest() {
- return request;
- }
-
- @Override
- public void execute(RMOperationHandler handler) {
- handler.addContainerRequest(request);
- }
-
- @Override
- public String toString() {
- return "request container ";
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RMOperationHandler.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RMOperationHandler.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RMOperationHandler.java
deleted file mode 100644
index 4106b16..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RMOperationHandler.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-
-import java.util.List;
-
-public abstract class RMOperationHandler {
-
-
- public abstract void releaseAssignedContainer(ContainerId containerId);
-
- public abstract void addContainerRequest(AMRMClient.ContainerRequest req);
-
-
- /**
- * Execute an entire list of operations
- * @param operations ops
- */
- public void execute(List<AbstractRMOperation> operations) {
- for (AbstractRMOperation operation : operations) {
- operation.execute(this);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
index a204bea..1b01f76 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
@@ -26,7 +26,7 @@ import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
-import org.apache.slider.server.appmaster.state.AbstractRMOperation
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.state.AppState
import org.apache.slider.server.appmaster.state.RoleInstance
import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
index 6be158e..80937dc 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
@@ -20,24 +20,14 @@ package org.apache.slider.server.appmaster.model.appstate
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
-import org.apache.hadoop.yarn.api.records.ContainerId
-import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler
import org.apache.slider.server.appmaster.model.mock.MockRoles
-import org.apache.slider.server.appmaster.state.AbstractRMOperation
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.state.AppState
import org.apache.slider.server.appmaster.state.ContainerAssignment
-import org.apache.slider.server.appmaster.state.ContainerReleaseOperation
-import org.apache.slider.server.appmaster.state.ContainerRequestOperation
-import org.apache.slider.server.appmaster.state.RMOperationHandler
import org.apache.slider.server.appmaster.state.RoleInstance
import org.junit.Test
-import static org.apache.slider.server.appmaster.state.ContainerPriority.buildPriority
-import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole
-
@Slf4j
class TestMockAppStateFlexing extends BaseMockAppStateTest implements MockRoles {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
index 6e18e39..6a5df5c 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
@@ -25,6 +25,10 @@ import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockFactory
import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler
import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation
+import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation
+import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
+import org.apache.slider.server.appmaster.operations.RMOperationHandler
import org.apache.slider.server.appmaster.state.*
import org.junit.Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
index 630be29..c142c77 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
@@ -26,6 +26,7 @@ import org.apache.slider.api.StatusKeys
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.state.*
import org.junit.Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
index b592f83..e7ddf79 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
@@ -24,6 +24,9 @@ import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation
+import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation
+import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
import org.apache.slider.server.appmaster.state.*
import org.junit.Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.groovy
index 495baf0..de4d493 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.groovy
@@ -24,7 +24,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
-import org.apache.slider.server.appmaster.state.AbstractRMOperation
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.state.RoleInstance
import org.junit.Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
index d018c12..0bf9a90 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
@@ -26,8 +26,8 @@ import org.apache.slider.core.conf.ConfTree
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockRoles
-import org.apache.slider.server.appmaster.state.AbstractRMOperation
-import org.apache.slider.server.appmaster.state.ContainerRequestOperation
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation
+import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
import org.junit.Test
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
index 987bc9b..504ca82 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.common.tools.SliderFileSystem
import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.main.LauncherExitCodes
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.state.*
import org.apache.slider.test.SliderTestBase
import org.junit.Before
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
index 0f7bb08..315537c 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
@@ -24,6 +24,7 @@ import org.apache.hadoop.service.LifecycleEvent
import org.apache.hadoop.service.ServiceStateChangeListener
import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.api.ClusterDescription
import org.apache.slider.common.tools.SliderFileSystem
import org.apache.slider.core.conf.AggregateConf
@@ -246,4 +247,14 @@ class MockProviderService implements ProviderService {
ContainerReleaseSelector createContainerReleaseSelector() {
return new MostRecentContainerReleaseSelector()
}
+
+ @Override
+ public void releaseAssignedContainer(ContainerId containerId) {
+ // no-op
+ }
+
+ @Override
+ public void addContainerRequest(AMRMClient.ContainerRequest req) {
+ // no-op
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
index 10a7708..8ccd463 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
@@ -21,10 +21,10 @@ package org.apache.slider.server.appmaster.model.mock
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.hadoop.yarn.client.api.AMRMClient
-import org.apache.slider.server.appmaster.state.AbstractRMOperation
-import org.apache.slider.server.appmaster.state.ContainerReleaseOperation
-import org.apache.slider.server.appmaster.state.ContainerRequestOperation
-import org.apache.slider.server.appmaster.state.RMOperationHandler
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation
+import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation
+import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
+import org.apache.slider.server.appmaster.operations.RMOperationHandler
@Slf4j
class MockRMOperationHandler extends RMOperationHandler {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0646c234/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
index b57a79a..f405188 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
@@ -25,9 +25,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationId
import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.hadoop.yarn.client.api.AMRMClient
-import org.apache.slider.server.appmaster.state.AbstractRMOperation
-import org.apache.slider.server.appmaster.state.ContainerReleaseOperation
-import org.apache.slider.server.appmaster.state.ContainerRequestOperation
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation
+import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation
+import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
/**
* This is an evolving engine to mock YARN operations
[23/50] [abbrv] git commit: SLIDER-296 removing use of
RoleHistory/nodemap target release selection logic - cutting methods out
entirely
Posted by st...@apache.org.
SLIDER-296 removing use of RoleHistory/nodemap target release selection logic - cutting methods out entirely
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3bb202f5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3bb202f5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3bb202f5
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 3bb202f54ce74541a7926b04555732326df7f5d0
Parents: 8cecce1
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 6 17:13:40 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 6 17:13:40 2014 +0100
----------------------------------------------------------------------
.../slider/server/appmaster/state/NodeMap.java | 54 -------
.../server/appmaster/state/RoleHistory.java | 16 --
.../TestRoleHistoryFindNodesForRelease.groovy | 146 -------------------
3 files changed, 216 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3bb202f5/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
index 32b1656..372a0a5 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
@@ -102,60 +102,6 @@ public class NodeMap extends HashMap<String, NodeInstance> {
}
return purged;
}
-
-
-
- /**
- * Find a list of node for release; algorithm may make its own
- * decisions on which to release.
- * @param role role index
- * @param count number of nodes to release
- * @return a possibly empty list of nodes.
- */
- public List<NodeInstance> findNodesForRelease(int role, int count) {
- List<NodeInstance> targets = new ArrayList<>(count);
- List<NodeInstance> active = listActiveNodes(role);
- List<NodeInstance> multiple = new ArrayList<>();
- int nodesRemaining = count;
- log.debug("searching for {} nodes with candidate set size {}",
- nodesRemaining, active.size());
- ListIterator<NodeInstance> it = active.listIterator();
- while (it.hasNext() && nodesRemaining > 0) {
- NodeInstance ni = it.next();
- int load = ni.getActiveRoleInstances(role);
- log.debug("Node {} load={}", ni, load);
- assert load != 0;
- if (load == 1) {
- // at the tail of the list, from here active[*] is a load=1 entry
- break;
- }
- // load is >1. Add an entry to the target list FOR EACH INSTANCE ABOVE 1
- for (int i = 0; i < (load - 1) && nodesRemaining > 0; i++) {
- nodesRemaining--;
- log.debug("Push {} #{}", ni, i);
- targets.add(ni);
- }
- // and add to the multiple list
- multiple.add(ni);
- // then pop it from the active list
- it.remove();
- }
- //here either the number is found or there is still some left.
-
- if (nodesRemaining > 0) {
- // leftovers. Append any of the multiple node entries to the tail of
- // the active list (so they get chosen last)
- active.addAll(multiple);
- // all the entries in the list have exactly one node
- // so ask for as many as are needed
- int ask = Math.min(nodesRemaining, active.size());
- log.debug("load=1 nodes to select={} multiples={} available={} ask={}",
- nodesRemaining, multiple.size(),active.size(), ask);
- targets.addAll(active.subList(0, ask));
- }
- return targets;
- }
-
/**
* Clone point
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3bb202f5/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index dbb104c..e9da081 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -517,19 +517,6 @@ public class RoleHistory {
return requestInstanceOnNode(node, role, resource);
}
-
- /**
- * Find a list of node for release; algorithm may make its own
- * decisions on which to release.
- * @param role role index
- * @param count number of nodes to release
- * @return a possibly empty list of nodes.
- */
- public synchronized List<NodeInstance> findNodesForRelease(int role, int count) {
- return nodemap.findNodesForRelease(role, count);
- }
-
-
/**
* Get the list of active nodes ... walks the node map so
* is O(nodes)
@@ -539,9 +526,6 @@ public class RoleHistory {
public synchronized List<NodeInstance> listActiveNodes(int role) {
return nodemap.listActiveNodes(role);
}
-
-
-
/**
* Get the node entry of a container
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3bb202f5/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForRelease.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForRelease.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForRelease.groovy
deleted file mode 100644
index 64db899..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForRelease.groovy
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.state.NodeInstance
-import org.apache.slider.server.appmaster.state.NodeMap
-import org.junit.Before
-import org.junit.Test
-
-@Slf4j
-@CompileStatic
-class TestRoleHistoryFindNodesForRelease extends BaseMockAppStateTest {
-
-
- @Override
- String getTestName() {
- return "TestFindNodesForRelease"
- }
- NodeInstance age1Active4 = nodeInstance(1, 4, 0, 0)
- NodeInstance age2Active2 = nodeInstance(2, 2, 0, 0)
- NodeInstance age3Active0 = nodeInstance(3, 0, 0, 0)
- NodeInstance age4Active1 = nodeInstance(4, 1, 0, 0)
- NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
-
- List<NodeInstance> nodes = [age2Active2, age4Active1, age1Active4, age3Active0]
- NodeMap nodeMap = new NodeMap(MockFactory.ROLE_COUNT);
-
-
- @Before
- public void setupNodeMap() {
- nodeMap.insert(nodes)
- }
-
- private void assertReleased(
- int count,
- List<NodeInstance> expected,
- int role = 0) {
- List<NodeInstance> released = nodeMap.findNodesForRelease(role, count)
- assertListEquals(released, expected)
- }
- private void assertReleased(
- List<NodeInstance> expected,
- int role = 0) {
- List<NodeInstance> released = nodeMap.findNodesForRelease(role, expected.size())
- assertListEquals(released, expected)
- }
-
- @Test
- public void testListActiveNodes() throws Throwable {
- assertListEquals(nodeMap.listActiveNodes(0),
- [age1Active4,age2Active2, age4Active1])
- }
-
- @Test
- public void testReleaseMinus1() throws Throwable {
- try {
- nodeMap.findNodesForRelease(0, -1)
- fail("Expected an exception")
- } catch (IllegalArgumentException e) {
- }
- }
- @Test
- public void testReleaseO() throws Throwable {
- assertReleased(0, [])
- }
-
- @Test
- public void testRelease1() throws Throwable {
- assertReleased(1, [age1Active4])
- }
-
- @Test
- public void testRelease2() throws Throwable {
- assertReleased(2, [age1Active4, age1Active4])
- }
-
- @Test
- public void testRelease3() throws Throwable {
- assertReleased(3, [age1Active4, age1Active4, age1Active4 ])
- }
-
- @Test
- public void testRelease4() throws Throwable {
- assertReleased(4, [age1Active4, age1Active4, age1Active4 , age2Active2])
- }
-
- @Test
- public void testRelease5() throws Throwable {
- assertReleased([age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1])
- }
-
- @Test
- public void testRelease6() throws Throwable {
- assertReleased(
- [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1, age1Active4])
- }
-
- @Test
- public void testRelease7() throws Throwable {
- assertReleased(
- [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1,
- age1Active4, age2Active2])
- }
-
- @Test
- public void testRelease8() throws Throwable {
- assertReleased(8,
- [age1Active4, age1Active4, age1Active4 , age2Active2, age4Active1,
- age1Active4, age2Active2])
- }
-
- @Test
- public void testPurgeInactiveTime3() throws Throwable {
- assert nodeMap.purgeUnusedEntries(3) == 0;
- }
-
- @Test
- public void testPurgeInactiveTime4() throws Throwable {
- assert nodeMap.purgeUnusedEntries(4) == 1;
- }
- @Test
- public void testPurgeInactiveTime5() throws Throwable {
- assert nodeMap.purgeUnusedEntries(5) == 1;
- }
-
-}
[30/50] [abbrv] git commit: SLIDER-286 new test for AM restart,
and some renaming of methods to make function clearer.
Posted by st...@apache.org.
SLIDER-286 new test for AM restart, and some renaming of methods to make function clearer.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1cbd890f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1cbd890f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1cbd890f
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 1cbd890fcc805fe2a7ccf9468201d4ecb4c6caeb
Parents: 373aed3
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 7 17:26:48 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 7 17:26:48 2014 +0100
----------------------------------------------------------------------
.../agent/AgentMiniClusterTestBase.groovy | 7 +-
.../agent/actions/TestActionExists.groovy | 7 +-
.../slider/agent/actions/TestActionList.groovy | 5 +-
.../agent/actions/TestActionStatus.groovy | 7 +-
.../agent/freezethaw/TestFreezeCommands.groovy | 6 +-
.../TestFreezeThawMasterlessAM.groovy | 5 +-
.../agent/standalone/TestBadAMHeap.groovy | 2 +-
.../agent/standalone/TestBadYarnQueue.groovy | 2 +-
.../standalone/TestDestroyMasterlessAM.groovy | 153 ------------------
.../standalone/TestDestroyStandaloneAM.groovy | 156 +++++++++++++++++++
.../standalone/TestKillMasterlessAM.groovy | 73 ---------
.../standalone/TestKillStandaloneAM.groovy | 73 +++++++++
.../standalone/TestRestartStandaloneAM.groovy | 96 ++++++++++++
.../standalone/TestStandaloneAgentAM.groovy | 6 +-
.../TestStandaloneBadClusterName.groovy | 2 +-
.../standalone/TestStandaloneRegistryAM.groovy | 2 +-
.../slider/test/YarnMiniClusterTestBase.groovy | 7 +-
17 files changed, 358 insertions(+), 251 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cbd890f/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
index 8a4e5d8..74f7a3f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
@@ -133,13 +133,12 @@ extends YarnZKMiniClusterTestBase {
* @param blockUntilRunning block until the AM is running
* @return launcher which will have executed the command.
*/
- public ServiceLauncher<SliderClient> createMasterlessAM(
+ public ServiceLauncher<SliderClient> createStandaloneAM(
String clustername,
- int size,
boolean deleteExistingData,
boolean blockUntilRunning) {
List<String> args = [];
- return createMasterlessAMWithArgs(
+ return createStandaloneAMWithArgs(
clustername,
args,
deleteExistingData,
@@ -155,7 +154,7 @@ extends YarnZKMiniClusterTestBase {
* @param blockUntilRunning block until the AM is running
* @return launcher which will have executed the command.
*/
- public ServiceLauncher<SliderClient> createMasterlessAMWithArgs(
+ public ServiceLauncher<SliderClient> createStandaloneAMWithArgs(
String clustername,
List<String> extraArgs,
boolean deleteExistingData,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cbd890f/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
index ee8baff..9bfeb8c 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionExists.groovy
@@ -29,8 +29,6 @@ import org.apache.slider.common.params.SliderActions
import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.test.SliderTestUtils
-import org.junit.Assert
import org.junit.Before
import org.junit.Test
@@ -72,7 +70,10 @@ class TestActionExists extends AgentMiniClusterTestBase {
public void testExistsLiveCluster() throws Throwable {
//launch the cluster
String clustername = createClusterName()
- ServiceLauncher<SliderClient> launcher = createMasterlessAM(clustername, 0, true, false)
+ ServiceLauncher<SliderClient> launcher = createStandaloneAM(
+ clustername,
+ true,
+ false)
SliderClient sliderClient = launcher.service
addToTeardown(launcher)
ApplicationReport report = waitForClusterLive(sliderClient)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cbd890f/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
index 7d62f37..b7196ac 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
@@ -88,7 +88,10 @@ class TestActionList extends AgentMiniClusterTestBase {
public void testListLiveCluster() throws Throwable {
//launch the cluster
String clustername = createClusterName()
- ServiceLauncher<SliderClient> launcher = createMasterlessAM(clustername, 0, true, false)
+ ServiceLauncher<SliderClient> launcher = createStandaloneAM(
+ clustername,
+ true,
+ false)
addToTeardown(launcher)
//do the low level operations to get a better view of what is going on
SliderClient sliderClient = launcher.service
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cbd890f/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionStatus.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionStatus.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionStatus.groovy
index 01d7e9e..bae8cea 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionStatus.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionStatus.groovy
@@ -30,10 +30,8 @@ import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
import org.apache.slider.common.params.Arguments
import org.apache.slider.client.SliderClient
import org.apache.slider.common.params.ActionStatusArgs
-import org.apache.slider.common.params.ClientArgs
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.core.main.ServiceLauncher
import org.junit.Before
import org.junit.Test
@@ -87,7 +85,10 @@ class TestActionStatus extends AgentMiniClusterTestBase {
String clustername = "teststatuslivecluster"
//launch the cluster
- ServiceLauncher<SliderClient> launcher = createMasterlessAM(clustername, 0, true, false)
+ ServiceLauncher<SliderClient> launcher = createStandaloneAM(
+ clustername,
+ true,
+ false)
SliderClient sliderClient = launcher.service
ApplicationReport report = waitForClusterLive(sliderClient)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cbd890f/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy
index 9e349d6..f5eff25 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy
@@ -47,9 +47,9 @@ class TestFreezeCommands extends AgentMiniClusterTestBase {
describe "create a masterless AM, freeze it, try to freeze again"
- ServiceLauncher<SliderClient> launcher = createMasterlessAM(
- clustername,
- 0,
+ ServiceLauncher<SliderClient> launcher = createStandaloneAM(
+ clustername
+ ,
true,
true);
addToTeardown(launcher.service);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cbd890f/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawMasterlessAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawMasterlessAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawMasterlessAM.groovy
index b102fa0..59db5f7 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawMasterlessAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeThawMasterlessAM.groovy
@@ -58,7 +58,10 @@ class TestFreezeThawMasterlessAM extends AgentMiniClusterTestBase {
SliderUtils.copyDirectory(conf, resConfPath, tempConfPath, null)
- ServiceLauncher<SliderClient> launcher = createMasterlessAM(clustername, 0, true, true)
+ ServiceLauncher<SliderClient> launcher = createStandaloneAM(
+ clustername,
+ true,
+ true)
SliderClient sliderClient = launcher.service
addToTeardown(sliderClient);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cbd890f/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBadAMHeap.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBadAMHeap.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBadAMHeap.groovy
index 2c9ce79..b253c17 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBadAMHeap.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBadAMHeap.groovy
@@ -45,7 +45,7 @@ class TestBadAMHeap extends AgentMiniClusterTestBase {
try {
ServiceLauncher<SliderClient> launcher =
- createMasterlessAMWithArgs(clustername,
+ createStandaloneAMWithArgs(clustername,
[
Arguments.ARG_COMP_OPT,
SliderKeys.COMPONENT_AM,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cbd890f/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBadYarnQueue.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBadYarnQueue.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBadYarnQueue.groovy
index a426d2b..13d5398 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBadYarnQueue.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBadYarnQueue.groovy
@@ -50,7 +50,7 @@ class TestBadYarnQueue extends AgentMiniClusterTestBase {
try {
ServiceLauncher<SliderClient> launcher =
- createMasterlessAMWithArgs(clustername,
+ createStandaloneAMWithArgs(clustername,
[
Arguments.ARG_DEFINE,
SliderXmlConfKeys.KEY_YARN_QUEUE + "=noqueue",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cbd890f/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestDestroyMasterlessAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestDestroyMasterlessAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestDestroyMasterlessAM.groovy
deleted file mode 100644
index 61c0c80..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestDestroyMasterlessAM.groovy
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.agent.standalone
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.slider.agent.AgentMiniClusterTestBase
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.params.ActionEchoArgs
-import org.apache.slider.common.params.Arguments
-import org.apache.slider.common.params.SliderActions
-import org.apache.slider.common.tools.SliderFileSystem
-import org.apache.slider.core.exceptions.ErrorStrings
-import org.apache.slider.core.exceptions.SliderException
-import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
-import org.apache.slider.core.main.ServiceLauncher
-import org.junit.Test
-
-/**
- * destroy a masterless AM
- */
-@CompileStatic
-@Slf4j
-
-class TestDestroyMasterlessAM extends AgentMiniClusterTestBase {
-
- @Test
- public void testDestroyMasterlessAM() throws Throwable {
- String clustername = createMiniCluster("", configuration, 1, false)
-
- describe "create a masterless AM, stop it, try to create" +
- "a second cluster with the same name, destroy it, try a third time"
-
- ServiceLauncher<SliderClient> launcher1 = launchClientAgainstMiniMR(
- configuration,
- [
- SliderActions.ACTION_DESTROY,
- "no-cluster-of-this-name",
- Arguments.ARG_FILESYSTEM, fsDefaultName,
- ])
- assert launcher1.serviceExitCode == 0
-
-
-
- ServiceLauncher<SliderClient> launcher = createMasterlessAM(clustername, 0, true, true)
- SliderClient sliderClient = launcher.service
- addToTeardown(sliderClient);
-
- SliderFileSystem sliderFileSystem = createSliderFileSystem()
- def fs = sliderFileSystem.fileSystem
- def instanceDir = sliderFileSystem.buildClusterDirPath(clustername)
-
- assertPathExists(
- fs,
- "cluster path not found",
- instanceDir)
-
- sliderFileSystem.locateInstanceDefinition(clustername)
- clusterActionFreeze(sliderClient, clustername,"stopping first cluster")
- waitForAppToFinish(sliderClient)
-
-
- describe "Warnings below are expected"
-
- //now try to create instance #2, and expect an in-use failure
- try {
- createMasterlessAM(clustername, 0, false, false)
- fail("expected a failure, got an AM")
- } catch (SliderException e) {
- assertExceptionDetails(e,
- SliderExitCodes.EXIT_INSTANCE_EXISTS,
- ErrorStrings.E_ALREADY_EXISTS)
- }
-
- describe "END EXPECTED WARNINGS"
-
-
- describe "destroying $clustername"
- //now: destroy it
-
- int exitCode = sliderClient.actionDestroy(clustername);
- assert 0 == exitCode
-
- describe "post destroy checks"
- sliderFileSystem.verifyDirectoryNonexistent(instanceDir)
-
- describe "thaw expected to fail"
- //expect thaw to now fail
- def ex = launchExpectingException(SliderClient,
- configuration,
- "",
- [
- SliderActions.ACTION_THAW,
- clustername,
- Arguments.ARG_FILESYSTEM, fsDefaultName,
- Arguments.ARG_MANAGER, RMAddr,
- ])
- assert ex instanceof UnknownApplicationInstanceException
-
- describe "thaw completed, checking dir is still absent"
- sliderFileSystem.verifyDirectoryNonexistent(instanceDir)
-
-
- describe "recreating $clustername"
-
- //and create a new cluster
- launcher = createMasterlessAM(clustername, 0, false, true)
- SliderClient cluster2 = launcher.service
-
- // do an echo here of a large string
- // Hadoop RPC couldn't handle strings > 32K chars, this
- // check here allows us to be confident that large JSON Reports are handled
- StringBuilder sb = new StringBuilder()
- for (int i = 0; i < 65536; i++) {
- sb.append(Integer.toString(i, 16))
- }
- ActionEchoArgs args = new ActionEchoArgs();
- args.message = sb.toString();
- def echoed = cluster2.actionEcho(clustername, args)
- assert echoed == args.message
- log.info(
- "Successful echo of a text document ${echoed.size()} characters long")
- //try to destroy it while live
- try {
- int ec = cluster2.actionDestroy(clustername)
- fail("expected a failure from the destroy, got error code $ec")
- } catch (SliderException e) {
- assertFailureClusterInUse(e);
- }
-
- //and try to destroy a completely different cluster just for the fun of it
- assert 0 == sliderClient.actionDestroy("no-cluster-of-this-name")
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cbd890f/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestDestroyStandaloneAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestDestroyStandaloneAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestDestroyStandaloneAM.groovy
new file mode 100644
index 0000000..ed0604b
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestDestroyStandaloneAM.groovy
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.params.ActionEchoArgs
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.common.tools.SliderFileSystem
+import org.apache.slider.core.exceptions.ErrorStrings
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * destroy a masterless AM
+ */
+@CompileStatic
+@Slf4j
+
+class TestDestroyStandaloneAM extends AgentMiniClusterTestBase {
+
+ @Test
+ public void testDestroyStandaloneAM() throws Throwable {
+ String clustername = createMiniCluster("", configuration, 1, false)
+
+ describe "create a Standalone AM, stop it, try to create" +
+ "a second cluster with the same name, destroy it, try a third time"
+
+ ServiceLauncher<SliderClient> launcher1 = launchClientAgainstMiniMR(
+ configuration,
+ [
+ SliderActions.ACTION_DESTROY,
+ "no-cluster-of-this-name",
+ Arguments.ARG_FILESYSTEM, fsDefaultName,
+ ])
+ assert launcher1.serviceExitCode == 0
+
+
+
+ ServiceLauncher<SliderClient> launcher = createStandaloneAM(
+ clustername,
+ true,
+ true)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+ SliderFileSystem sliderFileSystem = createSliderFileSystem()
+ def fs = sliderFileSystem.fileSystem
+ def instanceDir = sliderFileSystem.buildClusterDirPath(clustername)
+
+ assertPathExists(
+ fs,
+ "cluster path not found",
+ instanceDir)
+
+ sliderFileSystem.locateInstanceDefinition(clustername)
+ clusterActionFreeze(sliderClient, clustername,"stopping first cluster")
+ waitForAppToFinish(sliderClient)
+
+
+ describe "Warnings below are expected"
+
+ //now try to create instance #2, and expect an in-use failure
+ try {
+ createStandaloneAM(clustername, false, false)
+ fail("expected a failure, got an AM")
+ } catch (SliderException e) {
+ assertExceptionDetails(e,
+ SliderExitCodes.EXIT_INSTANCE_EXISTS,
+ ErrorStrings.E_ALREADY_EXISTS)
+ }
+
+ describe "END EXPECTED WARNINGS"
+
+
+ describe "destroying $clustername"
+ //now: destroy it
+
+ int exitCode = sliderClient.actionDestroy(clustername);
+ assert 0 == exitCode
+
+ describe "post destroy checks"
+ sliderFileSystem.verifyDirectoryNonexistent(instanceDir)
+
+ describe "thaw expected to fail"
+ //expect thaw to now fail
+ def ex = launchExpectingException(SliderClient,
+ configuration,
+ "",
+ [
+ SliderActions.ACTION_THAW,
+ clustername,
+ Arguments.ARG_FILESYSTEM, fsDefaultName,
+ Arguments.ARG_MANAGER, RMAddr,
+ ])
+ assert ex instanceof UnknownApplicationInstanceException
+
+ describe "thaw completed, checking dir is still absent"
+ sliderFileSystem.verifyDirectoryNonexistent(instanceDir)
+
+
+ describe "recreating $clustername"
+
+ //and create a new cluster
+ launcher = createStandaloneAM(clustername, false, true)
+ SliderClient cluster2 = launcher.service
+
+ // do an echo here of a large string
+ // Hadoop RPC couldn't handle strings > 32K chars, this
+ // check here allows us to be confident that large JSON Reports are handled
+ StringBuilder sb = new StringBuilder()
+ for (int i = 0; i < 65536; i++) {
+ sb.append(Integer.toString(i, 16))
+ }
+ ActionEchoArgs args = new ActionEchoArgs();
+ args.message = sb.toString();
+ def echoed = cluster2.actionEcho(clustername, args)
+ assert echoed == args.message
+ log.info(
+ "Successful echo of a text document ${echoed.size()} characters long")
+ //try to destroy it while live
+ try {
+ int ec = cluster2.actionDestroy(clustername)
+ fail("expected a failure from the destroy, got error code $ec")
+ } catch (SliderException e) {
+ assertFailureClusterInUse(e);
+ }
+
+ //and try to destroy a completely different cluster just for the fun of it
+ assert 0 == sliderClient.actionDestroy("no-cluster-of-this-name")
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cbd890f/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestKillMasterlessAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestKillMasterlessAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestKillMasterlessAM.groovy
deleted file mode 100644
index d8e77f0..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestKillMasterlessAM.groovy
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.agent.standalone
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.ApplicationReport
-import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.agent.AgentMiniClusterTestBase
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderXmlConfKeys
-import org.apache.slider.common.params.Arguments
-import org.apache.slider.core.main.ServiceLauncher
-import org.junit.Test
-
-/**
- * kill a masterless AM and verify it shuts down. This test
- * also sets the retry count to 1 to stop recreation attempts
- */
-@CompileStatic
-@Slf4j
-
-class TestKillMasterlessAM extends AgentMiniClusterTestBase {
-
-
- @Test
- public void testKillMasterlessAM() throws Throwable {
- String clustername = createMiniCluster("", configuration, 1, true)
-
- describe "kill a masterless AM and verify that it shuts down"
- ServiceLauncher<SliderClient> launcher =
- createMasterlessAMWithArgs(clustername,
- [
- Arguments.ARG_OPTION, SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, "1"
- ],
- true,
- false)
- SliderClient sliderClient = launcher.service
- addToTeardown(sliderClient);
- ApplicationReport report = waitForClusterLive(sliderClient)
-
- describe("listing Java processes")
- lsJavaProcesses();
- describe("killing AM")
- killAM(SIGTERM);
- waitWhileClusterLive(sliderClient);
- //give yarn some time to notice
- sleep(10000)
- describe("final listing")
- lsJavaProcesses();
- report = sliderClient.applicationReport
- assert YarnApplicationState.FAILED == report.yarnApplicationState;
- clusterActionFreeze(sliderClient, clustername)
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cbd890f/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestKillStandaloneAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestKillStandaloneAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestKillStandaloneAM.groovy
new file mode 100644
index 0000000..fef98a7
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestKillStandaloneAM.groovy
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * kill a masterless AM and verify it shuts down. This test
+ * also sets the retry count to 1 to stop recreation attempts
+ */
+@CompileStatic
+@Slf4j
+
+class TestKillStandaloneAM extends AgentMiniClusterTestBase {
+
+
+ @Test
+ public void testKillStandaloneAM() throws Throwable {
+ String clustername = createMiniCluster("", configuration, 1, true)
+
+ describe "kill a Standalone AM and verify that it shuts down"
+ ServiceLauncher<SliderClient> launcher =
+ createStandaloneAMWithArgs(clustername,
+ [
+ Arguments.ARG_OPTION, SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, "1"
+ ],
+ true,
+ false)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+ ApplicationReport report = waitForClusterLive(sliderClient)
+
+ describe("listing Java processes")
+ lsJavaProcesses();
+ describe("killing AM")
+ killAM(SIGTERM);
+ waitWhileClusterLive(sliderClient);
+ //give yarn some time to notice
+ sleep(10000)
+ describe("final listing")
+ lsJavaProcesses();
+ report = sliderClient.applicationReport
+ assert YarnApplicationState.FAILED == report.yarnApplicationState;
+ clusterActionFreeze(sliderClient, clustername)
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cbd890f/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestRestartStandaloneAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestRestartStandaloneAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestRestartStandaloneAM.groovy
new file mode 100644
index 0000000..62dd1f9
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestRestartStandaloneAM.groovy
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.ApplicationReport
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.ActionAMSuicideArgs
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Test
+
+/**
+ * kill a masterless AM and verify it shuts down. This test
+ * also sets the retry count to 1 to stop recreation attempts
+ */
+@CompileStatic
+@Slf4j
+
+class TestRestartStandaloneAM extends AgentMiniClusterTestBase {
+
+
+ @Test
+ public void testRestartStandaloneAM() throws Throwable {
+ describe "kill a Standalone AM and verify that it restarts"
+ // patch the configuration for AM restart
+ YarnConfiguration conf = getRestartableConfiguration(5)
+
+ String clustername = createMiniCluster("", conf, 1, true)
+ ServiceLauncher<SliderClient> launcher =
+ createStandaloneAMWithArgs(clustername,
+ [
+ Arguments.ARG_OPTION, SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, "4"
+ ],
+ true,
+ false)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+ ApplicationReport report = waitForClusterLive(sliderClient)
+ logReport(report)
+ waitUntilClusterLive(sliderClient, 30000)
+
+
+ ActionAMSuicideArgs args = new ActionAMSuicideArgs()
+ args.message = "test AM iteration"
+ args.waittime = 100
+ args.exitcode = 1
+ sliderClient.actionAmSuicide(clustername, args)
+ waitWhileClusterLive(sliderClient);
+ //give yarn some time to notice
+ sleep(20000)
+ waitUntilClusterLive(sliderClient, 40000)
+
+
+ // app should be running here
+ assert 0 == sliderClient.actionExists(clustername, true)
+
+
+ clusterActionFreeze(sliderClient, clustername)
+ }
+
+ /**
+ * Get a restartable configuration
+ * @param restarts
+ * @return
+ */
+ public YarnConfiguration getRestartableConfiguration(int restarts) {
+ def conf = new YarnConfiguration(configuration)
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, restarts)
+ conf.setInt(SliderXmlConfKeys.KEY_AM_RESTART_LIMIT, restarts)
+ conf
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cbd890f/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
index 3d9b9cd..8498ee2 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
@@ -46,7 +46,7 @@ class TestStandaloneAgentAM extends AgentMiniClusterTestBase {
//launch fake master
String clustername = createMiniCluster("", configuration, 1, true)
ServiceLauncher<SliderClient> launcher =
- createMasterlessAM(clustername, 0, true, false)
+ createStandaloneAM(clustername, true, false)
SliderClient client = launcher.service
addToTeardown(client);
@@ -123,7 +123,7 @@ class TestStandaloneAgentAM extends AgentMiniClusterTestBase {
assert oldInstance.yarnApplicationState >= YarnApplicationState.FINISHED
//create another AM
- launcher = createMasterlessAM(clustername, 0, true, true)
+ launcher = createStandaloneAM(clustername, true, true)
client = launcher.service
ApplicationId i2AppID = client.applicationId
@@ -140,7 +140,7 @@ class TestStandaloneAgentAM extends AgentMiniClusterTestBase {
describe("attempting to create instance #3")
//now try to create instance #3, and expect an in-use failure
try {
- createMasterlessAM(clustername, 0, false, true)
+ createStandaloneAM(clustername, false, true)
fail("expected a failure, got a masterless AM")
} catch (SliderException e) {
assertFailureClusterInUse(e);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cbd890f/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneBadClusterName.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneBadClusterName.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneBadClusterName.groovy
index b2cb670..cf97520 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneBadClusterName.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneBadClusterName.groovy
@@ -38,7 +38,7 @@ class TestStandaloneBadClusterName extends AgentMiniClusterTestBase {
describe "verify that bad cluster names are picked up"
try {
- addToTeardown(createMasterlessAM(clustername, 0, true, false).service);
+ addToTeardown(createStandaloneAM(clustername, true, false).service);
fail("expected a failure")
} catch (ServiceLaunchException e) {
assertExceptionDetails(e, LauncherExitCodes.EXIT_COMMAND_ARGUMENT_ERROR)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cbd890f/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
index de9f184..b8a590e 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneRegistryAM.groovy
@@ -64,7 +64,7 @@ class TestStandaloneRegistryAM extends AgentMiniClusterTestBase {
//launch fake master
String clustername = createMiniCluster(configuration, 1, true)
ServiceLauncher<SliderClient> launcher
- launcher = createMasterlessAM(clustername, 0, true, false)
+ launcher = createStandaloneAM(clustername, true, false)
SliderClient client = launcher.service
addToTeardown(client);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1cbd890f/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index ed66190..b6f863b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -445,9 +445,10 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
assert clustername != null
assert miniCluster != null
// update action should keep existing data
+ def config = miniCluster.config
if (deleteExistingData && !SliderActions.ACTION_UPDATE.equals(action)) {
- HadoopFS dfs = HadoopFS.get(new URI(fsDefaultName), miniCluster.config)
- Path clusterDir = new SliderFileSystem(dfs, miniCluster.config).buildClusterDirPath(clustername)
+ HadoopFS dfs = HadoopFS.get(new URI(fsDefaultName), config)
+ Path clusterDir = new SliderFileSystem(dfs, config).buildClusterDirPath(clustername)
log.info("deleting customer data at $clusterDir")
//this is a safety check to stop us doing something stupid like deleting /
assert clusterDir.toString().contains("/.slider/")
@@ -486,7 +487,7 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
}
ServiceLauncher<SliderClient> launcher = launchClientAgainstMiniMR(
//config includes RM binding info
- new YarnConfiguration(miniCluster.config),
+ new YarnConfiguration(config),
//varargs list of command line params
argsList
)
[05/50] [abbrv] git commit: SLIDER-270 rename test so that
TestClusterFlex\* is a wildcard for all flex tests
Posted by st...@apache.org.
SLIDER-270 rename test so that TestClusterFlex\* is a wildcard for all flex tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ae4d0734
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ae4d0734
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ae4d0734
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: ae4d07340b0c0fe54b3dc75e58744393d5949d31
Parents: 34c00e8
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 5 21:06:54 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 5 21:06:54 2014 +0100
----------------------------------------------------------------------
.../TestClusterFlexHMasterFlex1To2.groovy | 45 ++++++++++++++++++++
.../flexing/TestHMasterFlex1To2.groovy | 45 --------------------
2 files changed, 45 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ae4d0734/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexHMasterFlex1To2.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexHMasterFlex1To2.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexHMasterFlex1To2.groovy
new file mode 100644
index 0000000..298f40f
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexHMasterFlex1To2.groovy
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.flexing
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.junit.Test
+
+/**
+ * Create a master against the File:// fs
+ */
+@CompileStatic
+@Slf4j
+class TestClusterFlexHMasterFlex1To2 extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testClusterMasterFlex1To2() throws Throwable {
+ assert flexHBaseClusterTestRun(
+ "",
+ 1,
+ 2,
+ 1,
+ 1,
+ true)
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ae4d0734/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy
deleted file mode 100644
index ca26572..0000000
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.hbase.minicluster.flexing
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
-import org.junit.Test
-
-/**
- * Create a master against the File:// fs
- */
-@CompileStatic
-@Slf4j
-class TestHMasterFlex1To2 extends HBaseMiniClusterTestBase {
-
- @Test
- public void testClusterFlex1To2() throws Throwable {
- assert flexHBaseClusterTestRun(
- "",
- 1,
- 2,
- 1,
- 1,
- true)
- }
-
-
-}
[20/50] [abbrv] SLIDER-296 ContainerReleaseSelectors have the job of
choosing containers to release.
Posted by st...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
new file mode 100644
index 0000000..6e18e39
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.state.*
+import org.junit.Test
+
+import static org.apache.slider.server.appmaster.state.ContainerPriority.buildPriority
+import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole
+
+@Slf4j
+class TestMockAppStateRMOperations extends BaseMockAppStateTest implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestMockRMOperations"
+ }
+
+ @Test
+ public void testPriorityOnly() throws Throwable {
+ assert 5 == extractRole(buildPriority(5, false))
+ }
+
+ @Test
+ public void testPriorityRoundTrip() throws Throwable {
+ assert 5 == extractRole(buildPriority(5, false))
+ }
+
+ @Test
+ public void testPriorityRoundTripWithRequest() throws Throwable {
+ int priority = buildPriority(5, false)
+ assert 5 == extractRole(priority)
+ }
+
+ @Test
+ public void testMockAddOp() throws Throwable {
+ role0Status.desired = 1
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.size() == 1
+ ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
+ int priority = operation.request.priority.priority
+ assert extractRole(priority) == MockFactory.PROVIDER_ROLE0.id
+ RMOperationHandler handler = new MockRMOperationHandler()
+ handler.execute(ops)
+
+ //tell the container its been allocated
+ AbstractRMOperation op = handler.operations[0]
+ assert op instanceof ContainerRequestOperation
+ }
+
+ @Test
+ public void testAllocateReleaseOp() throws Throwable {
+ role0Status.desired = 1
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
+ AMRMClient.ContainerRequest request = operation.request
+ Container cont = engine.allocateContainer(request)
+ List<Container> allocated = [cont]
+ List<ContainerAssignment> assignments = [];
+ List<AbstractRMOperation> operations = []
+ appState.onContainersAllocated(allocated, assignments, operations)
+ assert operations.size() == 0
+ assert assignments.size() == 1
+ ContainerAssignment assigned = assignments[0]
+ Container target = assigned.container
+ assert target.id == cont.id
+ int roleId = assigned.role.priority
+ assert roleId == extractRole(request.priority)
+ assert assigned.role.name == ROLE0
+ RoleInstance ri = roleInstance(assigned)
+ //tell the app it arrived
+ appState.containerStartSubmitted(target, ri);
+ appState.innerOnNodeManagerContainerStarted(target.id)
+ assert role0Status.started == 1
+
+ //now release it by changing the role status
+ role0Status.desired = 0
+ ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.size() == 1
+
+ assert ops[0] instanceof ContainerReleaseOperation
+ ContainerReleaseOperation release = (ContainerReleaseOperation) ops[0]
+ assert release.containerId == cont.id
+ }
+
+ @Test
+ public void testComplexAllocation() throws Throwable {
+ role0Status.desired = 1
+ role1Status.desired = 3
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ List<Container> allocations = engine.execute(ops)
+ List<ContainerAssignment> assignments = [];
+ List<AbstractRMOperation> releases = []
+ appState.onContainersAllocated(allocations, assignments, releases)
+ assert releases.size() == 0
+ assert assignments.size() == 4
+ assignments.each { ContainerAssignment assigned ->
+ Container target = assigned.container
+ RoleInstance ri = roleInstance(assigned)
+ appState.containerStartSubmitted(target, ri);
+ }
+ //insert some async operation here
+ assignments.each { ContainerAssignment assigned ->
+ Container target = assigned.container
+ appState.innerOnNodeManagerContainerStarted(target.id)
+ }
+ assert engine.containerCount() == 4;
+ role1Status.desired = 0
+ ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.size() == 3
+ allocations = engine.execute(ops)
+ assert engine.containerCount() == 1;
+
+ appState.onContainersAllocated(allocations, assignments, releases)
+ assert assignments.empty
+ assert releases.empty
+ }
+
+ @Test
+ public void testDoubleNodeManagerStartEvent() throws Throwable {
+ role0Status.desired = 1
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ List<Container> allocations = engine.execute(ops)
+ List<ContainerAssignment> assignments = [];
+ List<AbstractRMOperation> releases = []
+ appState.onContainersAllocated(allocations, assignments, releases)
+ assert assignments.size() == 1
+ ContainerAssignment assigned = assignments[0]
+ Container target = assigned.container
+ RoleInstance ri = roleInstance(assigned)
+ appState.containerStartSubmitted(target, ri);
+ RoleInstance ri2 = appState.innerOnNodeManagerContainerStarted(target.id)
+ assert ri2 == ri
+ //try a second time, expect an error
+ try {
+ appState.innerOnNodeManagerContainerStarted(target.id)
+ fail("Expected an exception")
+ } catch (RuntimeException expected) {
+ // expected
+ }
+ //and non-faulter should not downgrade to a null
+ log.warn("Ignore any exception/stack trace that appears below")
+ log.warn("===============================================================")
+ RoleInstance ri3 = appState.onNodeManagerContainerStarted(target.id)
+ log.warn("===============================================================")
+ log.warn("Ignore any exception/stack trace that appeared above")
+ assert ri3 == null
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
new file mode 100644
index 0000000..630be29
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.slider.api.StatusKeys
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.state.*
+import org.junit.Test
+
+/**
+ * Test that if you have >1 role, the right roles are chosen for release.
+ */
+@CompileStatic
+@Slf4j
+class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest
+ implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestAppStateRebuildOnAMRestart"
+ }
+
+ @Test
+ public void testRebuild() throws Throwable {
+
+ int r0 = 1
+ int r1 = 2
+ int r2 = 3
+ role0Status.desired = r0
+ role1Status.desired = r1
+ role2Status.desired = r2
+ List<RoleInstance> instances = createAndStartNodes()
+
+ int clusterSize = r0 + r1 + r2
+ assert instances.size() == clusterSize
+
+ //clone the list
+ List<RoleInstance> cloned = [];
+ List<Container> containers = []
+ instances.each { RoleInstance elt ->
+ cloned.add(elt.clone() as RoleInstance)
+ containers.add(elt.container)
+ }
+ NodeMap nodemap = appState.roleHistory.cloneNodemap()
+
+ // now destroy the app state
+ appState = new AppState(new MockRecordFactory())
+
+ //and rebuild
+ appState.buildInstance(
+ factory.newInstanceDefinition(r0, r1, r2),
+ new Configuration(),
+ new Configuration(false),
+ factory.ROLES,
+ fs,
+ historyPath,
+ containers,
+ null, new SimpleReleaseSelector())
+
+ assert appState.getStartedCountainerCount() == clusterSize
+
+ appState.getRoleHistory().dump();
+
+ //check that the app state direct structures match
+ List<RoleInstance> r0live = appState.enumLiveNodesInRole(ROLE0)
+ List<RoleInstance> r1live = appState.enumLiveNodesInRole(ROLE1)
+ List<RoleInstance> r2live = appState.enumLiveNodesInRole(ROLE2)
+
+ assert r0 == r0live.size()
+ assert r1 == r1live.size()
+ assert r2 == r2live.size()
+
+ //now examine the role history
+ NodeMap newNodemap = appState.roleHistory.cloneNodemap()
+
+ for (NodeInstance nodeInstance : newNodemap.values()) {
+ String hostname = nodeInstance.hostname
+ NodeInstance orig = nodemap[hostname]
+ assertNotNull("Null entry in original nodemap for " + hostname, orig)
+
+ for (int i = 0; i < ROLE_COUNT; i++) {
+
+ assert (nodeInstance.getActiveRoleInstances(i) ==
+ orig.getActiveRoleInstances(i))
+ NodeEntry origRE = orig.getOrCreate(i)
+ NodeEntry newRE = nodeInstance.getOrCreate(i)
+ assert origRE.live == newRE.live
+ assert newRE.starting == 0
+ }
+ }
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.size() == 0
+
+ def status = appState.getClusterStatus()
+ // verify the AM restart container count was set
+ String restarted = status.getInfo(
+ StatusKeys.INFO_CONTAINERS_AM_RESTART)
+ assert restarted != null;
+ //and that the count == 1 master + the region servers
+ assert Integer.parseInt(restarted) == containers.size()
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
new file mode 100644
index 0000000..b592f83
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRolePlacement.groovy
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.state.*
+import org.junit.Test
+
+import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole
+
+/**
+ * Test that the app state lets you ask for nodes, get a specific host,
+ * release it and then get that one back again.
+ */
+@CompileStatic
+@Slf4j
+class TestMockAppStateRolePlacement extends BaseMockAppStateTest
+ implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestAppStateRolePlacement"
+ }
+
+
+ @Test
+ public void testAllocateReleaseRealloc() throws Throwable {
+ role0Status.desired = 1
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
+ AMRMClient.ContainerRequest request = operation.request
+ Container allocated = engine.allocateContainer(request)
+ List<ContainerAssignment> assignments = [];
+ List<AbstractRMOperation> operations = []
+ appState.onContainersAllocated([(Container)allocated], assignments, operations)
+ assert operations.size() == 0
+ assert assignments.size() == 1
+ ContainerAssignment assigned = assignments[0]
+ Container container = assigned.container
+ assert container.id == allocated.id
+ int roleId = assigned.role.priority
+ assert roleId == extractRole(request.priority)
+ assert assigned.role.name == ROLE0
+ String containerHostname = RoleHistoryUtils.hostnameOf(container);
+ RoleInstance ri = roleInstance(assigned)
+ //tell the app it arrived
+ appState.containerStartSubmitted(container, ri);
+ assert appState.onNodeManagerContainerStarted(container.id)
+ assert role0Status.started == 1
+ ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.size() == 0
+
+ //now it is surplus
+ role0Status.desired = 0
+ ops = appState.reviewRequestAndReleaseNodes()
+ ContainerReleaseOperation release = (ContainerReleaseOperation) ops[0]
+
+ assert release.containerId == container.id
+ engine.execute(ops)
+ assert appState.onCompletedNode(containerStatus(container)).roleInstance
+
+ //view the world
+ appState.getRoleHistory().dump();
+
+ //now ask for a new one
+ role0Status.desired = 1
+ ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.size() == 1
+ operation = (ContainerRequestOperation) ops[0]
+ AMRMClient.ContainerRequest request2 = operation.request
+ assert request2 != null
+ assert request2.nodes[0] == containerHostname
+ engine.execute(ops)
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.groovy
new file mode 100644
index 0000000..495baf0
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRoleRelease.groovy
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.state.AbstractRMOperation
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.junit.Test
+
+/**
+ * Test that if you have >1 role, the right roles are chosen for release.
+ */
+@CompileStatic
+@Slf4j
+class TestMockAppStateRoleRelease extends BaseMockAppStateTest
+ implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestAppStateRolePlacement"
+ }
+
+ /**
+ * Small cluster with multiple containers per node,
+ * to guarantee many container allocations on each node
+ * @return
+ */
+ @Override
+ MockYarnEngine createYarnEngine() {
+ return new MockYarnEngine(4, 4)
+ }
+
+ @Test
+ public void testAllocateReleaseRealloc() throws Throwable {
+ /**
+ * Allocate to all nodes
+ */
+ role0Status.desired = 6
+ role1Status.desired = 5
+ role2Status.desired = 4
+ List<RoleInstance> instances = createAndStartNodes()
+ assert instances.size() == 15
+
+ //now it is surplus
+ role0Status.desired = 0
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+
+ List<ContainerId> released = []
+ engine.execute(ops, released)
+ List<ContainerId> ids = extractContainerIds(instances, 0)
+ released.each { ContainerId cid ->
+ assert appState.onCompletedNode(containerStatus(cid)).roleInstance
+ assert ids.contains(cid)
+ }
+
+ //view the world
+ appState.getRoleHistory().dump();
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
new file mode 100644
index 0000000..d018c12
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.Resource
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.core.conf.ConfTree
+import org.apache.slider.core.conf.ConfTreeOperations
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.state.AbstractRMOperation
+import org.apache.slider.server.appmaster.state.ContainerRequestOperation
+import org.junit.Test
+
+/**
+ * Test the container resource allocation logic
+ */
+@CompileStatic
+@Slf4j
+class TestMockContainerResourceAllocations extends BaseMockAppStateTest {
+
+ @Override
+ String getTestName() {
+ "TestContainerResourceAllocations"
+ }
+
+ @Test
+ public void testNormalAllocations() throws Throwable {
+ ConfTree clusterSpec = factory.newConfTree(1, 0, 0)
+ ConfTreeOperations cto = new ConfTreeOperations(clusterSpec)
+
+ cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_MEMORY, 512)
+ cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_CORES, 2)
+ appState.updateResourceDefinitions(clusterSpec)
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.size() == 1
+ ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
+ Resource requirements = operation.request.capability
+ assert requirements.memory == 512
+ assert requirements.virtualCores == 2
+ }
+
+ @Test
+ public void testMaxMemAllocations() throws Throwable {
+ ConfTree clusterSpec = factory.newConfTree(1, 0, 0)
+ ConfTreeOperations cto = new ConfTreeOperations(clusterSpec)
+
+ cto.setComponentOpt(MockRoles.ROLE0, ResourceKeys.YARN_MEMORY,
+ ResourceKeys.YARN_RESOURCE_MAX)
+ cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_CORES, 2)
+ appState.updateResourceDefinitions(clusterSpec)
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.size() == 1
+ ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
+ Resource requirements = operation.request.capability
+ assert requirements.memory == RM_MAX_RAM
+ assert requirements.virtualCores == 2
+ }
+
+ @Test
+ public void testMaxCoreAllocations() throws Throwable {
+ ConfTree clusterSpec = factory.newConfTree(1, 0, 0)
+ ConfTreeOperations cto = new ConfTreeOperations(clusterSpec)
+ cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_MEMORY,
+ 512)
+ cto.setComponentOpt(MockRoles.ROLE0, ResourceKeys.YARN_CORES,
+ ResourceKeys.YARN_RESOURCE_MAX)
+ appState.updateResourceDefinitions(clusterSpec)
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.size() == 1
+ ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
+ Resource requirements = operation.request.capability
+ assert requirements.memory == 512
+ assert requirements.virtualCores == RM_MAX_CORES
+ }
+
+ @Test
+ public void testMaxDefaultAllocations() throws Throwable {
+
+ ConfTree clusterSpec = factory.newConfTree(1, 0, 0)
+ appState.updateResourceDefinitions(clusterSpec)
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.size() == 1
+ ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
+ Resource requirements = operation.request.capability
+ assert requirements.memory == ResourceKeys.DEF_YARN_MEMORY
+ assert requirements.virtualCores == ResourceKeys.DEF_YARN_CORES
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockFlexing.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockFlexing.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockFlexing.groovy
deleted file mode 100644
index d2dc028..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockFlexing.groovy
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate
-
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.Container
-import org.apache.hadoop.yarn.api.records.ContainerId
-import org.apache.hadoop.yarn.client.api.AMRMClient
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler
-import org.apache.slider.server.appmaster.model.mock.MockRoles
-import org.apache.slider.server.appmaster.state.AbstractRMOperation
-import org.apache.slider.server.appmaster.state.AppState
-import org.apache.slider.server.appmaster.state.ContainerAssignment
-import org.apache.slider.server.appmaster.state.ContainerReleaseOperation
-import org.apache.slider.server.appmaster.state.ContainerRequestOperation
-import org.apache.slider.server.appmaster.state.RMOperationHandler
-import org.apache.slider.server.appmaster.state.RoleInstance
-import org.junit.Test
-
-import static org.apache.slider.server.appmaster.state.ContainerPriority.buildPriority
-import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole
-
-@Slf4j
-class TestMockFlexing extends BaseMockAppStateTest implements MockRoles {
-
- @Override
- String getTestName() {
- return "TestMockFlexing"
- }
-
-
-
- @Test
- public void testFlexDuringLaunchPhase() throws Throwable {
- role0Status.desired = 1
-
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- List<Container> allocations = engine.execute(ops)
- List<ContainerAssignment> assignments = [];
- List<AbstractRMOperation> releases = []
- appState.onContainersAllocated(allocations, assignments, releases)
- assert assignments.size() == 1
- ContainerAssignment assigned = assignments[0]
- Container target = assigned.container
- RoleInstance ri = roleInstance(assigned)
-
- ops = appState.reviewRequestAndReleaseNodes()
- assert ops.empty
-
- //now this is the start point.
- appState.containerStartSubmitted(target, ri);
-
- ops = appState.reviewRequestAndReleaseNodes()
- assert ops.empty
-
- RoleInstance ri2 = appState.innerOnNodeManagerContainerStarted(target.id)
- }
-
- @Test
- public void testFlexBeforeAllocationPhase() throws Throwable {
- role0Status.desired = 1
-
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- assert !ops.empty
- List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes()
- assert ops2.empty
- }
-
-
- @Test
- public void testFlexDownTwice() throws Throwable {
- int r0 = 6
- int r1 = 0
- int r2 = 0
- role0Status.desired = r0
- role1Status.desired = r1
- role2Status.desired = r2
- List<RoleInstance> instances = createAndStartNodes()
-
- int clusterSize = r0 + r1 + r2
- assert instances.size() == clusterSize
- log.info("shrinking cluster")
- r0 = 4
- role0Status.desired = r0
- List<AppState.NodeCompletionResult> completionResults = []
- instances = createStartAndStopNodes(completionResults)
- assert instances.size() == 0
- // assert two nodes were released
- assert completionResults.size() == 2
-
- // no-op review
- completionResults = []
- instances = createStartAndStopNodes(completionResults)
- assert instances.size() == 0
- // assert two nodes were released
- assert completionResults.size() == 0
-
-
- // now shrink again
- role0Status.desired = r0 = 1
- completionResults = []
- instances = createStartAndStopNodes(completionResults)
- assert instances.size() == 0
- // assert two nodes were released
- assert completionResults.size() == 3
-
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy
deleted file mode 100644
index e27827b..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate
-
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.Container
-import org.apache.hadoop.yarn.client.api.AMRMClient
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler
-import org.apache.slider.server.appmaster.model.mock.MockRoles
-import org.apache.slider.server.appmaster.state.*
-import org.junit.Test
-
-import static org.apache.slider.server.appmaster.state.ContainerPriority.buildPriority
-import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole
-
-@Slf4j
-class TestMockRMOperations extends BaseMockAppStateTest implements MockRoles {
-
- @Override
- String getTestName() {
- return "TestMockRMOperations"
- }
-
- @Test
- public void testPriorityOnly() throws Throwable {
- assert 5 == extractRole(buildPriority(5, false))
- }
-
- @Test
- public void testPriorityRoundTrip() throws Throwable {
- assert 5 == extractRole(buildPriority(5, false))
- }
-
- @Test
- public void testPriorityRoundTripWithRequest() throws Throwable {
- int priority = buildPriority(5, false)
- assert 5 == extractRole(priority)
- }
-
- @Test
- public void testMockAddOp() throws Throwable {
- role0Status.desired = 1
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 1
- ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
- int priority = operation.request.priority.priority
- assert extractRole(priority) == MockFactory.PROVIDER_ROLE0.id
- RMOperationHandler handler = new MockRMOperationHandler()
- handler.execute(ops)
-
- //tell the container its been allocated
- AbstractRMOperation op = handler.operations[0]
- assert op instanceof ContainerRequestOperation
- }
-
- @Test
- public void testAllocateReleaseOp() throws Throwable {
- role0Status.desired = 1
-
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
- AMRMClient.ContainerRequest request = operation.request
- Container cont = engine.allocateContainer(request)
- List<Container> allocated = [cont]
- List<ContainerAssignment> assignments = [];
- List<AbstractRMOperation> operations = []
- appState.onContainersAllocated(allocated, assignments, operations)
- assert operations.size() == 0
- assert assignments.size() == 1
- ContainerAssignment assigned = assignments[0]
- Container target = assigned.container
- assert target.id == cont.id
- int roleId = assigned.role.priority
- assert roleId == extractRole(request.priority)
- assert assigned.role.name == ROLE0
- RoleInstance ri = roleInstance(assigned)
- //tell the app it arrived
- appState.containerStartSubmitted(target, ri);
- appState.innerOnNodeManagerContainerStarted(target.id)
- assert role0Status.started == 1
-
- //now release it by changing the role status
- role0Status.desired = 0
- ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 1
-
- assert ops[0] instanceof ContainerReleaseOperation
- ContainerReleaseOperation release = (ContainerReleaseOperation) ops[0]
- assert release.containerId == cont.id
- }
-
- @Test
- public void testComplexAllocation() throws Throwable {
- role0Status.desired = 1
- role1Status.desired = 3
-
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- List<Container> allocations = engine.execute(ops)
- List<ContainerAssignment> assignments = [];
- List<AbstractRMOperation> releases = []
- appState.onContainersAllocated(allocations, assignments, releases)
- assert releases.size() == 0
- assert assignments.size() == 4
- assignments.each { ContainerAssignment assigned ->
- Container target = assigned.container
- RoleInstance ri = roleInstance(assigned)
- appState.containerStartSubmitted(target, ri);
- }
- //insert some async operation here
- assignments.each { ContainerAssignment assigned ->
- Container target = assigned.container
- appState.innerOnNodeManagerContainerStarted(target.id)
- }
- assert engine.containerCount() == 4;
- role1Status.desired = 0
- ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 3
- allocations = engine.execute(ops)
- assert engine.containerCount() == 1;
-
- appState.onContainersAllocated(allocations, assignments, releases)
- assert assignments.empty
- assert releases.empty
- }
-
- @Test
- public void testDoubleNodeManagerStartEvent() throws Throwable {
- role0Status.desired = 1
-
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- List<Container> allocations = engine.execute(ops)
- List<ContainerAssignment> assignments = [];
- List<AbstractRMOperation> releases = []
- appState.onContainersAllocated(allocations, assignments, releases)
- assert assignments.size() == 1
- ContainerAssignment assigned = assignments[0]
- Container target = assigned.container
- RoleInstance ri = roleInstance(assigned)
- appState.containerStartSubmitted(target, ri);
- RoleInstance ri2 = appState.innerOnNodeManagerContainerStarted(target.id)
- assert ri2 == ri
- //try a second time, expect an error
- try {
- appState.innerOnNodeManagerContainerStarted(target.id)
- fail("Expected an exception")
- } catch (RuntimeException expected) {
- // expected
- }
- //and non-faulter should not downgrade to a null
- log.warn("Ignore any exception/stack trace that appears below")
- log.warn("===============================================================")
- RoleInstance ri3 = appState.onNodeManagerContainerStarted(target.id)
- log.warn("===============================================================")
- log.warn("Ignore any exception/stack trace that appeared above")
- assert ri3 == null
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
index 4a34e7f..987bc9b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -87,7 +87,7 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
factory.ROLES,
fs,
historyPath,
- null, null)
+ null, null, new SimpleReleaseSelector())
}
abstract String getTestName();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
index db28a22..0f7bb08 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
@@ -35,6 +35,8 @@ import org.apache.slider.core.registry.info.ServiceInstanceData
import org.apache.slider.providers.ProviderRole
import org.apache.slider.providers.ProviderService
import org.apache.slider.server.appmaster.AMViewForProviders
+import org.apache.slider.server.appmaster.state.ContainerReleaseSelector
+import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector
import org.apache.slider.server.appmaster.state.StateAccessForProviders
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations
import org.apache.slider.server.appmaster.web.rest.agent.HeartBeat
@@ -64,7 +66,8 @@ class MockProviderService implements ProviderService {
}
@Override
- public void validateInstanceDefinition(AggregateConf instanceDefinition) throws SliderException {
+ public void validateInstanceDefinition(AggregateConf instanceDefinition)
+ throws SliderException {
}
@Override
@@ -100,7 +103,7 @@ class MockProviderService implements ProviderService {
return null
}
- @Override
+ @Override
public long getStartTime() {
return 0;
}
@@ -131,7 +134,7 @@ class MockProviderService implements ProviderService {
}
@Override
- public Map<String,String> getBlockers() {
+ public Map<String, String> getBlockers() {
return null;
}
@@ -156,7 +159,8 @@ class MockProviderService implements ProviderService {
}
@Override
- public Configuration loadProviderConfigurationInformation(File confDir) throws BadCommandArgumentsException, IOException {
+ public Configuration loadProviderConfigurationInformation(File confDir)
+ throws BadCommandArgumentsException, IOException {
return null;
}
@@ -169,7 +173,7 @@ class MockProviderService implements ProviderService {
@Override
- public Map<String,String> buildProviderStatus() {
+ public Map<String, String> buildProviderStatus() {
return null;
}
@@ -188,7 +192,8 @@ class MockProviderService implements ProviderService {
}
@Override
- public Map<String, String> buildMonitorDetails(ClusterDescription clusterSpec) {
+ public Map<String, String> buildMonitorDetails(
+ ClusterDescription clusterSpec) {
return null;
}
@@ -201,23 +206,23 @@ class MockProviderService implements ProviderService {
}
@Override
- AgentRestOperations getAgentRestOperations() {
- return new AgentRestOperations() {
- @Override
- public RegistrationResponse handleRegistration(Register registration) {
- // dummy impl
- RegistrationResponse response = new RegistrationResponse();
- response.setResponseStatus(RegistrationStatus.OK);
- return response;
- }
+ AgentRestOperations getAgentRestOperations() {
+ return new AgentRestOperations() {
+ @Override
+ public RegistrationResponse handleRegistration(Register registration) {
+ // dummy impl
+ RegistrationResponse response = new RegistrationResponse();
+ response.setResponseStatus(RegistrationStatus.OK);
+ return response;
+ }
- @Override
- public HeartBeatResponse handleHeartBeat(HeartBeat heartBeat) {
- // dummy impl
- return new HeartBeatResponse();
- }
- }
+ @Override
+ public HeartBeatResponse handleHeartBeat(HeartBeat heartBeat) {
+ // dummy impl
+ return new HeartBeatResponse();
+ }
}
+ }
@Override
void buildEndpointDetails(Map<String, String> details) {
@@ -226,7 +231,9 @@ class MockProviderService implements ProviderService {
@Override
void applyInitialRegistryDefinitions(
- URL unsecureWebAPI, URL secureWebAPI, ServiceInstanceData registryInstanceData)
+ URL unsecureWebAPI,
+ URL secureWebAPI,
+ ServiceInstanceData registryInstanceData)
throws MalformedURLException, IOException {
}
@@ -234,4 +241,9 @@ class MockProviderService implements ProviderService {
@Override
public void notifyContainerCompleted(ContainerId containerId) {
}
+
+ @Override
+ ContainerReleaseSelector createContainerReleaseSelector() {
+ return new MostRecentContainerReleaseSelector()
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
index 96a4510..7e2ab3c 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
@@ -37,6 +37,7 @@ import org.apache.slider.server.appmaster.model.mock.MockRecordFactory;
import org.apache.slider.server.appmaster.model.mock.MockSliderClusterProtocol;
import org.apache.slider.server.appmaster.state.AppState;
import org.apache.slider.server.appmaster.state.ProviderAppState;
+import org.apache.slider.server.appmaster.state.SimpleReleaseSelector;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.WebAppApiImpl;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
@@ -128,7 +129,7 @@ public class TestAMAgentWebServices {
factory.ROLES,
fs,
historyPath,
- null, null);
+ null, null, new SimpleReleaseSelector());
} catch (Exception e) {
log.error("Failed to set up app {}", e);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
index 96e196f..fd66bee 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
@@ -45,6 +45,7 @@ import org.apache.slider.server.appmaster.model.mock.MockRecordFactory;
import org.apache.slider.server.appmaster.model.mock.MockSliderClusterProtocol;
import org.apache.slider.server.appmaster.state.AppState;
import org.apache.slider.server.appmaster.state.ProviderAppState;
+import org.apache.slider.server.appmaster.state.SimpleReleaseSelector;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.WebAppApiImpl;
import org.apache.slider.server.appmaster.web.rest.AMWebServices;
@@ -175,7 +176,7 @@ public class TestAMManagementWebServices extends JerseyTest {
factory.ROLES,
fs,
historyPath,
- null, null);
+ null, null, new SimpleReleaseSelector());
} catch (IOException | BadClusterStateException | URISyntaxException | BadConfigException e) {
log.error("{}", e, e);
}
[26/50] [abbrv] git commit: Merge branch
'feature/SLIDER-296-AM_to_notify_containers_on_managed_container_release'
into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-296-AM_to_notify_containers_on_managed_container_release' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d19c8022
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d19c8022
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d19c8022
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: d19c802264afc9121cd9470860d2ac66caf5357f
Parents: 32fd3f5 0646c23
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 6 19:25:17 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 6 19:25:17 2014 +0100
----------------------------------------------------------------------
.../apache/slider/common/tools/Comparators.java | 58 +++++
.../providers/AbstractProviderService.java | 29 ++-
.../slider/providers/ProviderService.java | 20 +-
.../appmaster/AsyncRMOperationHandler.java | 51 ----
.../server/appmaster/SliderAppMaster.java | 19 +-
.../operations/AbstractRMOperation.java | 29 +++
.../operations/AsyncRMOperationHandler.java | 51 ++++
.../operations/ContainerReleaseOperation.java | 44 ++++
.../operations/ContainerRequestOperation.java | 44 ++++
.../ProviderNotifyingOperationHandler.java | 43 ++++
.../operations/RMOperationHandler.java | 35 +++
.../operations/RMOperationHandlerActions.java | 28 ++
.../appmaster/state/AbstractRMOperation.java | 31 ---
.../slider/server/appmaster/state/AppState.java | 135 +++++++---
.../state/ContainerReleaseOperation.java | 44 ----
.../state/ContainerReleaseSelector.java | 38 +++
.../state/ContainerRequestOperation.java | 44 ----
.../MostRecentContainerReleaseSelector.java | 52 ++++
.../server/appmaster/state/NodeInstance.java | 12 +-
.../slider/server/appmaster/state/NodeMap.java | 54 ----
.../appmaster/state/RMOperationHandler.java | 43 ----
.../server/appmaster/state/RoleHistory.java | 14 +-
.../appmaster/state/SimpleReleaseSelector.java | 34 +++
.../agent/TestAgentAMManagementWS.groovy | 11 +-
.../TestAppStateContainerFailure.groovy | 166 ------------
.../appstate/TestAppStateDynamicRoles.groovy | 93 -------
.../TestAppStateRebuildOnAMRestart.groovy | 124 ---------
.../appstate/TestAppStateRolePlacement.groovy | 99 -------
.../appstate/TestAppStateRoleRelease.groovy | 82 ------
.../TestContainerResourceAllocations.groovy | 108 --------
.../model/appstate/TestFlexDynamicRoles.groovy | 189 --------------
.../TestMockAppStateContainerFailure.groovy | 166 ++++++++++++
.../TestMockAppStateDynamicRoles.groovy | 94 +++++++
.../TestMockAppStateFlexDynamicRoles.groovy | 190 ++++++++++++++
.../appstate/TestMockAppStateFlexing.groovy | 116 +++++++++
.../TestMockAppStateRMOperations.groovy | 180 +++++++++++++
.../TestMockAppStateRebuildOnAMRestart.groovy | 125 +++++++++
.../TestMockAppStateRolePlacement.groovy | 102 ++++++++
.../appstate/TestMockAppStateRoleRelease.groovy | 82 ++++++
.../TestMockContainerResourceAllocations.groovy | 108 ++++++++
.../model/appstate/TestMockFlexing.groovy | 128 ---------
.../model/appstate/TestMockRMOperations.groovy | 176 -------------
.../history/TestFindNodesForNewInstances.groovy | 128 ---------
.../history/TestFindNodesForRelease.groovy | 146 -----------
.../model/history/TestHistoryRW.groovy | 258 -------------------
.../model/history/TestHistoryRWOrdering.groovy | 145 -----------
.../model/history/TestNIComparators.groovy | 77 ------
.../TestOutstandingRequestTracker.groovy | 73 ------
.../TestRoleHistoryContainerEvents.groovy | 17 +-
...stRoleHistoryFindNodesForNewInstances.groovy | 128 +++++++++
.../history/TestRoleHistoryNIComparators.groovy | 77 ++++++
...tRoleHistoryOutstandingRequestTracker.groovy | 73 ++++++
.../model/history/TestRoleHistoryRW.groovy | 258 +++++++++++++++++++
.../history/TestRoleHistoryRWOrdering.groovy | 145 +++++++++++
.../model/mock/BaseMockAppStateTest.groovy | 3 +-
.../model/mock/MockProviderService.groovy | 69 +++--
.../model/mock/MockRMOperationHandler.groovy | 8 +-
.../appmaster/model/mock/MockYarnEngine.groovy | 6 +-
.../web/rest/agent/TestAMAgentWebServices.java | 3 +-
.../management/TestAMManagementWebServices.java | 3 +-
60 files changed, 2544 insertions(+), 2364 deletions(-)
----------------------------------------------------------------------
[27/50] [abbrv] git commit: Merge branch 'develop' of
https://git-wip-us.apache.org/repos/asf/incubator-slider into develop
Posted by st...@apache.org.
Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ec760682
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ec760682
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ec760682
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: ec76068292dd5d9c53ce3983dbbbdea1bf793558
Parents: d19c802 533b229
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 6 19:25:24 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 6 19:25:24 2014 +0100
----------------------------------------------------------------------
.../java/org/apache/slider/server/appmaster/state/AppState.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ec760682/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
[03/50] [abbrv] git commit: SLIDER-283 Add role for HBase THRIFT
gateway
Posted by st...@apache.org.
SLIDER-283 Add role for HBase THRIFT gateway
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a9a91b46
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a9a91b46
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a9a91b46
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: a9a91b46662ecf969cbc7c5e69cef214fc91e086
Parents: 0b52d4b
Author: tedyu <yu...@gmail.com>
Authored: Tue Aug 5 09:32:53 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Tue Aug 5 09:32:53 2014 -0700
----------------------------------------------------------------------
app-packages/hbase/appConfig.json | 2 +-
app-packages/hbase/metainfo.xml | 15 +++++
.../hbase/package/scripts/hbase_service.py | 2 +
.../hbase/package/scripts/hbase_thrift.py | 62 ++++++++++++++++++++
app-packages/hbase/package/scripts/params.py | 1 +
app-packages/hbase/resources.json | 5 ++
.../providers/hbase/HBaseClientProvider.java | 2 +-
.../slider/providers/hbase/HBaseKeys.java | 5 ++
.../providers/hbase/HBaseProviderService.java | 5 ++
.../slider/providers/hbase/HBaseRoles.java | 3 +
10 files changed, 100 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/app-packages/hbase/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json
index 8b0825d..f0ab144 100644
--- a/app-packages/hbase/appConfig.json
+++ b/app-packages/hbase/appConfig.json
@@ -23,7 +23,7 @@
"site.global.ganglia_server_host": "${NN_HOST}",
"site.global.ganglia_server_port": "8667",
"site.global.ganglia_server_id": "Application1",
- "site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
+ "site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
"site.hbase-site.hbase.hstore.flush.retries.number": "120",
"site.hbase-site.hbase.client.keyvalue.maxsize": "10485760",
"site.hbase-site.hbase.hstore.compactionThreshold": "3",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/app-packages/hbase/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase/metainfo.xml b/app-packages/hbase/metainfo.xml
index c14b06f..ee6ddea 100644
--- a/app-packages/hbase/metainfo.xml
+++ b/app-packages/hbase/metainfo.xml
@@ -45,6 +45,10 @@
<value>http://${HBASE_REST_HOST}:${site.global.hbase_rest_port}</value>
</export>
<export>
+ <name>org.apache.slider.hbase.thrift</name>
+ <value>http://${HBASE_THRIFT_HOST}:${site.global.hbase_thrift_port}</value>
+ </export>
+ <export>
<name>org.apache.slider.metrics</name>
<value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value>
</export>
@@ -63,6 +67,7 @@
<commandOrder>
<command>HBASE_MASTER-START</command>
<requires>HBASE_REST-INSTALLED</requires>
+ <requires>HBASE_THRIFT-INSTALLED</requires>
</commandOrder>
</commandOrders>
<components>
@@ -99,6 +104,16 @@
</component>
<component>
+ <name>HBASE_THRIFT</name>
+ <category>SLAVE</category>
+ <minInstanceCount>1</minInstanceCount>
+ <commandScript>
+ <script>scripts/hbase_thrift.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
+
+ <component>
<name>HBASE_CLIENT</name>
<category>CLIENT</category>
<minInstanceCount>0</minInstanceCount>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/app-packages/hbase/package/scripts/hbase_service.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/scripts/hbase_service.py b/app-packages/hbase/package/scripts/hbase_service.py
index cb9c599..01d38b3 100644
--- a/app-packages/hbase/package/scripts/hbase_service.py
+++ b/app-packages/hbase/package/scripts/hbase_service.py
@@ -37,6 +37,8 @@ def hbase_service(
daemon_cmd = format("{cmd} start {role}")
if name == 'rest':
daemon_cmd = format("{daemon_cmd} -p {rest_port}")
+ elif name == 'thrift':
+ daemon_cmd = format("{daemon_cmd} -p {thrift_port}")
no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
elif action == 'stop':
daemon_cmd = format("{cmd} stop {role} && rm -f {pid_file}")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/app-packages/hbase/package/scripts/hbase_thrift.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/scripts/hbase_thrift.py b/app-packages/hbase/package/scripts/hbase_thrift.py
new file mode 100644
index 0000000..84bfc62
--- /dev/null
+++ b/app-packages/hbase/package/scripts/hbase_thrift.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+from hbase import hbase
+from hbase_service import hbase_service
+
+
+class HbaseThrift(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ hbase(name='thrift')
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env) # for security
+
+ hbase_service( 'thrift',
+ action = 'start'
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ hbase_service( 'thrift',
+ action = 'stop'
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ pid_file = format("{pid_dir}/hbase-{hbase_user}-thrift.pid")
+ check_process_status(pid_file)
+
+if __name__ == "__main__":
+ HbaseThrift().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/app-packages/hbase/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/scripts/params.py b/app-packages/hbase/package/scripts/params.py
index ef48782..2d05289 100644
--- a/app-packages/hbase/package/scripts/params.py
+++ b/app-packages/hbase/package/scripts/params.py
@@ -58,6 +58,7 @@ ganglia_server_host = default('/configurations/global/ganglia_server_host', '')
ganglia_server_port = default('/configurations/global/ganglia_server_port', '8663')
rest_port = config['configurations']['global']['hbase_rest_port']
+thrift_port = config['configurations']['global']['hbase_thrift_port']
if security_enabled:
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/app-packages/hbase/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/resources.json b/app-packages/hbase/resources.json
index 9925c9e..2d2b4f9 100644
--- a/app-packages/hbase/resources.json
+++ b/app-packages/hbase/resources.json
@@ -22,5 +22,10 @@
"yarn.component.instances": "1",
"yarn.memory": "256"
}
+ "HBASE_THRIFT": {
+ "yarn.role.priority": "3",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
index 39ab16f..fde2bf7 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
@@ -239,7 +239,7 @@ public class HBaseClientProvider extends AbstractClientProvider implements
providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_MASTER,
0, -1);
- providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_REST_GATEWAY,
+ providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_THRIFT_GATEWAY,
0, -1);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
index f91e527..9c38fb2 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
@@ -28,11 +28,16 @@ public interface HBaseKeys {
String ROLE_REST_GATEWAY = "REST";
+ String ROLE_THRIFT_GATEWAY = "THRIFT";
+
/** {@value */
String REGION_SERVER = "regionserver";
/** {@value */
String REST_GATEWAY = "rest";
+
+ /** {@value */
+ String THRIFT_GATEWAY = "thrift";
/**
* What is the command for hbase to print a version: {@value}
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index 6fa7ca8..953ee79 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -196,6 +196,11 @@ public class HBaseProviderService extends AbstractProviderService implements
logfile = "/rest-gateway.txt";
break;
+ case ROLE_THRIFT_GATEWAY:
+ roleCommand = THRIFT_GATEWAY;
+
+ logfile = "/thrift-gateway.txt";
+ break;
default:
throw new SliderInternalStateException("Cannot start role %s", role);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
index 41beb0f..2cc457b 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
@@ -39,6 +39,8 @@ public class HBaseRoles {
public static final int KEY_REST_GATEWAY = SliderKeys.ROLE_AM_PRIORITY_INDEX + 3;
+ public static final int KEY_THRIFT_GATEWAY = SliderKeys.ROLE_AM_PRIORITY_INDEX + 4;
+
/**
* Initialize role list
*/
@@ -46,6 +48,7 @@ public class HBaseRoles {
ROLES.add(new ProviderRole(HBaseKeys.ROLE_WORKER, KEY_WORKER));
ROLES.add(new ProviderRole(HBaseKeys.ROLE_MASTER, KEY_MASTER));
ROLES.add(new ProviderRole(HBaseKeys.ROLE_REST_GATEWAY, KEY_REST_GATEWAY));
+ ROLES.add(new ProviderRole(HBaseKeys.ROLE_THRIFT_GATEWAY, KEY_THRIFT_GATEWAY));
}
[29/50] [abbrv] git commit: SLIDER-298. Add an application package
template
Posted by st...@apache.org.
SLIDER-298. Add an application package template
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/373aed3d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/373aed3d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/373aed3d
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 373aed3d61744342f76b7a859d3660fc52cc6709
Parents: 609fcd3
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Aug 6 22:40:01 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Aug 6 22:40:01 2014 -0700
----------------------------------------------------------------------
app-packages/app-pkg-template/myapp-1.0.0.zip | Bin 5455 -> 0 bytes
1 file changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/373aed3d/app-packages/app-pkg-template/myapp-1.0.0.zip
----------------------------------------------------------------------
diff --git a/app-packages/app-pkg-template/myapp-1.0.0.zip b/app-packages/app-pkg-template/myapp-1.0.0.zip
deleted file mode 100644
index 80dbbb7..0000000
Binary files a/app-packages/app-pkg-template/myapp-1.0.0.zip and /dev/null differ
[39/50] [abbrv] git commit: SLIDER-302 providers switch to using
queued action, methods and interfaces changed appropriately
Posted by st...@apache.org.
SLIDER-302 providers switch to using queued action, methods and interfaces changed appropriately
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8f777f31
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8f777f31
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8f777f31
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 8f777f31ceb46e7b84a32524789b48664080bf41
Parents: e75f605
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 8 17:53:21 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 8 17:53:21 2014 +0100
----------------------------------------------------------------------
.../providers/AbstractProviderService.java | 12 ++---
.../slider/providers/ProviderService.java | 16 ++-----
.../providers/agent/AgentProviderService.java | 15 +++----
.../providers/agent/HeartbeatMonitor.java | 9 +---
.../server/appmaster/AMViewForProviders.java | 46 -------------------
.../appmaster/ContainerStartOperation.java | 40 -----------------
.../server/appmaster/SliderAppMaster.java | 39 +++++++++-------
.../server/appmaster/actions/ActionHalt.java | 4 +-
.../appmaster/actions/ActionStartContainer.java | 4 +-
.../appmaster/actions/ActionStopQueue.java | 4 +-
.../appmaster/actions/ActionStopSlider.java | 4 +-
.../server/appmaster/actions/AsyncAction.java | 2 +-
.../actions/ProviderReportedContainerLoss.java | 47 ++++++++++++++++++++
.../server/appmaster/actions/QueueExecutor.java | 14 +++---
.../server/appmaster/actions/TestActions.groovy | 2 +-
.../model/mock/MockProviderService.groovy | 4 +-
.../providers/agent/TestHeartbeatMonitor.java | 6 ---
.../publisher/TestAgentProviderService.java | 6 +--
18 files changed, 106 insertions(+), 168 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index b431a10..89b862e 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -33,7 +33,7 @@ import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.main.ExitCodeProvider;
import org.apache.slider.core.registry.info.RegisteredEndpoint;
import org.apache.slider.core.registry.info.ServiceInstanceData;
-import org.apache.slider.server.appmaster.AMViewForProviders;
+import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
@@ -71,8 +71,8 @@ public abstract class AbstractProviderService
protected AgentRestOperations restOps;
protected RegistryViewForProviders registry;
protected ServiceInstanceData registryInstanceData;
- protected AMViewForProviders amView;
protected URL amWebAPI;
+ protected QueueAccess queueAccess;
public AbstractProviderService(String name) {
super(name);
@@ -87,8 +87,8 @@ public abstract class AbstractProviderService
return amState;
}
- public AMViewForProviders getAppMaster() {
- return amView;
+ public QueueAccess getQueueAccess() {
+ return queueAccess;
}
public void setAmState(StateAccessForProviders amState) {
@@ -98,11 +98,11 @@ public abstract class AbstractProviderService
@Override
public void bind(StateAccessForProviders stateAccessor,
RegistryViewForProviders reg,
- AMViewForProviders amView,
+ QueueAccess queueAccess,
List<Container> liveContainers) {
this.amState = stateAccessor;
this.registry = reg;
- this.amView = amView;
+ this.queueAccess = queueAccess;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index 1466350..e717158 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -32,7 +32,7 @@ import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.core.main.ExitCodeProvider;
import org.apache.slider.core.registry.info.ServiceInstanceData;
-import org.apache.slider.server.appmaster.AMViewForProviders;
+import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
@@ -146,17 +146,9 @@ public interface ProviderService extends ProviderCore,
*/
Map<String, String> buildMonitorDetails(ClusterDescription clusterSpec);
- /**
- * bind operation -invoked before the service is started
- * @param stateAccessor interface offering read access to the state
- * @param registry registry view
- * @param amView AM access
- * @param liveContainers list of live containers. If non-empty, the AM was
- * restarted with one more running containers.
- */
- void bind(StateAccessForProviders stateAccessor,
- RegistryViewForProviders registry,
- AMViewForProviders amView,
+ public void bind(StateAccessForProviders stateAccessor,
+ RegistryViewForProviders reg,
+ QueueAccess queueAccess,
List<Container> liveContainers);
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 4d54509..419a454 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -58,7 +58,7 @@ import org.apache.slider.providers.agent.application.metadata.ExportGroup;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.providers.agent.application.metadata.OSPackage;
import org.apache.slider.providers.agent.application.metadata.OSSpecific;
-import org.apache.slider.server.appmaster.AMViewForProviders;
+import org.apache.slider.server.appmaster.actions.ProviderReportedContainerLoss;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
@@ -591,19 +591,16 @@ public class AgentProviderService extends AbstractProviderService implements
/**
* Lost heartbeat from the container - release it and ask for a replacement
- *
- *
- * @param label
+ * (async operation)
+ * @param label
* @param containerId
*
- * @return outcome of the operation
*/
- protected AMViewForProviders.ContainerLossReportOutcomes lostContainer(
+ protected void lostContainer(
String label,
- ContainerId containerId) throws
- SliderException {
+ ContainerId containerId) {
getComponentStatuses().remove(label);
- return getAppMaster().providerLostContainer(containerId);
+ getQueueAccess().put(new ProviderReportedContainerLoss(containerId));
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java b/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java
index cb22d2e..0a1beca 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java
@@ -19,7 +19,6 @@ package org.apache.slider.providers.agent;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.core.exceptions.SliderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -114,13 +113,7 @@ public class HeartbeatMonitor implements Runnable {
timeSinceLastHeartbeat);
ContainerId containerId =
componentInstanceState.getContainerId();
- try {
- provider.lostContainer(containerLabel, containerId);
- } catch (SliderException e) {
- log.info(
- "Error while requesting container release for {}. Message: {}",
- containerId, e, e);
- }
+ provider.lostContainer(containerLabel, containerId);
}
break;
case HEARTBEAT_LOST:
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/server/appmaster/AMViewForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/AMViewForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/AMViewForProviders.java
deleted file mode 100644
index f426551..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/AMViewForProviders.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.core.exceptions.SliderException;
-
-/** Operations available to a provider from AppMaster */
-public interface AMViewForProviders {
-
- /**
- * Outcomes from container loss
- */
- enum ContainerLossReportOutcomes {
- /**
- * The container doesn't exist...either it wasn't in use or it
- * has been released
- */
- CONTAINER_NOT_IN_USE,
-
- /**
- * The container is known about and a review has been initated
- */
- CONTAINER_LOST_REVIEW_INITIATED,
- }
-
- /** Provider can ask AppMaster to release a specific container */
- ContainerLossReportOutcomes providerLostContainer(ContainerId containerId) throws SliderException;
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/server/appmaster/ContainerStartOperation.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/ContainerStartOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/ContainerStartOperation.java
deleted file mode 100644
index 50c99f3..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/ContainerStartOperation.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-
-/**
- * Callback for container start requests
- */
-public interface ContainerStartOperation {
- /**
- * Add a node to the list of starting
- * nodes then trigger the NM start operation with the given
- * launch context
- * @param container container
- * @param ctx context
- * @param instance node details
- */
- void startContainer(Container container,
- ContainerLaunchContext ctx,
- RoleInstance instance) ;
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index b82967f..52f7f8f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -80,6 +80,7 @@ import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.exceptions.SliderInternalStateException;
import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
+import org.apache.slider.core.main.ExitCodeProvider;
import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.slider.core.main.RunService;
import org.apache.slider.core.main.ServiceLauncher;
@@ -168,9 +169,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
SliderClusterProtocol,
ServiceStateChangeListener,
RoleKeys,
- ProviderCompleted,
- ContainerStartOperation,
- AMViewForProviders {
+ ProviderCompleted {
protected static final Logger log =
LoggerFactory.getLogger(SliderAppMaster.class);
@@ -763,9 +762,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//Give the provider restricted access to the state, registry
- providerService.bind(stateForProviders, registry, this,
+ providerService.bind(stateForProviders, registry, actionQueues,
liveContainers);
- sliderAMProvider.bind(stateForProviders, registry, this,
+ sliderAMProvider.bind(stateForProviders, registry, actionQueues,
liveContainers);
// now do the registration
@@ -1219,7 +1218,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
public void onError(Throwable e) {
//callback says it's time to finish
LOG_YARN.error("AMRMClientAsync.onError() received " + e, e);
- signalAMComplete(EXIT_EXCEPTION_THROWN, "AMRMClientAsync.onError() received " + e);
+ signalAMComplete(EXIT_EXCEPTION_THROWN,
+ "AMRMClientAsync.onError() received " + e);
}
/* =================================================================== */
@@ -1484,18 +1484,13 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
}
-
- /* =================================================================== */
- /* ProviderAMOperations */
- /* =================================================================== */
-
/**
* report container loss. If this isn't already known about, react
*
* @param containerId id of the container which has failed
* @throws SliderException
*/
- public synchronized ContainerLossReportOutcomes providerLostContainer(
+ public synchronized void providerLostContainer(
ContainerId containerId)
throws SliderException {
log.info("containerLostContactWithProvider: container {} lost",
@@ -1506,10 +1501,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// ask for more containers if needed
log.info("Container released; triggering review");
reviewRequestAndReleaseNodes();
- return ContainerLossReportOutcomes.CONTAINER_LOST_REVIEW_INITIATED;
} else {
log.info("Container not in active set - ignoring");
- return ContainerLossReportOutcomes.CONTAINER_NOT_IN_USE;
}
}
@@ -1570,7 +1563,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* @param ctx context
* @param instance node details
*/
- @Override // ContainerStartOperation
public void startContainer(Container container,
ContainerLaunchContext ctx,
RoleInstance instance) {
@@ -1676,6 +1668,22 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
actionQueues.delayedActions.add(action);
}
+
+ /**
+ * Handle any exception in a thread. If the exception provides an exit
+ * code, that is the one that will be used
+ * @param thread thread throwing the exception
+ * @param exception exception
+ */
+ public void onExceptionInThread(Thread thread, Exception exception) {
+ log.error("Exception in {}: {}", thread.getName(), exception, exception);
+ int exitCode = EXIT_EXCEPTION_THROWN;
+ if (exception instanceof ExitCodeProvider) {
+ exitCode = ((ExitCodeProvider) exception).getExitCode();
+ }
+ signalAMComplete(exitCode, exception.toString());
+ }
+
/**
* This is the main entry point for the service launcher.
* @param args command line arguments.
@@ -1691,4 +1699,5 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//now have the service launcher do its work
ServiceLauncher.serviceMain(extendedArgs);
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
index a16d207..1468cd8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
@@ -21,8 +21,6 @@ package org.apache.slider.server.appmaster.actions;
import org.apache.hadoop.util.ExitUtil;
import org.apache.slider.server.appmaster.SliderAppMaster;
-import java.io.IOException;
-
public class ActionHalt extends AsyncAction {
private final int status;
@@ -37,7 +35,7 @@ public class ActionHalt extends AsyncAction {
}
@Override
- public void execute(SliderAppMaster appMaster) throws IOException {
+ public void execute(SliderAppMaster appMaster) throws Exception {
ExitUtil.halt(status, text);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
index 2d643ed..6179f35 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStartContainer.java
@@ -23,8 +23,6 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.state.RoleInstance;
-import java.io.IOException;
-
public class ActionStartContainer extends AsyncAction {
private final Container container;
@@ -43,7 +41,7 @@ public class ActionStartContainer extends AsyncAction {
}
@Override
- public void execute(SliderAppMaster appMaster) throws IOException {
+ public void execute(SliderAppMaster appMaster) throws Exception {
appMaster.startContainer(container, ctx, instance);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
index b64fd73..a11bccb 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopQueue.java
@@ -20,8 +20,6 @@ package org.apache.slider.server.appmaster.actions;
import org.apache.slider.server.appmaster.SliderAppMaster;
-import java.io.IOException;
-
/**
* Action to tell a queue executor to stop -after handing this on/executing it
*/
@@ -32,7 +30,7 @@ public class ActionStopQueue extends AsyncAction {
}
@Override
- public void execute(SliderAppMaster appMaster) throws IOException {
+ public void execute(SliderAppMaster appMaster) throws Exception {
// no-op
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
index cffa3f2..0e6dc25 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionStopSlider.java
@@ -21,8 +21,6 @@ package org.apache.slider.server.appmaster.actions;
import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.slider.server.appmaster.SliderAppMaster;
-import java.io.IOException;
-
public class ActionStopSlider extends AsyncAction {
public ActionStopSlider(String message,
int delay) {
@@ -30,7 +28,7 @@ public class ActionStopSlider extends AsyncAction {
}
@Override
- public void execute(SliderAppMaster appMaster) throws IOException {
+ public void execute(SliderAppMaster appMaster) throws Exception {
String message = name;
SliderAppMaster.getLog().info("SliderAppMasterApi.stopCluster: {}",
message);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
index 09bb517..a755064 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
@@ -112,7 +112,7 @@ public abstract class AsyncAction implements Delayed {
* @param appMaster
* @throws IOException
*/
- public abstract void execute(SliderAppMaster appMaster) throws IOException;
+ public abstract void execute(SliderAppMaster appMaster) throws Exception;
public enum ActionAttributes {
SHRINKS_CLUSTER,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
new file mode 100644
index 0000000..63b40d5
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ProviderReportedContainerLoss.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+
+/**
+ * Report container loss to the AM
+ * {@link SliderAppMaster#providerLostContainer(ContainerId)}
+ */
+public class ProviderReportedContainerLoss extends AsyncAction {
+
+ private final ContainerId containerId;
+
+ public ProviderReportedContainerLoss(ContainerId containerId) {
+ super("lost container " + containerId);
+ this.containerId = containerId;
+ }
+
+ public ProviderReportedContainerLoss(
+ ContainerId containerId, int delayMillis) {
+ super("lost container " + containerId, delayMillis);
+ this.containerId = containerId;
+ }
+
+ @Override
+ public void execute(SliderAppMaster appMaster) throws Exception {
+ appMaster.providerLostContainer(containerId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
index 7eeaddf..072f324 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueExecutor.java
@@ -24,8 +24,6 @@ import org.apache.slider.server.appmaster.SliderAppMaster;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-
/**
* Executor for async actions - hands them off to the AM as
* appropriate
@@ -41,7 +39,7 @@ public class QueueExecutor implements Runnable {
QueueService actionQueues) {
Preconditions.checkNotNull(appMaster);
Preconditions.checkNotNull(actionQueues);
-
+
this.appMaster = appMaster;
this.actionQueues = actionQueues;
}
@@ -52,7 +50,7 @@ public class QueueExecutor implements Runnable {
this.appMaster = null;
this.actionQueues = actionQueues;
}
-
+
/**
* Run until the queue has been told to stop
*/
@@ -67,9 +65,11 @@ public class QueueExecutor implements Runnable {
take.execute(appMaster);
} while (!(take instanceof ActionStopQueue));
log.info("Queue Executor run() stopped");
- } catch (InterruptedException | IOException e) {
- log.error("Exception processing {}: {}", take, e, e);
-
+ } catch (Exception e) {
+ log.error("Exception processing {}: {}", take, e, e);
+ if (appMaster != null) {
+ appMaster.onExceptionInThread(Thread.currentThread(), e);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
index 7e6338b..5dffea0 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
@@ -159,7 +159,7 @@ class TestActions {
}
@Override
- public void execute(SliderAppMaster appMaster) throws IOException {
+ public void execute(SliderAppMaster appMaster) throws Exception {
log.info("Executing $name");
executed.set(true);
executionTimeNanos.set(System.nanoTime())
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
index c46b828..458cfac 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
@@ -35,7 +35,7 @@ import org.apache.slider.core.launch.ContainerLauncher
import org.apache.slider.core.registry.info.ServiceInstanceData
import org.apache.slider.providers.ProviderRole
import org.apache.slider.providers.ProviderService
-import org.apache.slider.server.appmaster.AMViewForProviders
+import org.apache.slider.server.appmaster.actions.QueueAccess
import org.apache.slider.server.appmaster.state.ContainerReleaseSelector
import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector
import org.apache.slider.server.appmaster.state.StateAccessForProviders
@@ -202,7 +202,7 @@ class MockProviderService implements ProviderService {
void bind(
StateAccessForProviders stateAccessor,
RegistryViewForProviders registry,
- AMViewForProviders amView,
+ QueueAccess queueAccess,
List<Container> liveContainers) {
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
index b16b145..1ace44e 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
@@ -17,7 +17,6 @@
package org.apache.slider.providers.agent;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.server.appmaster.AMViewForProviders;
import org.apache.slider.server.appmaster.model.mock.MockContainerId;
import org.junit.Assert;
import org.junit.Test;
@@ -98,11 +97,6 @@ public class TestHeartbeatMonitor {
slaveState.setState(State.STARTED);
slaveState.heartbeat(now);
expect(provider.getComponentStatuses()).andReturn(statuses).anyTimes();
- expect(provider.lostContainer(slaveLabel, slaveContainer)).andReturn(
- AMViewForProviders.ContainerLossReportOutcomes.CONTAINER_LOST_REVIEW_INITIATED).once();
- // expect the second iteration to report no container any more
- expect(provider.lostContainer(slaveLabel, slaveContainer)).andReturn(
- AMViewForProviders.ContainerLossReportOutcomes.CONTAINER_NOT_IN_USE).once();
replay(provider);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8f777f31/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
index 5579592..a97e351 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
@@ -18,7 +18,7 @@ package org.apache.slider.server.appmaster.web.rest.publisher;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.slider.providers.agent.AgentProviderService;
-import org.apache.slider.server.appmaster.AMViewForProviders;
+import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.services.registry.RegistryViewForProviders;
import org.slf4j.Logger;
@@ -43,9 +43,9 @@ public class TestAgentProviderService extends AgentProviderService {
@Override
public void bind(StateAccessForProviders stateAccessor,
RegistryViewForProviders reg,
- AMViewForProviders amView,
+ QueueAccess queueAccess,
List<Container> liveContainers) {
- super.bind(stateAccessor, reg, amView, liveContainers);
+ super.bind(stateAccessor, reg, queueAccess, liveContainers);
Map<String,String> dummyProps = new HashMap<>();
dummyProps.put("prop1", "val1");
dummyProps.put("prop2", "val2");
[50/50] [abbrv] git commit: Merge branch 'develop' into
feature/SLIDER-151_REST_API
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-151_REST_API
Conflicts:
slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bfe71dc0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bfe71dc0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bfe71dc0
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: bfe71dc0531bf18021410ef9be1096be7d39c836
Parents: 621a981 8ac6748
Author: Steve Loughran <st...@apache.org>
Authored: Sat Aug 9 14:04:18 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Sat Aug 9 14:04:18 2014 +0100
----------------------------------------------------------------------
app-packages/accumulo/appConfig.json | 5 +-
app-packages/accumulo/pom.xml | 93 ++-
app-packages/accumulo/resources.json | 15 +-
app-packages/accumulo/src/assembly/accumulo.xml | 1 +
.../AccumuloAgentCommandTestBase.groovy | 52 ++
.../funtest/accumulo/AccumuloBasicIT.groovy | 117 ++++
.../accumulo/AccumuloMonitorSSLIT.groovy | 73 +++
.../funtest/accumulo/AccumuloReadWriteIT.groovy | 116 ++++
.../StubToForceGroovyTestsToCompile.java | 22 +
.../test/resources/appConfig_monitor_ssl.json | 62 ++
.../accumulo/src/test/resources/resources.json | 36 ++
app-packages/app-pkg-template/README.txt | 34 ++
app-packages/app-pkg-template/appConfig.json | 21 +
app-packages/app-pkg-template/metainfo.xml | 57 ++
.../package/files/myapp-1.0.0.tar.REPLACE | 16 +
.../package/scripts/myapp_master.py | 46 ++
.../app-pkg-template/package/scripts/params.py | 30 +
app-packages/app-pkg-template/resources.json | 16 +
.../command-logger/application-pkg/pom.xml | 7 +-
.../command-logger/slider-pkg/appConfig.json | 1 -
app-packages/command-logger/slider-pkg/pom.xml | 2 +-
app-packages/hbase/README.txt | 13 +-
app-packages/hbase/appConfig.json | 13 +-
app-packages/hbase/get-hbase-site.sh | 17 +
app-packages/hbase/links.py | 35 ++
app-packages/hbase/metainfo.xml | 48 ++
.../hbase/package/scripts/hbase_regionserver.py | 3 -
.../hbase/package/scripts/hbase_rest.py | 62 ++
.../hbase/package/scripts/hbase_service.py | 6 +
.../hbase/package/scripts/hbase_thrift.py | 62 ++
.../hbase/package/scripts/hbase_thrift2.py | 62 ++
app-packages/hbase/package/scripts/params.py | 4 +
app-packages/hbase/pom.xml | 154 ++++-
app-packages/hbase/resources.json | 23 +-
app-packages/hbase/src/assembly/hbase.xml | 1 +
.../hbase/HBaseAgentCommandTestBase.groovy | 52 ++
.../slider/funtest/hbase/HBaseBasicIT.groovy | 114 ++++
.../funtest/hbase/HBaseMonitorSSLIT.groovy | 73 +++
.../hbase/StubToForceGroovyTestsToCompile.java | 22 +
.../test/resources/appConfig_monitor_ssl.json | 68 +++
.../hbase/src/test/resources/resources.json | 21 +
app-packages/memcached-win/README.txt | 36 ++
app-packages/memcached-win/appConfig.json | 26 +
app-packages/memcached-win/metainfo.xml | 57 ++
.../jmemcached-cli-1.0.0.jar.REPLACEME | 16 +
.../jmemcached-core-1.0.0.jar.REPLACEME | 16 +
.../memcached-win/package/scripts/memcached.py | 54 ++
.../memcached-win/package/scripts/params.py | 33 ++
app-packages/memcached-win/resources.json | 16 +
app-packages/memcached/README.txt | 35 ++
app-packages/memcached/appConfig.json | 25 +
app-packages/memcached/metainfo.xml | 57 ++
.../package/files/jmemcached-1.0.0.tar.REPLACE | 20 +
.../memcached/package/scripts/memcached.py | 53 ++
.../memcached/package/scripts/params.py | 33 ++
app-packages/memcached/resources.json | 16 +
app-packages/storm-v0_91/README.txt | 33 --
app-packages/storm-v0_91/appConfig.json | 126 ----
.../storm-v0_91/configuration/global.xml | 39 --
.../storm-v0_91/configuration/storm-site.xml | 587 -------------------
app-packages/storm-v0_91/ganglia_metrics.json | 31 -
app-packages/storm-v0_91/jmx_metrics.json | 31 -
app-packages/storm-v0_91/metainfo.xml | 145 -----
...pache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE | 16 -
.../storm-v0_91/package/scripts/drpc_server.py | 55 --
.../storm-v0_91/package/scripts/nimbus.py | 55 --
.../storm-v0_91/package/scripts/params.py | 58 --
.../storm-v0_91/package/scripts/rest_api.py | 57 --
.../storm-v0_91/package/scripts/service.py | 95 ---
.../package/scripts/status_params.py | 36 --
.../storm-v0_91/package/scripts/storm.py | 50 --
.../storm-v0_91/package/scripts/supervisor.py | 61 --
.../storm-v0_91/package/scripts/ui_server.py | 55 --
.../storm-v0_91/package/scripts/yaml_config.py | 69 ---
.../package/templates/config.yaml.j2 | 48 --
.../package/templates/storm_jaas.conf.j2 | 26 -
app-packages/storm-v0_91/resources.json | 31 -
app-packages/storm/README.txt | 37 ++
app-packages/storm/appConfig.json | 126 ++++
app-packages/storm/configuration/global.xml | 39 ++
app-packages/storm/configuration/storm-site.xml | 587 +++++++++++++++++++
app-packages/storm/ganglia_metrics.json | 31 +
app-packages/storm/jmx_metrics.json | 31 +
app-packages/storm/metainfo.xml | 145 +++++
...pache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE | 16 +
.../storm/package/scripts/drpc_server.py | 55 ++
app-packages/storm/package/scripts/nimbus.py | 55 ++
app-packages/storm/package/scripts/params.py | 58 ++
app-packages/storm/package/scripts/rest_api.py | 57 ++
app-packages/storm/package/scripts/service.py | 95 +++
.../storm/package/scripts/status_params.py | 36 ++
app-packages/storm/package/scripts/storm.py | 50 ++
.../storm/package/scripts/supervisor.py | 61 ++
app-packages/storm/package/scripts/ui_server.py | 55 ++
.../storm/package/scripts/yaml_config.py | 69 +++
.../storm/package/templates/config.yaml.j2 | 48 ++
.../storm/package/templates/storm_jaas.conf.j2 | 26 +
app-packages/storm/resources.json | 31 +
pom.xml | 72 ++-
slider-agent/conf/agent.ini | 6 +-
slider-agent/pom.xml | 6 +-
.../src/main/python/agent/AgentConfig.py | 12 +-
.../src/main/python/agent/Controller.py | 32 +-
.../python/agent/CustomServiceOrchestrator.py | 114 +++-
slider-agent/src/main/python/agent/NetUtil.py | 8 +-
.../src/main/python/agent/ProcessHelper.py | 3 +-
.../src/main/python/agent/PythonExecutor.py | 7 +-
slider-agent/src/main/python/agent/main.py | 44 +-
slider-agent/src/main/python/agent/security.py | 7 +-
slider-agent/src/main/python/agent/shell.py | 102 +++-
.../python/resource_management/core/logger.py | 5 +-
.../core/providers/__init__.py | 5 +
.../core/providers/accounts.py | 20 +-
.../core/providers/system.py | 28 +-
.../core/providers/windows/__init__.py | 20 +
.../core/providers/windows/service.py | 65 ++
.../core/providers/windows/system.py | 190 ++++++
.../libraries/functions/__init__.py | 2 +-
.../libraries/functions/os_check.py | 166 ++++--
.../libraries/providers/__init__.py | 2 +
.../libraries/providers/monitor_webserver.py | 2 +-
.../libraries/script/script.py | 15 +-
.../src/test/python/agent/TestActionQueue.py | 2 +-
.../src/test/python/agent/TestController.py | 9 +-
.../agent/TestCustomServiceOrchestrator.py | 127 +++-
slider-agent/src/test/python/agent/TestGrep.py | 2 +-
slider-agent/src/test/python/agent/TestMain.py | 34 +-
.../src/test/python/agent/TestNetUtil.py | 2 +-
.../src/test/python/agent/TestRegistration.py | 10 +-
slider-agent/src/test/python/agent/TestShell.py | 33 +-
.../TestDirectoryResource.py | 2 +-
.../resource_management/TestGroupResource.py | 138 -----
.../TestGroupResource.py.disabled | 138 +++++
.../resource_management/TestUserResource.py | 192 ------
.../TestUserResource.py.disabled | 192 ++++++
slider-agent/src/test/python/unitTests.py | 3 +-
slider-assembly/pom.xml | 185 +++++-
slider-assembly/src/conf-hdp/log4j.properties | 58 ++
slider-assembly/src/conf-hdp/slider-client.xml | 77 +++
slider-assembly/src/conf/slider-client.xml | 3 -
slider-core/pom.xml | 17 +-
.../java/org/apache/slider/api/StatusKeys.java | 3 +
.../org/apache/slider/client/SliderClient.java | 151 ++++-
.../org/apache/slider/common/SliderKeys.java | 16 +-
.../common/SliderXMLConfKeysForTesting.java | 5 +
.../apache/slider/common/SliderXmlConfKeys.java | 2 +
.../common/params/AbstractActionArgs.java | 14 +-
.../slider/common/params/ActionUpdateArgs.java | 32 +
.../org/apache/slider/common/params/ArgOps.java | 7 +-
.../apache/slider/common/params/ClientArgs.java | 14 +-
.../apache/slider/common/params/CommonArgs.java | 11 +-
.../slider/common/params/SliderActions.java | 3 +
.../apache/slider/common/tools/Comparators.java | 58 ++
.../slider/common/tools/CoreFileSystem.java | 35 ++
.../apache/slider/common/tools/SliderUtils.java | 25 +-
.../slider/core/build/InstanceBuilder.java | 6 +-
.../slider/core/exceptions/ErrorStrings.java | 2 +-
.../slider/core/launch/AbstractLauncher.java | 3 +
.../slider/core/launch/AppMasterLauncher.java | 7 +-
.../core/launch/ClasspathConstructor.java | 1 -
.../core/launch/JavaCommandLineBuilder.java | 3 +
.../slider/core/main/ServiceLauncher.java | 116 +++-
.../registry/info/CustomRegistryConstants.java | 2 +-
.../slider/core/zk/BlockingZKWatcher.java | 8 +-
.../apache/slider/core/zk/ZKIntegration.java | 23 +-
.../providers/AbstractProviderService.java | 53 +-
.../slider/providers/ProviderService.java | 49 +-
.../providers/agent/AgentClientProvider.java | 22 +-
.../slider/providers/agent/AgentKeys.java | 6 +-
.../providers/agent/AgentProviderService.java | 577 +++++++++++-------
.../providers/agent/ComponentCommandOrder.java | 8 +-
.../providers/agent/ComponentInstanceState.java | 80 ++-
.../providers/agent/HeartbeatMonitor.java | 59 +-
.../agent/application/metadata/Application.java | 9 +
.../agent/application/metadata/Component.java | 13 +
.../application/metadata/MetainfoParser.java | 5 +
.../slideram/SliderAMClientProvider.java | 7 +-
.../slideram/SliderAMProviderService.java | 18 +-
.../server/appmaster/AMViewForProviders.java | 27 -
.../appmaster/AsyncRMOperationHandler.java | 51 --
.../appmaster/ContainerStartOperation.java | 40 --
.../server/appmaster/RoleLaunchService.java | 40 +-
.../server/appmaster/SliderAppMaster.java | 316 +++++++---
.../server/appmaster/actions/ActionHalt.java | 46 ++
.../appmaster/actions/ActionStartContainer.java | 51 ++
.../appmaster/actions/ActionStopQueue.java | 43 ++
.../appmaster/actions/ActionStopSlider.java | 39 ++
.../server/appmaster/actions/AsyncAction.java | 157 +++++
.../actions/ProviderReportedContainerLoss.java | 47 ++
.../actions/ProviderStartupCompleted.java | 37 ++
.../server/appmaster/actions/QueueAccess.java | 38 ++
.../server/appmaster/actions/QueueExecutor.java | 75 +++
.../server/appmaster/actions/QueueService.java | 76 +++
.../appmaster/actions/RenewingAction.java | 105 ++++
.../operations/AbstractRMOperation.java | 29 +
.../operations/AsyncRMOperationHandler.java | 51 ++
.../operations/ContainerReleaseOperation.java | 44 ++
.../operations/ContainerRequestOperation.java | 44 ++
.../ProviderNotifyingOperationHandler.java | 43 ++
.../operations/RMOperationHandler.java | 35 ++
.../operations/RMOperationHandlerActions.java | 28 +
.../appmaster/state/AbstractRMOperation.java | 31 -
.../slider/server/appmaster/state/AppState.java | 275 ++++++---
.../state/ContainerReleaseOperation.java | 44 --
.../state/ContainerReleaseSelector.java | 38 ++
.../state/ContainerRequestOperation.java | 44 --
.../MostRecentContainerReleaseSelector.java | 52 ++
.../server/appmaster/state/NodeEntry.java | 1 -
.../server/appmaster/state/NodeInstance.java | 12 +-
.../slider/server/appmaster/state/NodeMap.java | 54 --
.../appmaster/state/RMOperationHandler.java | 43 --
.../server/appmaster/state/RoleHistory.java | 20 +-
.../server/appmaster/state/RoleStatus.java | 4 +-
.../appmaster/state/SimpleReleaseSelector.java | 34 ++
.../server/appmaster/web/AgentService.java | 53 ++
.../server/appmaster/web/SliderAMWebApp.java | 2 +-
.../server/appmaster/web/SliderAmIpFilter.java | 19 +-
.../slider/server/appmaster/web/WebAppApi.java | 9 +-
.../server/appmaster/web/WebAppApiImpl.java | 16 +-
.../appmaster/web/rest/AMWebServices.java | 7 +-
.../server/appmaster/web/rest/RestPaths.java | 5 +-
.../appmaster/web/rest/agent/AgentResource.java | 25 +-
.../appmaster/web/rest/agent/AgentWebApp.java | 203 +++++++
.../web/rest/agent/AgentWebServices.java | 40 ++
.../server/services/curator/CuratorService.java | 2 +-
.../services/security/CertificateManager.java | 257 ++++++++
.../server/services/security/SecurityUtils.java | 225 +++++++
.../services/security/SignCertResponse.java | 67 +++
.../server/services/security/SignMessage.java | 54 ++
.../AbstractWorkflowExecutorService.java | 110 ----
.../services/workflow/ForkedProcessService.java | 6 +-
.../services/workflow/LongLivedProcess.java | 78 +--
.../workflow/WorkflowCallbackService.java | 3 +-
.../workflow/WorkflowExecutorService.java | 113 ++++
.../WorkflowScheduledExecutorService.java | 38 ++
.../server/services/workflow/package-info.java | 2 +-
.../providers/slideram/instance/appconf.json | 5 +-
.../main/resources/webapps/slideragent/.keep | 0
.../resources/webapps/static/yarn.dt.plugins.js | 4 +-
.../agent/AgentMiniClusterTestBase.groovy | 7 +-
.../agent/actions/TestActionExists.groovy | 9 +-
.../slider/agent/actions/TestActionList.groovy | 7 +-
.../agent/actions/TestActionStatus.groovy | 13 +-
.../agent/freezethaw/TestFreezeCommands.groovy | 9 +-
.../TestFreezeThawMasterlessAM.groovy | 8 +-
.../freezethaw/TestFreezeUnknownCluster.groovy | 3 +-
.../freezethaw/TestThawUnknownCluster.groovy | 3 +-
.../agent/standalone/TestBadAMHeap.groovy | 5 +-
.../agent/standalone/TestBadYarnQueue.groovy | 5 +-
.../standalone/TestBuildStandaloneAM.groovy | 62 +-
.../standalone/TestDestroyMasterlessAM.groovy | 154 -----
.../standalone/TestKillMasterlessAM.groovy | 74 ---
.../standalone/TestStandaloneAMDestroy.groovy | 154 +++++
.../standalone/TestStandaloneAMKill.groovy | 73 +++
.../standalone/TestStandaloneAMRestart.groovy | 96 +++
.../standalone/TestStandaloneAgentAM.groovy | 11 +-
.../TestStandaloneBadClusterName.groovy | 2 +-
.../standalone/TestStandaloneRegistryAM.groovy | 7 +-
.../slider/client/TestCommonArgParsing.groovy | 13 +
.../common/tools/TestZKIntegration.groovy | 53 ++
.../slider/providers/agent/AgentTestBase.groovy | 30 +
.../agent/TestAgentAMManagementWS.groovy | 65 +-
.../slider/providers/agent/TestAgentEcho.groovy | 4 +-
.../providers/agent/TestBuildBasicAgent.groovy | 171 ++++--
.../curator/TestRegistryRestResources.groovy | 10 +-
.../server/appmaster/actions/TestActions.groovy | 207 +++++++
.../TestAppStateContainerFailure.groovy | 166 ------
.../appstate/TestAppStateDynamicRoles.groovy | 91 ---
.../TestAppStateRebuildOnAMRestart.groovy | 122 ----
.../appstate/TestAppStateRolePlacement.groovy | 99 ----
.../appstate/TestAppStateRoleRelease.groovy | 82 ---
.../TestContainerResourceAllocations.groovy | 108 ----
.../model/appstate/TestFlexDynamicRoles.groovy | 187 ------
.../TestMockAppStateContainerFailure.groovy | 166 ++++++
.../TestMockAppStateDynamicRoles.groovy | 94 +++
.../TestMockAppStateFlexDynamicRoles.groovy | 190 ++++++
.../appstate/TestMockAppStateFlexing.groovy | 116 ++++
.../TestMockAppStateRMOperations.groovy | 180 ++++++
.../TestMockAppStateRebuildOnAMRestart.groovy | 125 ++++
.../TestMockAppStateRolePlacement.groovy | 102 ++++
.../appstate/TestMockAppStateRoleRelease.groovy | 82 +++
.../TestMockContainerResourceAllocations.groovy | 108 ++++
.../model/appstate/TestMockRMOperations.groovy | 213 -------
.../history/TestFindNodesForNewInstances.groovy | 128 ----
.../history/TestFindNodesForRelease.groovy | 146 -----
.../model/history/TestHistoryRW.groovy | 258 --------
.../model/history/TestHistoryRWOrdering.groovy | 145 -----
.../model/history/TestNIComparators.groovy | 77 ---
.../TestOutstandingRequestTracker.groovy | 73 ---
.../TestRoleHistoryContainerEvents.groovy | 19 +-
...stRoleHistoryFindNodesForNewInstances.groovy | 128 ++++
.../history/TestRoleHistoryNIComparators.groovy | 77 +++
...tRoleHistoryOutstandingRequestTracker.groovy | 73 +++
.../model/history/TestRoleHistoryRW.groovy | 258 ++++++++
.../history/TestRoleHistoryRWOrdering.groovy | 145 +++++
.../model/mock/BaseMockAppStateTest.groovy | 60 +-
.../appmaster/model/mock/MockContainer.groovy | 2 +-
.../model/mock/MockProviderService.groovy | 81 ++-
.../model/mock/MockRMOperationHandler.groovy | 8 +-
.../appmaster/model/mock/MockYarnEngine.groovy | 12 +-
.../publisher/TestPublisherRestResources.groovy | 5 +-
.../view/TestClusterSpecificationBlock.groovy | 2 +-
.../web/view/TestContainerStatsBlock.groovy | 2 +-
.../appmaster/web/view/TestIndexBlock.groovy | 2 +-
.../apache/slider/test/SliderTestBase.groovy | 22 +-
.../apache/slider/test/SliderTestUtils.groovy | 2 +-
.../slider/test/YarnMiniClusterTestBase.groovy | 36 +-
.../test/YarnZKMiniClusterTestBase.groovy | 22 +-
.../agent/TestAgentProviderService.java | 326 +++++-----
.../agent/TestComponentCommandOrder.java | 17 +-
.../agent/TestComponentInstanceState.java | 124 ++--
.../providers/agent/TestHeartbeatMonitor.java | 114 ++--
.../web/rest/agent/TestAMAgentWebServices.java | 219 +++----
.../management/TestAMManagementWebServices.java | 6 +-
.../publisher/TestAgentProviderService.java | 16 +-
.../security/TestCertificateManager.java | 60 ++
.../workflow/TestWorkflowExecutorService.java | 9 +-
slider-core/src/test/python/agent.py | 17 +-
slider-core/src/test/python/agent/main.py | 29 +-
slider-core/src/test/python/echo.py | 21 +-
slider-funtest/pom.xml | 34 +-
.../abstracttests/AbstractTestBuildSetup.groovy | 3 -
.../framework/AgentCommandTestBase.groovy | 190 ++++++
.../funtest/framework/CommandTestBase.groovy | 57 +-
.../funtest/framework/FuntestProperties.groovy | 3 -
.../slider/funtest/basic/BuildSetupIT.groovy | 36 ++
.../funtest/basic/ClusterConnectivityIT.groovy | 104 ++++
.../funtest/basic/SignCorrectionIT.groovy | 43 ++
.../slider/funtest/basic/TestBuildSetup.groovy | 36 --
.../basic/TestClusterConnectivity.groovy | 96 ---
.../funtest/basic/TestSignCorrection.groovy | 43 --
.../funtest/commands/ListCommandIT.groovy | 36 ++
.../funtest/commands/SimpleCommandsIT.groovy | 45 ++
.../funtest/commands/TestListCommand.groovy | 41 --
.../funtest/commands/TestSimpleCommands.groovy | 45 --
.../TestUnknownClusterOperations.groovy | 105 ----
.../commands/UnknownClusterOperationsIT.groovy | 100 ++++
.../lifecycle/AgentClusterLifecycleIT.groovy | 192 ++++++
.../lifecycle/AgentCommandTestBase.groovy | 233 --------
.../funtest/lifecycle/AgentFailures2IT.groovy | 104 ++++
.../funtest/lifecycle/AgentFailuresIT.groovy | 104 ++++
.../funtest/lifecycle/AppsThroughAgentIT.groovy | 84 +++
.../lifecycle/ClusterBuildDestroyIT.groovy | 82 +++
.../lifecycle/TestAgentClusterLifecycle.groovy | 191 ------
.../funtest/lifecycle/TestAgentFailures.groovy | 103 ----
.../funtest/lifecycle/TestAgentFailures2.groovy | 103 ----
.../lifecycle/TestAppsThroughAgent.groovy | 83 ---
.../lifecycle/TestClusterBuildDestroy.groovy | 81 ---
slider-install/pom.xml | 111 ++++
.../accumulo/accumulo-funtests/pom.xml | 29 +-
.../slider/providers/accumulo/TestStub.groovy | 32 -
.../funtest/AccumuloBuildSetupIT.groovy | 37 ++
.../accumulo/funtest/AccumuloCIIT.groovy | 109 ++++
.../funtest/AccumuloCommandTestBase.groovy | 53 +-
.../funtest/FunctionalAccumuloClusterIT.groovy | 145 +++++
.../FunctionalAccumuloM1T1GC1Mon1IT.groovy | 57 ++
.../funtest/TestAccumuloBuildSetup.groovy | 38 --
.../accumulo/funtest/TestAccumuloCI.groovy | 115 ----
.../TestFunctionalAccumuloCluster.groovy | 146 -----
.../TestFunctionalAccumuloM1T1GC1Mon1.groovy | 59 --
.../StubToForceGroovyTestsToCompile.java | 22 +
.../accumulo/slider-accumulo-provider/pom.xml | 10 +-
.../accumulo/AccumuloProviderService.java | 4 +
.../providers/accumulo/AccumuloTestBase.groovy | 2 +-
.../live/TestAccCorrectInstanceName.groovy | 4 +-
.../live/TestAccFlexTablets133Mgr113.groovy | 3 +-
.../accumulo/live/TestAccFlexTablets1to3.groovy | 2 +-
.../accumulo/live/TestAccFreezeThaw.groovy | 4 +-
.../accumulo/live/TestAccLiveHDFSArchive.groovy | 4 +-
.../live/TestAccLiveLocalArchive.groovy | 4 +-
.../accumulo/live/TestAccM1T1GC1Mon1.groovy | 3 +-
.../accumulo/live/TestAccM2T2GC1Mon1.groovy | 4 +-
.../accumulo/live/TestAccumuloAMWebApp.groovy | 4 +-
.../live/TestInvalidMonitorAddress.groovy | 3 +-
slider-providers/hbase/hbase-funtests/pom.xml | 29 +-
.../slider/providers/hbase/TestStub.groovy | 32 -
.../hbase/funtest/DemoHBaseCluster.groovy | 2 +-
.../funtest/FunctionalHBaseClusterIT.groovy | 223 +++++++
.../hbase/funtest/HBaseBuildSetupIT.groovy | 36 ++
.../funtest/HBaseClusterBuildDestroyIT.groovy | 87 +++
.../funtest/HBaseClusterLifecycleIT.groovy | 183 ++++++
.../hbase/funtest/HBaseCommandTestBase.groovy | 1 -
.../hbase/funtest/HBaseIntegrationIT.groovy | 69 +++
.../providers/hbase/funtest/HBaseLoadIT.groovy | 64 ++
.../hbase/funtest/HBaseNodeFailureIT.groovy | 135 +++++
.../providers/hbase/funtest/ImagesIT.groovy | 64 ++
.../funtest/TestFunctionalHBaseCluster.groovy | 223 -------
.../hbase/funtest/TestHBaseBuildSetup.groovy | 37 --
.../funtest/TestHBaseClusterBuildDestroy.groovy | 88 ---
.../funtest/TestHBaseClusterLifecycle.groovy | 183 ------
.../hbase/funtest/TestHBaseIntegration.groovy | 69 ---
.../hbase/funtest/TestHBaseLoad.groovy | 64 --
.../hbase/funtest/TestHBaseNodeFailure.groovy | 135 -----
.../providers/hbase/funtest/TestImages.groovy | 65 --
.../hbase/slider-hbase-provider/pom.xml | 20 +-
.../providers/hbase/HBaseClientProvider.java | 6 +
.../slider/providers/hbase/HBaseKeys.java | 14 +
.../providers/hbase/HBaseProviderService.java | 20 +-
.../slider/providers/hbase/HBaseRoles.java | 8 +
.../minicluster/HBaseMiniClusterTestBase.groovy | 150 +++--
.../TestFreezeThawClusterFromArchive.groovy | 3 +-
.../archives/TestLiveClusterFromArchive.groovy | 4 +-
.../TestLiveClusterFromArchiveOnHDFS.groovy | 2 +-
.../build/TestBuildThawClusterM1W1.groovy | 3 +-
.../failures/TestFailedRegionService.groovy | 7 +-
.../failures/TestFailureThreshold.groovy | 5 +-
.../failures/TestKilledHBaseAM.groovy | 3 +-
.../failures/TestKilledHBaseMaster.groovy | 4 +-
.../flexing/TestClusterFlex0To1.groovy | 2 +-
.../flexing/TestClusterFlex1To1.groovy | 4 +-
.../flexing/TestClusterFlex1To2.groovy | 4 +-
.../flexing/TestClusterFlex2DownTo1.groovy | 2 +-
.../flexing/TestClusterFlex2To5.groovy | 4 +-
.../flexing/TestClusterFlexDownMultiple.groovy | 58 ++
.../flexing/TestClusterFlexDownToZero.groovy | 2 +-
.../TestClusterFlexHMasterFlex1To2.groovy | 45 ++
.../flexing/TestHMasterFlex1To2.groovy | 45 --
...reezeReconfigureThawLiveRegionService.groovy | 3 +-
.../TestFreezeThawLiveRegionService.groovy | 3 +-
.../minicluster/live/Test2Master2RS.groovy | 3 +-
.../minicluster/live/TestHBaseMaster.groovy | 3 +-
.../live/TestHBaseMasterOnHDFS.groovy | 4 +-
.../live/TestHBaseMasterWithBadHeap.groovy | 3 +-
.../live/TestLiveRegionServiceOnHDFS.groovy | 4 +-
.../live/TestLiveTwoNodeRegionService.groovy | 4 +-
.../minicluster/live/TestTwoLiveClusters.groovy | 9 +-
.../masterless/TestRoleOptPropagation.groovy | 6 +-
.../TestSliderConfDirToMasterlessAM.groovy | 3 +-
.../src/test/resources/log4j.properties | 1 +
src/test/clusters/c6401/slider/log4j.properties | 30 -
.../clusters/morzine/slider/log4j.properties | 30 -
.../clusters/offline/slider/log4j.properties | 30 -
.../clusters/offline/slider/slider-client.xml | 11 -
.../clusters/remote/slider/log4j.properties | 30 -
.../clusters/sandbox/slider/log4j.properties | 30 -
.../ubuntu-secure/slider/log4j.properties | 1 -
436 files changed, 15726 insertions(+), 8941 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfe71dc0/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfe71dc0/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
----------------------------------------------------------------------
diff --cc slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
index 35d0f15,0571ca1..a97255e
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
@@@ -24,10 -24,12 +24,13 @@@ package org.apache.slider.server.appmas
public class RestPaths {
public static final String WS_CONTEXT = "ws";
+ public static final String AGENT_WS_CONTEXT = "ws";
public static final String WS_CONTEXT_ROOT = "/" + WS_CONTEXT;
+ public static final String WS_AGENT_CONTEXT_ROOT = "/" + AGENT_WS_CONTEXT;
public static final String SLIDER_CONTEXT_ROOT = WS_CONTEXT_ROOT +"/v1/slider";
+ public static final String SLIDER_AGENT_CONTEXT_ROOT = WS_AGENT_CONTEXT_ROOT +"/v1/slider";
- public static final String SLIDER_SUBPATH_MANAGEMENT = "/mgmt";
+ public static final String MANAGEMENT = "mgmt";
+ public static final String SLIDER_SUBPATH_MANAGEMENT = "/" + MANAGEMENT;
public static final String SLIDER_SUBPATH_AGENTS = "/agents";
public static final String SLIDER_SUBPATH_PUBLISHER = "/publisher";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfe71dc0/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java
----------------------------------------------------------------------
diff --cc slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java
index d9dcdd5,9d1e840..baa9611
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java
@@@ -17,7 -17,8 +17,9 @@@
package org.apache.slider.server.appmaster.web.rest.agent;
import org.apache.slider.server.appmaster.web.WebAppApi;
+import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource;
+ import org.apache.slider.server.services.security.SignCertResponse;
+ import org.apache.slider.server.services.security.SignMessage;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfe71dc0/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bfe71dc0/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
----------------------------------------------------------------------
[37/50] [abbrv] git commit: SLIDER-302: propagation working with
correct ordering
Posted by st...@apache.org.
SLIDER-302: propagation working with correct ordering
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3d02e1cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3d02e1cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3d02e1cb
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 3d02e1cbb072a795b58ac7e263b30d7370f5b133
Parents: 7ef046b
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 8 17:07:24 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 8 17:07:24 2014 +0100
----------------------------------------------------------------------
.../slider/server/appmaster/actions/QueueService.java | 10 +++++++---
.../slider/server/appmaster/actions/TestActions.groovy | 12 ++++++------
2 files changed, 13 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3d02e1cb/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
index dd39de0..69cb9f7 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
@@ -41,11 +41,13 @@ public class QueueService extends AbstractService
super("action queue");
}
- public void putImmediately(AsyncAction action) {
+ public void put(AsyncAction action) {
+ log.debug("Queueing {}", action);
actionQueue.add(action);
}
public void putDelayed(AsyncAction action) {
+ log.debug("Delayed Queueing {}", action);
delayedActions.add(action);
}
@@ -56,13 +58,15 @@ public class QueueService extends AbstractService
public void run() {
try {
- log.info("ActionExecutor Queue terminated");
+ log.info("QueueService processor started");
AsyncAction take;
do {
take = delayedActions.take();
- actionQueue.push(take);
+ log.debug("Propagating {}", take);
+ actionQueue.put(take);
} while (!(take instanceof ActionStopQueue));
+ log.info("QueueService processor terminated");
} catch (InterruptedException e) {
//game over
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3d02e1cb/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
index 24d4c40..7e6338b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/actions/TestActions.groovy
@@ -103,8 +103,8 @@ class TestActions {
@Test
public void testImmediateQueue() throws Throwable {
ActionNoteExecuted noteExecuted = new ActionNoteExecuted("executed", 0)
- queues.putImmediately(noteExecuted)
- queues.putImmediately(new ActionStopQueue(0))
+ queues.put(noteExecuted)
+ queues.put(new ActionStopQueue(0))
QueueExecutor ex = new QueueExecutor(queues)
ex.run();
assert queues.actionQueue.empty
@@ -120,9 +120,9 @@ class TestActions {
List<AsyncAction> actions = [note1, stop, note2]
Collections.sort(actions)
- assert actions.get(0) == note1
- assert actions.get(1) == note2
- assert actions.get(3) == stop
+ assert actions[0] == note1
+ assert actions[1] == note2
+ assert actions[2] == stop
}
@Test
@@ -136,8 +136,8 @@ class TestActions {
assert note1.nanos < note2.nanos
assert note2.nanos < stop.nanos
queues.putDelayed(note1)
- queues.putDelayed(stop)
queues.putDelayed(note2)
+ queues.putDelayed(stop)
// async to sync expected to run in order
queues.run();
assert queues.delayedActions.empty
[14/50] [abbrv] git commit: SLIDER-219. Multiple dynamic ports may be
asked as part of the same config value
Posted by st...@apache.org.
SLIDER-219. Multiple dynamic ports may be asked as part of the same config value
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/dca4818c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/dca4818c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/dca4818c
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: dca4818c551c8fff655c8898c0786aae2fb0fb6a
Parents: ad2c798
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Fri Aug 1 11:52:35 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Aug 5 18:51:58 2014 -0700
----------------------------------------------------------------------
.../slider/providers/ProviderService.java | 7 +
.../providers/agent/AgentProviderService.java | 329 ++++++++++---------
.../server/appmaster/SliderAppMaster.java | 2 +
.../agent/TestAgentProviderService.java | 49 ++-
4 files changed, 224 insertions(+), 163 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dca4818c/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index 56e24e9..b9fa34c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.conf.AggregateConf;
@@ -69,6 +70,12 @@ public interface ProviderService extends ProviderCore, Service,
SliderException;
/**
+ * Notify the providers of container completion
+ * @param containerId
+ */
+ void notifyContainerCompleted(ContainerId containerId);
+
+ /**
* Execute a process in the AM
* @param instanceDefinition cluster description
* @param confDir configuration directory
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dca4818c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index b39601a..bbe2001 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.slider.api.ClusterDescription;
@@ -110,6 +111,7 @@ public class AgentProviderService extends AbstractProviderService implements
private static final String GLOBAL_CONFIG_TAG = "global";
private static final String LOG_FOLDERS_TAG = "LogFolders";
private static final String COMPONENT_DATA_TAG = "ComponentInstanceData";
+ private static final String SHARED_PORT_TAG = "{SHARED}";
private static final int MAX_LOG_ENTRIES = 20;
private static final int DEFAULT_HEARTBEAT_MONITOR_INTERVAL = 60 * 1000;
private final Object syncLock = new Object();
@@ -290,116 +292,6 @@ public class AgentProviderService extends AbstractProviderService implements
}
/**
- * Reads and sets the heartbeat monitoring interval. If bad value is provided then log it and set to default.
- * @param instanceDefinition
- */
- private void readAndSetHeartbeatMonitoringInterval(AggregateConf instanceDefinition) {
- String hbMonitorInterval = instanceDefinition.getAppConfOperations().
- getGlobalOptions().getOption(AgentKeys.HEARTBEAT_MONITOR_INTERVAL,
- Integer.toString(DEFAULT_HEARTBEAT_MONITOR_INTERVAL));
- try {
- setHeartbeatMonitorInterval(Integer.parseInt(hbMonitorInterval));
- }catch (NumberFormatException e) {
- log.warn(
- "Bad value {} for {}. Defaulting to ",
- hbMonitorInterval,
- HEARTBEAT_MONITOR_INTERVAL,
- DEFAULT_HEARTBEAT_MONITOR_INTERVAL);
- }
- }
-
- /**
- * Reads and sets the heartbeat monitoring interval. If bad value is provided then log it and set to default.
- * @param instanceDefinition
- */
- private void initializeAgentDebugCommands(AggregateConf instanceDefinition) {
- String launchParameterStr = instanceDefinition.getAppConfOperations().
- getGlobalOptions().getOption(AgentKeys.AGENT_INSTANCE_DEBUG_DATA, "");
- agentLaunchParameter = new AgentLaunchParameter(launchParameterStr);
- }
-
- @VisibleForTesting
- protected Metainfo getMetainfo() {
- return this.metainfo;
- }
-
- @VisibleForTesting
- protected Map<String, ComponentInstanceState> getComponentStatuses() {
- return componentStatuses;
- }
-
- @VisibleForTesting
- protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
- String appDef) throws IOException {
- return AgentUtils.getApplicationMetainfo(fileSystem, appDef);
- }
-
- @VisibleForTesting
- protected void setHeartbeatMonitorInterval(int heartbeatMonitorInterval) {
- this.heartbeatMonitorInterval = heartbeatMonitorInterval;
- }
-
- private int getHeartbeatMonitorInterval() {
- return this.heartbeatMonitorInterval;
- }
-
- /**
- * Publish a named config bag that may contain name-value pairs for app configurations such as hbase-site
- * @param name
- * @param description
- * @param entries
- */
- protected void publishComponentConfiguration(String name, String description,
- Iterable<Map.Entry<String, String>> entries) {
- PublishedConfiguration pubconf = new PublishedConfiguration();
- pubconf.description = description;
- pubconf.putValues(entries);
- log.info("publishing {}", pubconf);
- getAmState().getPublishedSliderConfigurations().put(name, pubconf);
- }
-
- /**
- * Get a list of all hosts for all role/container per role
- * @return
- */
- protected Map<String, Map<String, ClusterNode>> getRoleClusterNodeMapping() {
- amState.refreshClusterStatus();
- return (Map<String, Map<String, ClusterNode>>)
- amState.getClusterStatus().status.get(
- ClusterDescriptionKeys.KEY_CLUSTER_LIVE);
- }
-
- private String getContainerLabel(Container container, String role) {
- return container.getId().toString() + LABEL_MAKER + role;
- }
-
- protected String getClusterInfoPropertyValue(String name) {
- StateAccessForProviders accessor = getAmState();
- assert accessor.isApplicationLive();
- ClusterDescription description = accessor.getClusterStatus();
- return description.getInfo(name);
- }
-
- /**
- * Lost heartbeat from the container - release it and ask for a replacement
- *
- * @param label
- *
- * @return if release is requested successfully
- */
- protected boolean releaseContainer(String label) {
- componentStatuses.remove(label);
- try {
- getAppMaster().refreshContainer(getContainerId(label), true);
- } catch (SliderException e) {
- log.info("Error while requesting container release for {}. Message: {}", label, e.getMessage());
- return false;
- }
-
- return true;
- }
-
- /**
* Run this service
*
* @param instanceDefinition component description
@@ -421,16 +313,6 @@ public class AgentProviderService extends AbstractProviderService implements
return false;
}
- /**
- * Build the provider status, can be empty
- *
- * @return the provider status - map of entries to add to the info section
- */
- public Map<String, String> buildProviderStatus() {
- Map<String, String> stats = new HashMap<>();
- return stats;
- }
-
@Override
public boolean isSupportedRole(String role) {
return true;
@@ -489,7 +371,7 @@ public class AgentProviderService extends AbstractProviderService implements
// Otherwise, wait till the master that can publish is ready
if (isMaster &&
(canAnyMasterPublishConfig() == false || canPublishConfig(roleName))) {
- processReturnedStatus(heartBeat, componentStatus);
+ publishConfigAndExportGroups(heartBeat, componentStatus);
}
List<CommandReport> reports = heartBeat.getReports();
@@ -511,7 +393,7 @@ public class AgentProviderService extends AbstractProviderService implements
log.info("Component operation. Status: {}", result);
if (command == Command.INSTALL && report.getFolders() != null && report.getFolders().size() > 0) {
- processFolderPaths(report.getFolders(), containerId, heartBeat.getFqdn());
+ publishLogFolderPaths(report.getFolders(), containerId, heartBeat.getFqdn());
}
}
@@ -558,13 +440,168 @@ public class AgentProviderService extends AbstractProviderService implements
return response;
}
+ @Override
+ public Map<String, String> buildMonitorDetails(ClusterDescription clusterDesc) {
+ Map<String, String> details = super.buildMonitorDetails(clusterDesc);
+ buildRoleHostDetails(details);
+ return details;
+ }
+
+ @Override
+ public void applyInitialRegistryDefinitions(URL unsecureWebAPI,
+ URL secureWebAPI,
+ ServiceInstanceData instanceData) throws IOException {
+ super.applyInitialRegistryDefinitions(unsecureWebAPI,
+ secureWebAPI,
+ instanceData
+ );
+
+ try {
+ instanceData.internalView.endpoints.put(
+ CustomRegistryConstants.AGENT_REST_API,
+ new RegisteredEndpoint(
+ new URL(secureWebAPI, SLIDER_PATH_AGENTS),
+ "Agent REST API"));
+ } catch (URISyntaxException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public void notifyContainerCompleted(ContainerId containerId) {
+ if(containerId != null) {
+ String containerIdStr = containerId.toString();
+ }
+ }
+
/**
- * Format the folder locations before publishing in the registry service
+ * Reads and sets the heartbeat monitoring interval. If bad value is provided then log it and set to default.
+ * @param instanceDefinition
+ */
+ private void readAndSetHeartbeatMonitoringInterval(AggregateConf instanceDefinition) {
+ String hbMonitorInterval = instanceDefinition.getAppConfOperations().
+ getGlobalOptions().getOption(AgentKeys.HEARTBEAT_MONITOR_INTERVAL,
+ Integer.toString(DEFAULT_HEARTBEAT_MONITOR_INTERVAL));
+ try {
+ setHeartbeatMonitorInterval(Integer.parseInt(hbMonitorInterval));
+ }catch (NumberFormatException e) {
+ log.warn(
+ "Bad value {} for {}. Defaulting to ",
+ hbMonitorInterval,
+ HEARTBEAT_MONITOR_INTERVAL,
+ DEFAULT_HEARTBEAT_MONITOR_INTERVAL);
+ }
+ }
+
+ /**
+ * Reads and sets the heartbeat monitoring interval. If bad value is provided then log it and set to default.
+ * @param instanceDefinition
+ */
+ private void initializeAgentDebugCommands(AggregateConf instanceDefinition) {
+ String launchParameterStr = instanceDefinition.getAppConfOperations().
+ getGlobalOptions().getOption(AgentKeys.AGENT_INSTANCE_DEBUG_DATA, "");
+ agentLaunchParameter = new AgentLaunchParameter(launchParameterStr);
+ }
+
+ @VisibleForTesting
+ protected Metainfo getMetainfo() {
+ return this.metainfo;
+ }
+
+ @VisibleForTesting
+ protected Map<String, ComponentInstanceState> getComponentStatuses() {
+ return componentStatuses;
+ }
+
+ @VisibleForTesting
+ protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
+ String appDef) throws IOException {
+ return AgentUtils.getApplicationMetainfo(fileSystem, appDef);
+ }
+
+ @VisibleForTesting
+ protected void setHeartbeatMonitorInterval(int heartbeatMonitorInterval) {
+ this.heartbeatMonitorInterval = heartbeatMonitorInterval;
+ }
+
+ private int getHeartbeatMonitorInterval() {
+ return this.heartbeatMonitorInterval;
+ }
+
+ /**
+ * Publish a named property bag that may contain name-value pairs for app configurations such as hbase-site
+ * @param name
+ * @param description
+ * @param entries
+ */
+ protected void publishComponentConfiguration(String name, String description,
+ Iterable<Map.Entry<String, String>> entries) {
+ PublishedConfiguration pubconf = new PublishedConfiguration();
+ pubconf.description = description;
+ pubconf.putValues(entries);
+ log.info("publishing {}", pubconf);
+ getAmState().getPublishedSliderConfigurations().put(name, pubconf);
+ }
+
+ /**
+ * Get a list of all hosts for all role/container per role
+ * @return
+ */
+ protected Map<String, Map<String, ClusterNode>> getRoleClusterNodeMapping() {
+ amState.refreshClusterStatus();
+ return (Map<String, Map<String, ClusterNode>>)
+ amState.getClusterStatus().status.get(
+ ClusterDescriptionKeys.KEY_CLUSTER_LIVE);
+ }
+
+ private String getContainerLabel(Container container, String role) {
+ return container.getId().toString() + LABEL_MAKER + role;
+ }
+
+ protected String getClusterInfoPropertyValue(String name) {
+ StateAccessForProviders accessor = getAmState();
+ assert accessor.isApplicationLive();
+ ClusterDescription description = accessor.getClusterStatus();
+ return description.getInfo(name);
+ }
+
+ /**
+ * Lost heartbeat from the container - release it and ask for a replacement
+ *
+ * @param label
+ *
+ * @return if release is requested successfully
+ */
+ protected boolean releaseContainer(String label) {
+ componentStatuses.remove(label);
+ try {
+ getAppMaster().refreshContainer(getContainerId(label), true);
+ } catch (SliderException e) {
+ log.info("Error while requesting container release for {}. Message: {}", label, e.getMessage());
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * Build the provider status, can be empty
+ *
+ * @return the provider status - map of entries to add to the info section
+ */
+ public Map<String, String> buildProviderStatus() {
+ Map<String, String> stats = new HashMap<>();
+ return stats;
+ }
+
+
+ /**
+ * Format the folder locations and publish in the registry service
* @param folders
* @param containerId
* @param hostFqdn
*/
- private void processFolderPaths(Map<String, String> folders, String containerId, String hostFqdn) {
+ private void publishLogFolderPaths(Map<String, String> folders, String containerId, String hostFqdn) {
for (String key : folders.keySet()) {
workFolders.put(String.format("%s-%s-%s", hostFqdn, containerId, key), folders.get(key));
}
@@ -578,7 +615,7 @@ public class AgentProviderService extends AbstractProviderService implements
* @param heartBeat
* @param componentStatus
*/
- protected void processReturnedStatus(HeartBeat heartBeat, ComponentInstanceState componentStatus) {
+ protected void publishConfigAndExportGroups(HeartBeat heartBeat, ComponentInstanceState componentStatus) {
List<ComponentStatus> statuses = heartBeat.getComponentStatus();
if (statuses != null && !statuses.isEmpty()) {
log.info("Processing {} status reports.", statuses.size());
@@ -658,11 +695,11 @@ public class AgentProviderService extends AbstractProviderService implements
boolean publishData = false;
String portValPattern = String.format(portVarFormat, portName);
if(templateToExport.contains(portValPattern)) {
- templateToExport.replace(portValPattern, ports.get(portName));
+ templateToExport = templateToExport.replace(portValPattern, ports.get(portName));
publishData = true;
}
if(templateToExport.contains(hostNamePattern)) {
- templateToExport.replace(hostNamePattern, hostFqdn);
+ templateToExport = templateToExport.replace(hostNamePattern, hostFqdn);
publishData = true;
}
if(publishData) {
@@ -1032,9 +1069,13 @@ public class AgentProviderService extends AbstractProviderService implements
//apply any port updates
if (!this.getAllocatedPorts().isEmpty()) {
for (String key : config.keySet()) {
- String lookupKey = configName + "." + key;
- if (this.getAllocatedPorts().containsKey(lookupKey)) {
- config.put(key, getAllocatedPorts().get(lookupKey));
+ String value = config.get(key);
+ if(!value.contains(SHARED_PORT_TAG)) {
+ // If the config property is not shared then do not pass on the already allocated value
+ String lookupKey = configName + "." + key;
+ if (this.getAllocatedPorts().containsKey(lookupKey)) {
+ config.put(key, getAllocatedPorts().get(lookupKey));
+ }
}
}
}
@@ -1065,13 +1106,6 @@ public class AgentProviderService extends AbstractProviderService implements
config.put("app_install_dir", "${AGENT_WORK_ROOT}/app/install");
}
- @Override
- public Map<String, String> buildMonitorDetails(ClusterDescription clusterDesc) {
- Map<String, String> details = super.buildMonitorDetails(clusterDesc);
- buildRoleHostDetails(details);
- return details;
- }
-
private void buildRoleHostDetails(Map<String, String> details) {
for (Map.Entry<String, Map<String, ClusterNode>> entry :
getRoleClusterNodeMapping().entrySet()) {
@@ -1080,25 +1114,4 @@ public class AgentProviderService extends AbstractProviderService implements
"");
}
}
-
- @Override
- public void applyInitialRegistryDefinitions(URL unsecureWebAPI,
- URL secureWebAPI,
- ServiceInstanceData instanceData) throws IOException {
- super.applyInitialRegistryDefinitions(unsecureWebAPI,
- secureWebAPI,
- instanceData
- );
-
- try {
- instanceData.internalView.endpoints.put(
- CustomRegistryConstants.AGENT_REST_API,
- new RegisteredEndpoint(
- new URL(secureWebAPI, SLIDER_PATH_AGENTS),
- "Agent REST API"));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
-
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dca4818c/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 7e0ae5e..a2dc36f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -1061,6 +1061,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
RoleInstance ri = result.roleInstance;
log.error("Role instance {} failed ", ri);
}
+
+ getProviderService().notifyContainerCompleted(containerId);
}
// ask for more containers if any failed
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dca4818c/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 4d63263..1f9cb41 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -355,6 +355,45 @@ public class TestAgentProviderService {
}
@Test
+ public void testComponentSpecificPublishes() throws Exception {
+ InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
+ Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
+ AgentProviderService aps = new AgentProviderService();
+ AgentProviderService mockAps = Mockito.spy(aps);
+ doNothing().when(mockAps).publishComponentConfiguration(anyString(), anyString(), anyCollection());
+ doReturn(metainfo).when(mockAps).getMetainfo();
+
+ Map<String, String> ports = new HashMap<>();
+ ports.put("global.listen_port", "10010");
+ mockAps.processComponentSpecificPublishes(ports,
+ "cid1",
+ "host1",
+ "HBASE_REGIONSERVER");
+ ArgumentCaptor<Collection> entriesCaptor = ArgumentCaptor.
+ forClass(Collection.class);
+ ArgumentCaptor<String> publishNameCaptor = ArgumentCaptor.
+ forClass(String.class);
+ Mockito.verify(mockAps, Mockito.times(1)).publishComponentConfiguration(
+ anyString(),
+ publishNameCaptor.capture(),
+ entriesCaptor.capture());
+ assert entriesCaptor.getAllValues().size() == 1;
+ for (Collection coll : entriesCaptor.getAllValues()) {
+ Set<Map.Entry<String, String>> entrySet = (Set<Map.Entry<String, String>>) coll;
+ for (Map.Entry entry : entrySet) {
+ log.info("{}:{}", entry.getKey(), entry.getValue().toString());
+ if (entry.getKey().equals("PropertyA")) {
+ assert entry.getValue().toString().equals("host1:10010");
+ }
+ }
+ }
+ assert publishNameCaptor.getAllValues().size() == 1;
+ for (String coll : publishNameCaptor.getAllValues()) {
+ assert coll.equals("ComponentInstanceData");
+ }
+ }
+
+ @Test
public void testProcessConfig() throws Exception {
InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
@@ -387,16 +426,16 @@ public class TestAgentProviderService {
doReturn(metainfo).when(mockAps).getMetainfo();
doReturn(roleClusterNodeMap).when(mockAps).getRoleClusterNodeMapping();
- mockAps.processReturnedStatus(hb, componentStatus);
+ mockAps.publishConfigAndExportGroups(hb, componentStatus);
assert componentStatus.getConfigReported() == true;
- ArgumentCaptor<Collection> commandCaptor = ArgumentCaptor.
+ ArgumentCaptor<Collection> entriesCaptor = ArgumentCaptor.
forClass(Collection.class);
Mockito.verify(mockAps, Mockito.times(3)).publishComponentConfiguration(
anyString(),
anyString(),
- commandCaptor.capture());
- assert commandCaptor.getAllValues().size() == 3;
- for (Collection coll : commandCaptor.getAllValues()) {
+ entriesCaptor.capture());
+ assert entriesCaptor.getAllValues().size() == 3;
+ for (Collection coll : entriesCaptor.getAllValues()) {
Set<Map.Entry<String, String>> entrySet = (Set<Map.Entry<String, String>>) coll;
for (Map.Entry entry : entrySet) {
log.info("{}:{}", entry.getKey(), entry.getValue().toString());
[19/50] [abbrv] git commit: SLIDER-295 Unit test failure caused by
wrong parameter type in AppState#onCompletedNode()
Posted by st...@apache.org.
SLIDER-295 Unit test failure caused by wrong parameter type in AppState#onCompletedNode()
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/533b229f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/533b229f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/533b229f
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 533b229f09d7fd691be53c2edaa364c72896412f
Parents: 32fd3f5
Author: tedyu <yu...@gmail.com>
Authored: Wed Aug 6 08:23:23 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Wed Aug 6 08:23:23 2014 -0700
----------------------------------------------------------------------
.../java/org/apache/slider/server/appmaster/state/AppState.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/533b229f/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 08d43db..3acd222 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1218,7 +1218,7 @@ public class AppState {
if (failedContainer != null) {
String completedLogsUrl = getLogsURLForContainer(failedContainer);
message = String.format("Failure %s on host %s: %s",
- roleInstance.getContainerId(),
+ roleInstance.getContainerId().toString(),
failedContainer.getNodeId().getHost(),
completedLogsUrl);
} else {
[42/50] [abbrv] git commit: SLIDER-302 reorder constructors for IDE
subclassing ease
Posted by st...@apache.org.
SLIDER-302 reorder constructors for IDE subclassing ease
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/cf2aa70e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/cf2aa70e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/cf2aa70e
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: cf2aa70e49d606a9c2bfee8194b84746166b835a
Parents: b7b9a15
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 8 19:16:55 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 8 19:16:55 2014 +0100
----------------------------------------------------------------------
.../slider/server/appmaster/actions/AsyncAction.java | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cf2aa70e/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
index a755064..5e325a0 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/AsyncAction.java
@@ -43,6 +43,13 @@ public abstract class AsyncAction implements Delayed {
}
protected AsyncAction(String name,
+ int delayMillis) {
+ this.name = name;
+ this.nanos = convertAndOffset(delayMillis);
+ attrs = EnumSet.noneOf(ActionAttributes.class);
+ }
+
+ protected AsyncAction(String name,
int delayMillis, EnumSet<ActionAttributes> attrs) {
this.name = name;
this.nanos = convertAndOffset(delayMillis);
@@ -56,13 +63,6 @@ public abstract class AsyncAction implements Delayed {
Collections.addAll(attrs, attributes);
}
- protected AsyncAction(String name,
- int delayMillis) {
- this.name = name;
- this.nanos = convertAndOffset(delayMillis);
- attrs = EnumSet.noneOf(ActionAttributes.class);
- }
-
private long convertAndOffset(int delay) {
return now() + TimeUnit.NANOSECONDS.convert(delay, TimeUnit.MILLISECONDS);
}
[46/50] [abbrv] git commit: Merge branch 'develop' of
https://git-wip-us.apache.org/repos/asf/incubator-slider into develop
Posted by st...@apache.org.
Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/incubator-slider into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/05bf45d1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/05bf45d1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/05bf45d1
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 05bf45d179cd52ea0d9de79ffd1a84dec90958fd
Parents: 6f912df c215045
Author: tedyu <yu...@gmail.com>
Authored: Fri Aug 8 11:24:31 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Fri Aug 8 11:24:31 2014 -0700
----------------------------------------------------------------------
.../apache/slider/common/tools/SliderUtils.java | 17 +-
.../providers/AbstractProviderService.java | 12 +-
.../slider/providers/ProviderService.java | 16 +-
.../providers/agent/AgentProviderService.java | 15 +-
.../providers/agent/HeartbeatMonitor.java | 9 +-
.../server/appmaster/AMViewForProviders.java | 46 -----
.../appmaster/ContainerStartOperation.java | 40 -----
.../server/appmaster/RoleLaunchService.java | 40 +++--
.../server/appmaster/SliderAppMaster.java | 134 ++++++++++----
.../server/appmaster/actions/ActionHalt.java | 41 +++++
.../appmaster/actions/ActionStartContainer.java | 47 +++++
.../appmaster/actions/ActionStopQueue.java | 36 ++++
.../appmaster/actions/ActionStopSlider.java | 39 ++++
.../server/appmaster/actions/AsyncAction.java | 124 +++++++++++++
.../actions/ProviderReportedContainerLoss.java | 47 +++++
.../actions/ProviderStartupCompleted.java | 37 ++++
.../server/appmaster/actions/QueueAccess.java | 35 ++++
.../server/appmaster/actions/QueueExecutor.java | 75 ++++++++
.../server/appmaster/actions/QueueService.java | 76 ++++++++
.../appmaster/actions/StartContainerAction.java | 41 +++++
.../AbstractWorkflowExecutorService.java | 110 ------------
.../services/workflow/ForkedProcessService.java | 6 +-
.../workflow/WorkflowCallbackService.java | 3 +-
.../workflow/WorkflowExecutorService.java | 113 ++++++++++++
.../WorkflowScheduledExecutorService.java | 38 ++++
.../server/services/workflow/package-info.java | 2 +-
.../standalone/TestDestroyStandaloneAM.groovy | 156 ----------------
.../standalone/TestKillStandaloneAM.groovy | 73 --------
.../standalone/TestRestartStandaloneAM.groovy | 96 ----------
.../standalone/TestStandaloneAMDestroy.groovy | 154 ++++++++++++++++
.../standalone/TestStandaloneAMKill.groovy | 73 ++++++++
.../standalone/TestStandaloneAMRestart.groovy | 96 ++++++++++
.../server/appmaster/actions/TestActions.groovy | 180 +++++++++++++++++++
.../model/history/TestRoleHistoryRW.groovy | 2 +-
.../model/mock/MockProviderService.groovy | 4 +-
.../providers/agent/TestHeartbeatMonitor.java | 6 -
.../publisher/TestAgentProviderService.java | 6 +-
.../workflow/TestWorkflowExecutorService.java | 9 +-
38 files changed, 1428 insertions(+), 626 deletions(-)
----------------------------------------------------------------------
[12/50] [abbrv] git commit: SLIDER-219. Multiple dynamic ports may be
asked as part of the same config value
Posted by st...@apache.org.
SLIDER-219. Multiple dynamic ports may be asked as part of the same config value
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1894a0ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1894a0ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1894a0ca
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 1894a0ca206d4d024f7b3062133ab1039ca5988b
Parents: a9a91b4
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Jul 29 23:33:00 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Aug 5 18:51:58 2014 -0700
----------------------------------------------------------------------
app-packages/memcached/appConfig.json | 3 +-
app-packages/memcached/metainfo.xml | 6 ++
.../python/agent/CustomServiceOrchestrator.py | 57 ++++++++--
.../agent/TestCustomServiceOrchestrator.py | 107 +++++++++++++++++--
slider-agent/src/test/python/agent/TestGrep.py | 2 +-
slider-agent/src/test/python/agent/TestMain.py | 4 +-
.../src/test/python/agent/TestRegistration.py | 4 +-
.../providers/agent/AgentProviderService.java | 3 +
.../agent/application/metadata/Component.java | 13 +++
.../application/metadata/MetainfoParser.java | 4 +
.../agent/TestAgentProviderService.java | 25 ++++-
11 files changed, 202 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/app-packages/memcached/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached/appConfig.json b/app-packages/memcached/appConfig.json
index d205591..11fb863 100644
--- a/app-packages/memcached/appConfig.json
+++ b/app-packages/memcached/appConfig.json
@@ -10,7 +10,8 @@
"site.global.additional_cp": "/usr/lib/hadoop/lib/*",
"site.global.xmx_val": "256m",
"site.global.xms_val": "128m",
- "site.global.memory_val": "200M"
+ "site.global.memory_val": "200M",
+ "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DEFAULT_11211}"
},
"components": {
"slider-appmaster": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/app-packages/memcached/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/memcached/metainfo.xml b/app-packages/memcached/metainfo.xml
index d435b84..0550b7d 100644
--- a/app-packages/memcached/metainfo.xml
+++ b/app-packages/memcached/metainfo.xml
@@ -27,6 +27,12 @@
<component>
<name>MEMCACHED</name>
<category>MASTER</category>
+ <exports>
+ <export>
+ <name>host_port</name>
+ <value>${THIS_HOST}:${site.global.listen_port}</value>
+ </export>
+ </exports>
<commandScript>
<script>scripts/memcached.py</script>
<scriptType>PYTHON</scriptType>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 4eb22bf..7d740c2 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -52,9 +52,9 @@ class CustomServiceOrchestrator():
self.tmp_dir = config.getResolvedPath(AgentConfig.APP_TASK_DIR)
self.python_executor = PythonExecutor(self.tmp_dir, config)
self.status_commands_stdout = os.path.realpath(posixpath.join(self.tmp_dir,
- 'status_command_stdout.txt'))
+ 'status_command_stdout.txt'))
self.status_commands_stderr = os.path.realpath(posixpath.join(self.tmp_dir,
- 'status_command_stderr.txt'))
+ 'status_command_stderr.txt'))
self.public_fqdn = hostname.public_hostname()
self.applied_configs = {}
# Clean up old status command files if any
@@ -81,7 +81,7 @@ class CustomServiceOrchestrator():
script_tuple = (script_path, self.base_dir)
tmpstrucoutfile = os.path.realpath(posixpath.join(self.tmp_dir,
- "structured-out-{0}.json".format(task_id)))
+ "structured-out-{0}.json".format(task_id)))
if script_type.upper() != self.SCRIPT_TYPE_PYTHON:
# We don't support anything else yet
message = "Unknown script type {0}".format(script_type)
@@ -97,9 +97,9 @@ class CustomServiceOrchestrator():
for py_file, current_base_dir in filtered_py_file_list:
script_params = [command_name, json_path, current_base_dir]
python_paths = [os.path.realpath(posixpath.join(self.config.getWorkRootPath(),
- "infra", "agent", "slider-agent", "jinja2")),
+ "infra", "agent", "slider-agent", "jinja2")),
os.path.realpath(posixpath.join(self.config.getWorkRootPath(),
- "infra", "agent", "slider-agent"))]
+ "infra", "agent", "slider-agent"))]
if platform.system() != "Windows":
environment_vars = [("PYTHONPATH", ":".join(python_paths))]
else:
@@ -137,8 +137,8 @@ class CustomServiceOrchestrator():
# Irrespective of the outcome report the folder paths
if command_name == 'INSTALL':
ret[Constants.FOLDERS] = {
- Constants.AGENT_LOG_ROOT : self.config.getLogPath(),
- Constants.AGENT_WORK_ROOT : self.config.getWorkRootPath()
+ Constants.AGENT_LOG_ROOT: self.config.getLogPath(),
+ Constants.AGENT_WORK_ROOT: self.config.getWorkRootPath()
}
return ret
@@ -249,9 +249,7 @@ class CustomServiceOrchestrator():
value = value.replace("${AGENT_LOG_ROOT}",
self.config.getLogPath())
if port_allocation_req in value:
- port = self.allocate_port()
- value = value.replace(port_allocation_req, str(port))
- logger.info("Allocated port " + str(port) + " for " + port_allocation_req)
+ value = self.allocate_ports(value, port_allocation_req)
allocated_ports[k] = value
command['configurations'][key][k] = value
pass
@@ -266,7 +264,34 @@ class CustomServiceOrchestrator():
pass
- def allocate_port(self):
+ def allocate_ports(self, value, port_req_pattern):
+ default_port_pattern = "{DEFAULT_"
+ index = value.find(port_req_pattern)
+ while index != -1:
+ replaced_pattern = port_req_pattern
+ def_port = None
+ if index == value.find(port_req_pattern + default_port_pattern):
+ replaced_pattern = port_req_pattern + default_port_pattern
+ start_index = index + len(replaced_pattern)
+ end_index = value.find("}", start_index)
+ def_port_str = value[start_index:end_index]
+ def_port = int(def_port_str)
+ replaced_pattern = replaced_pattern + def_port_str + "}"
+ pass
+ port = self.allocate_port(def_port)
+ value = value.replace(replaced_pattern, str(port), 1)
+ logger.info("Allocated port " + str(port) + " for " + replaced_pattern)
+ index = value.find(port_req_pattern)
+ pass
+ return value
+ pass
+
+
+ def allocate_port(self, default_port=None):
+ if default_port != None:
+ if self.is_port_available(default_port):
+ return default_port
+
MAX_ATTEMPT = 5
iter = 0
port = -1
@@ -284,4 +309,14 @@ class CustomServiceOrchestrator():
logger.info("Allocated dynamic port: " + str(port))
return port
+ def is_port_available(self, port):
+ try:
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ sock.settimeout(0.2)
+ sock.connect(('127.0.0.1', port))
+ sock.close()
+ except:
+ return True
+ return False
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index d2439b1..3b41bd7 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -34,6 +34,7 @@ from CustomServiceOrchestrator import CustomServiceOrchestrator
from mock.mock import MagicMock, patch
import StringIO
import sys
+from socket import socket
class TestCustomServiceOrchestrator(TestCase):
@@ -178,13 +179,13 @@ class TestCustomServiceOrchestrator(TestCase):
pass
- @patch.object(CustomServiceOrchestrator, "allocate_port")
+ @patch.object(CustomServiceOrchestrator, "allocate_ports")
@patch.object(CustomServiceOrchestrator, "resolve_script_path")
@patch.object(PythonExecutor, "run_file")
def test_runCommand_get_port(self,
run_file_mock,
resolve_script_path_mock,
- allocate_port_mock):
+ allocate_ports_mock):
command = {
'role': 'HBASE_REGIONSERVER',
'hostLevelParams': {
@@ -212,7 +213,7 @@ class TestCustomServiceOrchestrator(TestCase):
config.getWorkRootPath.return_value = tempdir
config.getLogPath.return_value = tempdir
- allocate_port_mock.return_value = 10233
+ allocate_ports_mock.return_value = str(10233)
resolve_script_path_mock.return_value = "/basedir/scriptpath"
dummy_controller = MagicMock()
@@ -230,6 +231,100 @@ class TestCustomServiceOrchestrator(TestCase):
self.assertEqual(run_file_mock.call_count, 1)
+ @patch.object(socket, "close")
+ @patch.object(socket, "connect")
+ def test_allocate_port_def(self, socket_connect_mock, socket_close_mock):
+ e = OSError()
+ socket_connect_mock.side_effect = e
+ tempdir = tempfile.gettempdir()
+ config = MagicMock()
+ config.get.return_value = "something"
+ config.getResolvedPath.return_value = tempdir
+ config.getWorkRootPath.return_value = tempdir
+ config.getLogPath.return_value = tempdir
+
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ ret = orchestrator.allocate_port(10)
+ self.assertEqual(ret, 10)
+
+ @patch.object(socket, "getsockname")
+ @patch.object(socket, "bind")
+ @patch.object(socket, "close")
+ @patch.object(socket, "connect")
+ def test_allocate_port_new(self, socket_connect_mock, socket_close_mock,
+ socket_bind_mock, socket_getsockname_mock):
+ tempdir = tempfile.gettempdir()
+ config = MagicMock()
+ config.get.return_value = "something"
+ config.getResolvedPath.return_value = tempdir
+ config.getWorkRootPath.return_value = tempdir
+ config.getLogPath.return_value = tempdir
+
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ socket_getsockname_mock.return_value = [100, 101]
+ ret = orchestrator.allocate_port(10)
+ self.assertEqual(ret, 101)
+
+ @patch.object(socket, "getsockname")
+ @patch.object(socket, "bind")
+ def test_allocate_port_no_def(self, socket_bind_mock, socket_getsockname_mock):
+ tempdir = tempfile.gettempdir()
+ config = MagicMock()
+ config.get.return_value = "something"
+ config.getResolvedPath.return_value = tempdir
+ config.getWorkRootPath.return_value = tempdir
+ config.getLogPath.return_value = tempdir
+
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ socket_getsockname_mock.return_value = [100, 102]
+ ret = orchestrator.allocate_port()
+ self.assertEqual(ret, 102)
+
+
+ @patch.object(CustomServiceOrchestrator, "allocate_port")
+ def test_allocate_port_combinations(self, allocate_port_mock):
+ tempdir = tempfile.gettempdir()
+ config = MagicMock()
+ config.get.return_value = "something"
+ config.getResolvedPath.return_value = tempdir
+ config.getWorkRootPath.return_value = tempdir
+ config.getLogPath.return_value = tempdir
+
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+
+ allocate_port_mock.side_effect = [101, 102, 103]
+ ret = orchestrator.allocate_ports("1000", "${A.ALLOCATED_PORT}")
+ self.assertEqual(ret, "1000")
+ ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}", "${A.ALLOCATED_PORT}")
+ self.assertEqual(ret, "101")
+ ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT},${A.ALLOCATED_PORT}", "${A.ALLOCATED_PORT}")
+ self.assertEqual(ret, "102,103")
+
+
+ @patch.object(CustomServiceOrchestrator, "is_port_available")
+ def test_allocate_port_combinations2(self, is_port_available_mock):
+ tempdir = tempfile.gettempdir()
+ config = MagicMock()
+ config.get.return_value = "something"
+ config.getResolvedPath.return_value = tempdir
+ config.getWorkRootPath.return_value = tempdir
+ config.getLogPath.return_value = tempdir
+
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+
+ is_port_available_mock.return_value = True
+ ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_1005}", "${A.ALLOCATED_PORT}")
+ self.assertEqual(ret, "1005")
+ ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_1005}-${A.ALLOCATED_PORT}{DEFAULT_1006}",
+ "${A.ALLOCATED_PORT}")
+ self.assertEqual(ret, "1005-1006")
+
+
@patch("hostname.public_hostname")
@patch("os.path.isfile")
@patch("os.unlink")
@@ -353,8 +448,8 @@ class TestCustomServiceOrchestrator(TestCase):
status = orchestrator.requestComponentStatus(status_command)
self.assertEqual(CustomServiceOrchestrator.DEAD_STATUS, status['exitcode'])
- @patch.object(CustomServiceOrchestrator, "allocate_port")
- def test_finalize_command(self, mock_allocate_port):
+ @patch.object(CustomServiceOrchestrator, "allocate_ports")
+ def test_finalize_command(self, mock_allocate_ports):
dummy_controller = MagicMock()
tempdir = tempfile.gettempdir()
tempWorkDir = tempdir + "W"
@@ -363,7 +458,7 @@ class TestCustomServiceOrchestrator(TestCase):
config.getResolvedPath.return_value = tempdir
config.getWorkRootPath.return_value = tempWorkDir
config.getLogPath.return_value = tempdir
- mock_allocate_port.return_value = "10023"
+ mock_allocate_ports.return_value = "10023"
orchestrator = CustomServiceOrchestrator(config, dummy_controller)
command = {}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/slider-agent/src/test/python/agent/TestGrep.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestGrep.py b/slider-agent/src/test/python/agent/TestGrep.py
index 75f0093..351befb 100644
--- a/slider-agent/src/test/python/agent/TestGrep.py
+++ b/slider-agent/src/test/python/agent/TestGrep.py
@@ -19,7 +19,7 @@ limitations under the License.
'''
from unittest import TestCase
-from agent.Grep import Grep
+from Grep import Grep
import socket
import os, sys
import logging
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/slider-agent/src/test/python/agent/TestMain.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestMain.py b/slider-agent/src/test/python/agent/TestMain.py
index dc3a199..bc8d2a3 100644
--- a/slider-agent/src/test/python/agent/TestMain.py
+++ b/slider-agent/src/test/python/agent/TestMain.py
@@ -26,11 +26,11 @@ import unittest
from agent import ProcessHelper, main
import logging
import signal
-from agent.AgentConfig import AgentConfig
+from AgentConfig import AgentConfig
import ConfigParser
import os
import tempfile
-from agent.Controller import Controller
+from Controller import Controller
from optparse import OptionParser
logger = logging.getLogger()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/slider-agent/src/test/python/agent/TestRegistration.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestRegistration.py b/slider-agent/src/test/python/agent/TestRegistration.py
index 8647d0d..5245af9 100644
--- a/slider-agent/src/test/python/agent/TestRegistration.py
+++ b/slider-agent/src/test/python/agent/TestRegistration.py
@@ -25,8 +25,8 @@ import errno
import tempfile
from mock.mock import patch
from mock.mock import MagicMock
-from agent.Register import Register
-from agent.AgentConfig import AgentConfig
+from Register import Register
+from AgentConfig import AgentConfig
class TestRegistration(TestCase):
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index a73aaab..f878b36 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -109,6 +109,7 @@ public class AgentProviderService extends AbstractProviderService implements
private static final String CONTAINER_ID = "container_id";
private static final String GLOBAL_CONFIG_TAG = "global";
private static final String LOG_FOLDERS_TAG = "LogFolders";
+ private static final String COMPONENT_DATA_TAG = "ComponentData";
private static final int MAX_LOG_ENTRIES = 20;
private static final int DEFAULT_HEARTBEAT_MONITOR_INTERVAL = 60 * 1000;
private final Object syncLock = new Object();
@@ -126,6 +127,7 @@ public class AgentProviderService extends AbstractProviderService implements
return size() > MAX_LOG_ENTRIES;
}
});
+ private Map<String, Map<String, String>> componentData = new ConcurrentHashMap();
private Boolean canAnyMasterPublish = null;
private AgentLaunchParameter agentLaunchParameter = null;
@@ -567,6 +569,7 @@ public class AgentProviderService extends AbstractProviderService implements
publishComponentConfiguration(LOG_FOLDERS_TAG, LOG_FOLDERS_TAG, (new HashMap<>(this.workFolders)).entrySet());
}
+
/**
* Process return status for component instances
* @param heartBeat
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
index 6cd08e0..9dba87c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
@@ -16,6 +16,9 @@
*/
package org.apache.slider.providers.agent.application.metadata;
+import java.util.ArrayList;
+import java.util.List;
+
/**
*
*/
@@ -26,9 +29,11 @@ public class Component {
String minInstanceCount;
String maxInstanceCount;
CommandScript commandScript;
+ List<Export> exports;
public Component() {
publishConfig = Boolean.FALSE.toString();
+ exports = new ArrayList<>();
}
public String getName() {
@@ -79,6 +84,14 @@ public class Component {
this.commandScript = commandScript;
}
+ public void addExport(Export export) {
+ exports.add(export);
+ }
+
+ public List<Export> getExports() {
+ return exports;
+ }
+
@Override
public String toString() {
final StringBuilder sb =
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
index c7922a7..d2801d4 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
@@ -58,6 +58,10 @@ public class MetainfoParser {
digester.addBeanPropertySetter("*/component/publishConfig");
digester.addBeanPropertySetter("*/component/minInstanceCount");
digester.addBeanPropertySetter("*/component/maxInstanceCount");
+ digester.addObjectCreate("*/component/export", Export.class);
+ digester.addBeanPropertySetter("*/component/export/name");
+ digester.addBeanPropertySetter("*/component/export/value");
+ digester.addSetNext("*/component/export", "addExport");
digester.addSetNext("*/component", "addComponent");
digester.addObjectCreate("*/commandScript", CommandScript.class);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 0a42e94..c34d0ea 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -151,6 +151,16 @@ public class TestAgentProviderService {
+ " <script>scripts/hbase_regionserver.py</script>\n"
+ " <scriptType>PYTHON</scriptType>\n"
+ " </commandScript>\n"
+ + " <exports>\n"
+ + " <export>\n"
+ + " <name>PropertyA</name>\n"
+ + " <value>${THIS_HOST}:${site.global.listen_port}</value>\n"
+ + " </export>\n"
+ + " <export>\n"
+ + " <name>PropertyB</name>\n"
+ + " <value>AConstant</value>\n"
+ + " </export>\n"
+ + " </exports>\n"
+ " </component>\n"
+ " </components>\n"
+ " <osSpecifics>\n"
@@ -414,6 +424,7 @@ public class TestAgentProviderService {
Assert.assertEquals(component.getMaxInstanceCount(), "2");
Assert.assertEquals(component.getCommandScript().getScript(), "scripts/hbase_master.py");
Assert.assertEquals(component.getCategory(), "MASTER");
+ Assert.assertEquals(component.getExports().size(), 0);
found++;
}
if (component.getName().equals("HBASE_REGIONSERVER")) {
@@ -421,16 +432,24 @@ public class TestAgentProviderService {
Assert.assertNull(component.getMaxInstanceCount());
Assert.assertEquals(component.getCommandScript().getScript(), "scripts/hbase_regionserver.py");
Assert.assertEquals(component.getCategory(), "SLAVE");
+ Assert.assertEquals(component.getExports().size(), 2);
+ List<Export> es = component.getExports();
+ Export e = es.get(0);
+ Assert.assertEquals(e.getName(), "PropertyA");
+ Assert.assertEquals(e.getValue(), "${THIS_HOST}:${site.global.listen_port}");
+ e = es.get(1);
+ Assert.assertEquals(e.getName(), "PropertyB");
+ Assert.assertEquals(e.getValue(), "AConstant");
found++;
}
}
Assert.assertEquals(found, 2);
- assert application.getExportGroups().size() == 1;
+ Assert.assertEquals(application.getExportGroups().size(), 1);
List<ExportGroup> egs = application.getExportGroups();
ExportGroup eg = egs.get(0);
- assert eg.getName().equals("QuickLinks");
- assert eg.getExports().size() == 2;
+ Assert.assertEquals(eg.getName(), "QuickLinks");
+ Assert.assertEquals(eg.getExports().size(), 2);
found = 0;
for (Export export : eg.getExports()) {
[13/50] [abbrv] git commit: SLIDER-273. Add ability to specify a port
as shared across component instances or not
Posted by st...@apache.org.
SLIDER-273. Add ability to specify a port as shared across component instances or not
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/50dc76c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/50dc76c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/50dc76c9
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 50dc76c978ef4ff684263b93e8b56619cc947284
Parents: dca4818
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sun Aug 3 19:30:13 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Aug 5 18:51:58 2014 -0700
----------------------------------------------------------------------
app-packages/memcached/appConfig.json | 2 +-
.../python/agent/CustomServiceOrchestrator.py | 27 +++-
.../agent/TestCustomServiceOrchestrator.py | 18 ++-
.../providers/AbstractProviderService.java | 5 +
.../providers/agent/AgentProviderService.java | 126 ++++++++++++-------
.../model/mock/MockProviderService.groovy | 5 +
.../agent/TestAgentProviderService.java | 58 +++++++--
.../publisher/TestAgentProviderService.java | 4 +-
8 files changed, 185 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50dc76c9/app-packages/memcached/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached/appConfig.json b/app-packages/memcached/appConfig.json
index 11fb863..78cb486 100644
--- a/app-packages/memcached/appConfig.json
+++ b/app-packages/memcached/appConfig.json
@@ -11,7 +11,7 @@
"site.global.xmx_val": "256m",
"site.global.xms_val": "128m",
"site.global.memory_val": "200M",
- "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DEFAULT_11211}"
+ "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DEFAULT_11211}{DO_NOT_PROPAGATE}"
},
"components": {
"slider-appmaster": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50dc76c9/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index f6cdfb2..0f4acc6 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -233,12 +233,17 @@ class CustomServiceOrchestrator():
patch content
${AGENT_WORK_ROOT} -> AgentConfig.getWorkRootPath()
${AGENT_LOG_ROOT} -> AgentConfig.getLogPath()
+ ALLOCATED_PORT is a hint to allocate port. It works as follows:
+ Its of the form {component_name.ALLOCATED_PORT}[{DEFAULT_default_port}][{DO_NOT_PROPAGATE}]
+ Either a port gets allocated or if not then just set the value to "0"
"""
def finalize_command(self, command, store_config, allocated_ports):
component = command['componentName']
- allocated_port_format = "${{{0}.ALLOCATED_PORT}}"
- port_allocation_req = allocated_port_format.format(component)
+ allocated_for_this_component_format = "${{{0}.ALLOCATED_PORT}}"
+ allocated_for_any = ".ALLOCATED_PORT}"
+
+ port_allocation_req = allocated_for_this_component_format.format(component)
if 'configurations' in command:
for key in command['configurations']:
if len(command['configurations'][key]) > 0:
@@ -251,6 +256,10 @@ class CustomServiceOrchestrator():
if port_allocation_req in value:
value = self.allocate_ports(value, port_allocation_req)
allocated_ports[key + "." + k] = value
+ elif allocated_for_any in value:
+ ## All unallocated ports should be set to 0
+ logger.info("Assigning port 0 " + "for " + value)
+ value = "0"
command['configurations'][key][k] = value
pass
pass
@@ -264,8 +273,15 @@ class CustomServiceOrchestrator():
pass
+ """
+ Port allocation can asks for multiple dynamic ports
+ port_req_pattern is of type ${component_name.ALLOCATED_PORT}
+ append {DEFAULT_ and find the default value
+ append {DO_NOT_PROPAGATE} if it exists
+ """
def allocate_ports(self, value, port_req_pattern):
default_port_pattern = "{DEFAULT_"
+ do_not_propagate_pattern = "{DO_NOT_PROPAGATE}"
index = value.find(port_req_pattern)
while index != -1:
replaced_pattern = port_req_pattern
@@ -276,8 +292,15 @@ class CustomServiceOrchestrator():
end_index = value.find("}", start_index)
def_port_str = value[start_index:end_index]
def_port = int(def_port_str)
+ # default value of 0 means allocate any dynamic port
+ if def_port == 0:
+ def_port = None
+
replaced_pattern = replaced_pattern + def_port_str + "}"
pass
+ if index == value.find(replaced_pattern + do_not_propagate_pattern):
+ replaced_pattern = replaced_pattern + do_not_propagate_pattern
+ pass
port = self.allocate_port(def_port)
value = value.replace(replaced_pattern, str(port), 1)
logger.info("Allocated port " + str(port) + " for " + replaced_pattern)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50dc76c9/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index 3343a8b..c07f05f 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -284,8 +284,9 @@ class TestCustomServiceOrchestrator(TestCase):
self.assertEqual(ret, 102)
+ @patch.object(CustomServiceOrchestrator, "is_port_available")
@patch.object(CustomServiceOrchestrator, "allocate_port")
- def test_allocate_port_combinations(self, allocate_port_mock):
+ def test_allocate_port_combinations(self, allocate_port_mock, is_port_available_mock):
tempdir = tempfile.gettempdir()
config = MagicMock()
config.get.return_value = "something"
@@ -296,13 +297,20 @@ class TestCustomServiceOrchestrator(TestCase):
dummy_controller = MagicMock()
orchestrator = CustomServiceOrchestrator(config, dummy_controller)
- allocate_port_mock.side_effect = [101, 102, 103]
+ is_port_available_mock.return_value = False
+ allocate_port_mock.side_effect = [101, 102, 103, 104, 105, 106]
ret = orchestrator.allocate_ports("1000", "${A.ALLOCATED_PORT}")
self.assertEqual(ret, "1000")
ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}", "${A.ALLOCATED_PORT}")
self.assertEqual(ret, "101")
ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT},${A.ALLOCATED_PORT}", "${A.ALLOCATED_PORT}")
self.assertEqual(ret, "102,103")
+ ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_0}", "${A.ALLOCATED_PORT}")
+ self.assertEqual(ret, "104")
+ ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_0}{DO_NOT_PROPAGATE}", "${A.ALLOCATED_PORT}")
+ self.assertEqual(ret, "105")
+ ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DO_NOT_PROPAGATE}", "${A.ALLOCATED_PORT}")
+ self.assertEqual(ret, "106")
@patch.object(CustomServiceOrchestrator, "is_port_available")
@@ -320,10 +328,14 @@ class TestCustomServiceOrchestrator(TestCase):
is_port_available_mock.return_value = True
ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_1005}", "${A.ALLOCATED_PORT}")
self.assertEqual(ret, "1005")
+
ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_1005}-${A.ALLOCATED_PORT}{DEFAULT_1006}",
"${A.ALLOCATED_PORT}")
self.assertEqual(ret, "1005-1006")
+ ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_1006}{DO_NOT_PROPAGATE}",
+ "${A.ALLOCATED_PORT}")
+ self.assertEqual(ret, "1006")
@patch("hostname.public_hostname")
@patch("os.path.isfile")
@@ -492,7 +504,7 @@ class TestCustomServiceOrchestrator(TestCase):
orchestrator.finalize_command(command, True, {})
self.assertEqual(command['configurations']['hbase-site']['log_root'], tempdir + "/log")
self.assertEqual(command['configurations']['hbase-site']['blog_root'], "/b/" + tempdir + "/log")
- self.assertEqual(command['configurations']['oozie-site']['b_port'], "${HBASE_REGIONSERVER.ALLOCATED_PORT}")
+ self.assertEqual(command['configurations']['oozie-site']['b_port'], "0")
self.assertEqual(orchestrator.applied_configs, command['configurations'])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50dc76c9/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index e35227c..7639625 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -20,6 +20,7 @@ package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.ConfigHelper;
@@ -104,6 +105,10 @@ public abstract class AbstractProviderService
return restOps;
}
+ @Override
+ public void notifyContainerCompleted(ContainerId containerId) {
+ }
+
public void setAgentRestOperations(AgentRestOperations agentRestOperations) {
this.restOps = agentRestOperations;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50dc76c9/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index bbe2001..28c874b 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -111,11 +111,12 @@ public class AgentProviderService extends AbstractProviderService implements
private static final String GLOBAL_CONFIG_TAG = "global";
private static final String LOG_FOLDERS_TAG = "LogFolders";
private static final String COMPONENT_DATA_TAG = "ComponentInstanceData";
- private static final String SHARED_PORT_TAG = "{SHARED}";
+ private static final String SHARED_PORT_TAG = "SHARED";
+ private static final String DO_NOT_PROPAGATE_TAG = "{DO_NOT_PROPAGATE}";
private static final int MAX_LOG_ENTRIES = 20;
private static final int DEFAULT_HEARTBEAT_MONITOR_INTERVAL = 60 * 1000;
private final Object syncLock = new Object();
- private final Map<String, String> allocatedPorts = new ConcurrentHashMap<>();
+ private final Map<String, Map<String, String>> allocatedPorts = new ConcurrentHashMap<>();
private int heartbeatMonitorInterval = 0;
private AgentClientProvider clientProvider;
private Map<String, ComponentInstanceState> componentStatuses = new ConcurrentHashMap<>();
@@ -381,11 +382,12 @@ public class AgentProviderService extends AbstractProviderService implements
if (ports != null && !ports.isEmpty()) {
for (Map.Entry<String, String> port : ports.entrySet()) {
log.info("Recording allocated port for {} as {}", port.getKey(), port.getValue());
- this.allocatedPorts.put(port.getKey(), port.getValue());
+ this.getAllocatedPorts().put(port.getKey(), port.getValue());
+ this.getAllocatedPorts(containerId).put(port.getKey(), port.getValue());
}
// component specific publishes
- processComponentSpecificPublishes(ports, containerId, heartBeat.getFqdn(), roleName);
+ processAndPublishComponentSpecificData(ports, containerId, heartBeat.getFqdn(), roleName);
}
CommandResult result = CommandResult.getCommandResult(report.getStatus());
Command command = Command.getCommand(report.getRoleCommand());
@@ -469,8 +471,21 @@ public class AgentProviderService extends AbstractProviderService implements
@Override
public void notifyContainerCompleted(ContainerId containerId) {
- if(containerId != null) {
+ if (containerId != null) {
String containerIdStr = containerId.toString();
+ if (this.componentInstanceData.containsKey(containerIdStr)) {
+ synchronized (this.componentInstanceData) {
+ this.componentInstanceData.remove(containerIdStr);
+ }
+ log.info("Removing container specific data for {}", containerIdStr);
+ publishComponentInstanceData();
+ }
+
+ if (this.allocatedPorts.containsKey(containerIdStr)) {
+ synchronized (this.allocatedPorts) {
+ this.allocatedPorts.remove(containerIdStr);
+ }
+ }
}
}
@@ -534,8 +549,8 @@ public class AgentProviderService extends AbstractProviderService implements
* @param description
* @param entries
*/
- protected void publishComponentConfiguration(String name, String description,
- Iterable<Map.Entry<String, String>> entries) {
+ protected void publishApplicationInstanceData(String name, String description,
+ Iterable<Map.Entry<String, String>> entries) {
PublishedConfiguration pubconf = new PublishedConfiguration();
pubconf.description = description;
pubconf.putValues(entries);
@@ -606,7 +621,7 @@ public class AgentProviderService extends AbstractProviderService implements
workFolders.put(String.format("%s-%s-%s", hostFqdn, containerId, key), folders.get(key));
}
- publishComponentConfiguration(LOG_FOLDERS_TAG, LOG_FOLDERS_TAG, (new HashMap<>(this.workFolders)).entrySet());
+ publishApplicationInstanceData(LOG_FOLDERS_TAG, LOG_FOLDERS_TAG, (new HashMap<>(this.workFolders)).entrySet());
}
@@ -627,7 +642,7 @@ public class AgentProviderService extends AbstractProviderService implements
if (status.getConfigs() != null) {
for (String key : status.getConfigs().keySet()) {
Map<String, String> configs = status.getConfigs().get(key);
- publishComponentConfiguration(key, key, configs.entrySet());
+ publishApplicationInstanceData(key, key, configs.entrySet());
}
if (hasExportGroups) {
@@ -665,7 +680,7 @@ public class AgentProviderService extends AbstractProviderService implements
map.put(export.getName(), value);
log.info("Preparing to publish. Key {} and Value {}", export.getName(), value);
}
- publishComponentConfiguration(exportGroupName, exportGroupName, map.entrySet());
+ publishApplicationInstanceData(exportGroupName, exportGroupName, map.entrySet());
}
}
}
@@ -676,10 +691,10 @@ public class AgentProviderService extends AbstractProviderService implements
}
/** Publish component instance specific data if the component demands it */
- protected void processComponentSpecificPublishes(Map<String, String> ports,
- String containerId,
- String hostFqdn,
- String roleName) {
+ protected void processAndPublishComponentSpecificData(Map<String, String> ports,
+ String containerId,
+ String hostFqdn,
+ String roleName) {
String portVarFormat = "${site.%s}";
String hostNamePattern = "${THIS_HOST}";
Map<String, String> toPublish = new HashMap<>();
@@ -687,23 +702,25 @@ public class AgentProviderService extends AbstractProviderService implements
Application application = getMetainfo().getApplication();
for (Component component : application.getComponents()) {
if (component.getName().equals(roleName)) {
- if(component.getExports().size() > 0) {
+ if (component.getExports().size() > 0) {
- for(Export export : component.getExports()) {
+ for (Export export : component.getExports()) {
String templateToExport = export.getValue();
- for(String portName : ports.keySet()) {
+ for (String portName : ports.keySet()) {
boolean publishData = false;
String portValPattern = String.format(portVarFormat, portName);
- if(templateToExport.contains(portValPattern)) {
+ if (templateToExport.contains(portValPattern)) {
templateToExport = templateToExport.replace(portValPattern, ports.get(portName));
publishData = true;
}
- if(templateToExport.contains(hostNamePattern)) {
+ if (templateToExport.contains(hostNamePattern)) {
templateToExport = templateToExport.replace(hostNamePattern, hostFqdn);
publishData = true;
}
- if(publishData) {
+ if (publishData) {
toPublish.put(export.getName(), templateToExport);
+ log.info("Publishing {} for name {} and container {}",
+ templateToExport, export.getName(), containerId);
}
}
}
@@ -711,26 +728,30 @@ public class AgentProviderService extends AbstractProviderService implements
}
}
- if(toPublish.size() > 0) {
+ if (toPublish.size() > 0) {
Map<String, String> perContainerData = null;
- if(!this.componentInstanceData.containsKey(containerId)) {
+ if (!this.componentInstanceData.containsKey(containerId)) {
perContainerData = new ConcurrentHashMap<>();
} else {
perContainerData = this.componentInstanceData.get(containerId);
}
perContainerData.putAll(toPublish);
this.componentInstanceData.put(containerId, perContainerData);
- Map<String, String> dataToPublish = new HashMap<>();
- synchronized (this.componentInstanceData) {
- for(String container : this.componentInstanceData.keySet()) {
- for(String prop : this.componentInstanceData.get(container).keySet()) {
- dataToPublish.put(
- container + "." + prop, this.componentInstanceData.get(container).get(prop));
- }
+ publishComponentInstanceData();
+ }
+ }
+
+ private void publishComponentInstanceData() {
+ Map<String, String> dataToPublish = new HashMap<>();
+ synchronized (this.componentInstanceData) {
+ for (String container : this.componentInstanceData.keySet()) {
+ for (String prop : this.componentInstanceData.get(container).keySet()) {
+ dataToPublish.put(
+ container + "." + prop, this.componentInstanceData.get(container).get(prop));
}
}
- publishComponentConfiguration(COMPONENT_DATA_TAG, COMPONENT_DATA_TAG, dataToPublish.entrySet());
}
+ publishApplicationInstanceData(COMPONENT_DATA_TAG, COMPONENT_DATA_TAG, dataToPublish.entrySet());
}
/**
@@ -864,7 +885,7 @@ public class AgentProviderService extends AbstractProviderService implements
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
- setInstallCommandConfigurations(cmd);
+ setInstallCommandConfigurations(cmd, containerId);
cmd.setCommandParams(setCommandParameters(scriptPath, false));
@@ -914,9 +935,9 @@ public class AgentProviderService extends AbstractProviderService implements
return cmdParams;
}
- private void setInstallCommandConfigurations(ExecutionCommand cmd) throws SliderException {
+ private void setInstallCommandConfigurations(ExecutionCommand cmd, String containerId) throws SliderException {
ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf);
+ Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId);
cmd.setConfigurations(configurations);
}
@@ -943,7 +964,7 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setCommandParams(setCommandParameters(scriptPath, false));
- Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf);
+ Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId);
cmd.setConfigurations(configurations);
@@ -998,17 +1019,33 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setCommandParams(setCommandParameters(scriptPath, true));
- Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf);
+ Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId);
cmd.setConfigurations(configurations);
response.addExecutionCommand(cmd);
}
protected Map<String, String> getAllocatedPorts() {
- return this.allocatedPorts;
+ return getAllocatedPorts(SHARED_PORT_TAG);
}
- private Map<String, Map<String, String>> buildCommandConfigurations(ConfTreeOperations appConf)
+ protected Map<String, Map<String, String>> getComponentInstanceData() {
+ return this.componentInstanceData;
+ }
+
+ protected Map<String, String> getAllocatedPorts(String containerId) {
+ if (!this.allocatedPorts.containsKey(containerId)) {
+ synchronized (this.allocatedPorts) {
+ if (!this.allocatedPorts.containsKey(containerId)) {
+ this.allocatedPorts.put(containerId, new ConcurrentHashMap());
+ }
+ }
+ }
+ return this.allocatedPorts.get(containerId);
+ }
+
+ private Map<String, Map<String, String>> buildCommandConfigurations(
+ ConfTreeOperations appConf, String containerId)
throws SliderException {
Map<String, Map<String, String>> configurations = new TreeMap<>();
@@ -1019,7 +1056,7 @@ public class AgentProviderService extends AbstractProviderService implements
//Add global
for (String configType : configs) {
addNamedConfiguration(configType, appConf.getGlobalOptions().options,
- configurations, tokens);
+ configurations, tokens, containerId);
}
return configurations;
@@ -1057,7 +1094,7 @@ public class AgentProviderService extends AbstractProviderService implements
private void addNamedConfiguration(String configName, Map<String, String> sourceConfig,
Map<String, Map<String, String>> configurations,
- Map<String, String> tokens) {
+ Map<String, String> tokens, String containerId) {
Map<String, String> config = new HashMap<>();
if (configName.equals(GLOBAL_CONFIG_TAG)) {
addDefaultGlobalConfig(config);
@@ -1070,12 +1107,17 @@ public class AgentProviderService extends AbstractProviderService implements
if (!this.getAllocatedPorts().isEmpty()) {
for (String key : config.keySet()) {
String value = config.get(key);
- if(!value.contains(SHARED_PORT_TAG)) {
- // If the config property is not shared then do not pass on the already allocated value
- String lookupKey = configName + "." + key;
+ String lookupKey = configName + "." + key;
+ if(!value.contains(DO_NOT_PROPAGATE_TAG)) {
+ // If the config property is shared then pass on the already allocated value
+ // from any container
if (this.getAllocatedPorts().containsKey(lookupKey)) {
config.put(key, getAllocatedPorts().get(lookupKey));
}
+ } else {
+ if (this.getAllocatedPorts(containerId).containsKey(lookupKey)) {
+ config.put(key, getAllocatedPorts(containerId).get(lookupKey));
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50dc76c9/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
index 7b73451..db28a22 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path
import org.apache.hadoop.service.LifecycleEvent
import org.apache.hadoop.service.ServiceStateChangeListener
import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.slider.api.ClusterDescription
import org.apache.slider.common.tools.SliderFileSystem
import org.apache.slider.core.conf.AggregateConf
@@ -229,4 +230,8 @@ class MockProviderService implements ProviderService {
throws MalformedURLException, IOException {
}
+
+ @Override
+ public void notifyContainerCompleted(ContainerId containerId) {
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50dc76c9/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 1f9cb41..10b1eb7 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -360,20 +360,20 @@ public class TestAgentProviderService {
Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
AgentProviderService aps = new AgentProviderService();
AgentProviderService mockAps = Mockito.spy(aps);
- doNothing().when(mockAps).publishComponentConfiguration(anyString(), anyString(), anyCollection());
+ doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
doReturn(metainfo).when(mockAps).getMetainfo();
Map<String, String> ports = new HashMap<>();
ports.put("global.listen_port", "10010");
- mockAps.processComponentSpecificPublishes(ports,
- "cid1",
- "host1",
- "HBASE_REGIONSERVER");
+ mockAps.processAndPublishComponentSpecificData(ports,
+ "cid1",
+ "host1",
+ "HBASE_REGIONSERVER");
ArgumentCaptor<Collection> entriesCaptor = ArgumentCaptor.
forClass(Collection.class);
ArgumentCaptor<String> publishNameCaptor = ArgumentCaptor.
forClass(String.class);
- Mockito.verify(mockAps, Mockito.times(1)).publishComponentConfiguration(
+ Mockito.verify(mockAps, Mockito.times(1)).publishApplicationInstanceData(
anyString(),
publishNameCaptor.capture(),
entriesCaptor.capture());
@@ -422,7 +422,7 @@ public class TestAgentProviderService {
ComponentInstanceState componentStatus = new ComponentInstanceState("HBASE_MASTER", "aid", "cid");
AgentProviderService mockAps = Mockito.spy(aps);
- doNothing().when(mockAps).publishComponentConfiguration(anyString(), anyString(), anyCollection());
+ doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
doReturn(metainfo).when(mockAps).getMetainfo();
doReturn(roleClusterNodeMap).when(mockAps).getRoleClusterNodeMapping();
@@ -430,7 +430,7 @@ public class TestAgentProviderService {
assert componentStatus.getConfigReported() == true;
ArgumentCaptor<Collection> entriesCaptor = ArgumentCaptor.
forClass(Collection.class);
- Mockito.verify(mockAps, Mockito.times(3)).publishComponentConfiguration(
+ Mockito.verify(mockAps, Mockito.times(3)).publishApplicationInstanceData(
anyString(),
anyString(),
entriesCaptor.capture());
@@ -627,7 +627,7 @@ public class TestAgentProviderService {
anyString(),
anyString(),
any(HeartBeatResponse.class));
- doNothing().when(mockAps).publishComponentConfiguration(
+ doNothing().when(mockAps).publishApplicationInstanceData(
anyString(),
anyString(),
anyCollection());
@@ -797,13 +797,44 @@ public class TestAgentProviderService {
log.warn(he.getMessage());
}
- Mockito.verify(mockAps, Mockito.times(1)).publishComponentConfiguration(
+ Mockito.verify(mockAps, Mockito.times(1)).publishApplicationInstanceData(
anyString(),
anyString(),
anyCollection());
}
@Test
+ public void testNotifyContainerCompleted() {
+ AgentProviderService aps = new AgentProviderService();
+ AgentProviderService mockAps = Mockito.spy(aps);
+ doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
+
+ ContainerId cid = new MyContainerId(1);
+ String id = cid.toString();
+ mockAps.getAllocatedPorts().put("a", "100");
+ mockAps.getAllocatedPorts(id).put("b", "101");
+ mockAps.getAllocatedPorts("cid2").put("c", "102");
+
+ mockAps.getComponentInstanceData().put("cid2", new HashMap<String, String>());
+ mockAps.getComponentInstanceData().put(id, new HashMap<String, String>());
+
+ Assert.assertNotNull(mockAps.getComponentInstanceData().get(id));
+ Assert.assertNotNull(mockAps.getComponentInstanceData().get("cid2"));
+
+ Assert.assertEquals(mockAps.getAllocatedPorts().size(), 1);
+ Assert.assertEquals(mockAps.getAllocatedPorts(id).size(), 1);
+ Assert.assertEquals(mockAps.getAllocatedPorts("cid2").size(), 1);
+ mockAps.notifyContainerCompleted(new MyContainerId(1));
+
+ Assert.assertNull(mockAps.getComponentInstanceData().get(id));
+ Assert.assertNotNull(mockAps.getComponentInstanceData().get("cid2"));
+
+ Assert.assertEquals(mockAps.getAllocatedPorts().size(), 1);
+ Assert.assertEquals(mockAps.getAllocatedPorts(id).size(), 0);
+ Assert.assertEquals(mockAps.getAllocatedPorts("cid2").size(), 1);
+ }
+
+ @Test
public void testAddInstallCommand() throws Exception {
InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
@@ -867,6 +898,8 @@ public class TestAgentProviderService {
treeOps.set("config_types", "hbase-site");
treeOps.getGlobalOptions().put("site.hbase-site.a.port", "${HBASE_MASTER.ALLOCATED_PORT}");
treeOps.getGlobalOptions().put("site.hbase-site.b.port", "${HBASE_MASTER.ALLOCATED_PORT}");
+ treeOps.getGlobalOptions().put("site.hbase-site.random.port", "${HBASE_MASTER.ALLOCATED_PORT}{DO_NOT_PROPAGATE}");
+ treeOps.getGlobalOptions().put("site.hbase-site.random2.port", "${HBASE_MASTER.ALLOCATED_PORT}");
expect(access.getAppConfSnapshot()).andReturn(treeOps).anyTimes();
expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes();
@@ -885,6 +918,9 @@ public class TestAgentProviderService {
allocatedPorts.put("hbase-site.a.port", "10023");
allocatedPorts.put("hbase-site.b.port", "10024");
doReturn(allocatedPorts).when(mockAps).getAllocatedPorts();
+ Map<String, String> allocatedPorts2 = new HashMap<>();
+ allocatedPorts2.put("hbase-site.random.port", "10025");
+ doReturn(allocatedPorts2).when(mockAps).getAllocatedPorts(anyString());
replay(access);
@@ -894,6 +930,8 @@ public class TestAgentProviderService {
Assert.assertTrue(hbaseSiteConf.containsKey("a.port"));
Assert.assertTrue(hbaseSiteConf.get("a.port").equals("10023"));
Assert.assertTrue(hbaseSiteConf.get("b.port").equals("10024"));
+ Assert.assertTrue(hbaseSiteConf.get("random.port").equals("10025"));
+ Assert.assertTrue(hbaseSiteConf.get("random2.port").equals("${HBASE_MASTER.ALLOCATED_PORT}"));
}
private static class MyContainer extends Container {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50dc76c9/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
index 2427009..92e6491 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
@@ -46,8 +46,8 @@ public class TestAgentProviderService extends AgentProviderService{
dummyProps.put("prop1", "val1");
dummyProps.put("prop2", "val2");
log.info("publishing dummy-site.xml with values {}", dummyProps);
- publishComponentConfiguration("dummy-site", "dummy configuration",
- dummyProps.entrySet());
+ publishApplicationInstanceData("dummy-site", "dummy configuration",
+ dummyProps.entrySet());
}
[33/50] [abbrv] git commit: SLIDER-286 adds the live container list
as an argument in the bind() operation...all implementations currently
discard this.
Posted by st...@apache.org.
SLIDER-286 adds the live container list as an argument in the bind() operation...all implementations currently discard this.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/11335c1b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/11335c1b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/11335c1b
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 11335c1be7f41a27354417e15ed0aa83829684f2
Parents: 1350491
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 7 17:37:59 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 7 17:37:59 2014 +0100
----------------------------------------------------------------------
.../slider/providers/AbstractProviderService.java | 6 ++++--
.../org/apache/slider/providers/ProviderService.java | 14 +++++++++-----
.../slider/server/appmaster/SliderAppMaster.java | 13 ++++++++-----
.../appmaster/model/mock/MockProviderService.groovy | 3 ++-
.../web/rest/publisher/TestAgentProviderService.java | 10 +++++++---
5 files changed, 30 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/11335c1b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index 158640d..b431a10 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -20,6 +20,7 @@ package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.slider.api.ClusterDescription;
@@ -46,7 +47,6 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collection;
import java.util.HashMap;
@@ -97,7 +97,9 @@ public abstract class AbstractProviderService
@Override
public void bind(StateAccessForProviders stateAccessor,
- RegistryViewForProviders reg, AMViewForProviders amView) {
+ RegistryViewForProviders reg,
+ AMViewForProviders amView,
+ List<Container> liveContainers) {
this.amState = stateAccessor;
this.registry = reg;
this.amView = amView;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/11335c1b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index c0a75c1..1466350 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -42,6 +42,7 @@ import org.apache.slider.server.services.registry.RegistryViewForProviders;
import java.io.File;
import java.io.IOException;
import java.net.URL;
+import java.util.List;
import java.util.Map;
public interface ProviderService extends ProviderCore,
@@ -74,7 +75,7 @@ public interface ProviderService extends ProviderCore,
/**
* Notify the providers of container completion
- * @param containerId
+ * @param containerId container that has completed
*/
void notifyContainerCompleted(ContainerId containerId);
@@ -148,12 +149,15 @@ public interface ProviderService extends ProviderCore,
/**
* bind operation -invoked before the service is started
* @param stateAccessor interface offering read access to the state
- * @param registry
- * @param amView
+ * @param registry registry view
+ * @param amView AM access
+ * @param liveContainers list of live containers. If non-empty, the AM was
+ * restarted with one more running containers.
*/
void bind(StateAccessForProviders stateAccessor,
- RegistryViewForProviders registry,
- AMViewForProviders amView);
+ RegistryViewForProviders registry,
+ AMViewForProviders amView,
+ List<Container> liveContainers);
/**
* Returns the agent rest operations interface.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/11335c1b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index e84103c..bc63351 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -133,6 +133,7 @@ import java.net.URLClassLoader;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
@@ -561,7 +562,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
Map<String, String> envVars;
-
+ List<Container> liveContainers;
/**
* It is critical this section is synchronized, to stop async AM events
* arriving while registering a restarting AM.
@@ -663,8 +664,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
// extract container list
- List<Container> liveContainers =
- response.getContainersFromPreviousAttempts();
+
+ liveContainers = response.getContainersFromPreviousAttempts();
//now validate the installation
Configuration providerConf =
@@ -724,8 +725,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//Give the provider restricted access to the state, registry
- providerService.bind(stateForProviders, registry, this);
- sliderAMProvider.bind(stateForProviders, registry, null);
+ providerService.bind(stateForProviders, registry, this,
+ liveContainers);
+ sliderAMProvider.bind(stateForProviders, registry, this,
+ liveContainers);
// now do the registration
registerServiceInstance(clustername, appid);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/11335c1b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
index 315537c..c46b828 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
@@ -202,7 +202,8 @@ class MockProviderService implements ProviderService {
void bind(
StateAccessForProviders stateAccessor,
RegistryViewForProviders registry,
- AMViewForProviders amView) {
+ AMViewForProviders amView,
+ List<Container> liveContainers) {
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/11335c1b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
index 92e6491..5579592 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
@@ -16,6 +16,7 @@
*/
package org.apache.slider.server.appmaster.web.rest.publisher;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.slider.providers.agent.AgentProviderService;
import org.apache.slider.server.appmaster.AMViewForProviders;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
@@ -24,12 +25,13 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
/**
*
*/
-public class TestAgentProviderService extends AgentProviderService{
+public class TestAgentProviderService extends AgentProviderService {
protected static final Logger log =
LoggerFactory.getLogger(TestAgentProviderService.class);
@@ -40,8 +42,10 @@ public class TestAgentProviderService extends AgentProviderService{
@Override
public void bind(StateAccessForProviders stateAccessor,
- RegistryViewForProviders reg, AMViewForProviders amView) {
- super.bind(stateAccessor, reg, amView);
+ RegistryViewForProviders reg,
+ AMViewForProviders amView,
+ List<Container> liveContainers) {
+ super.bind(stateAccessor, reg, amView, liveContainers);
Map<String,String> dummyProps = new HashMap<>();
dummyProps.put("prop1", "val1");
dummyProps.put("prop2", "val2");
[21/50] [abbrv] git commit: SLIDER-296 ContainerReleaseSelectors have
the job of choosing containers to release.
Posted by st...@apache.org.
SLIDER-296 ContainerReleaseSelectors have the job of choosing containers to release.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5ce953ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5ce953ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5ce953ca
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 5ce953ca301a90e27b1cbe193a482a9cb83c3084
Parents: 32fd3f5
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 6 16:58:27 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 6 16:58:27 2014 +0100
----------------------------------------------------------------------
.../apache/slider/common/tools/Comparators.java | 58 ++++++
.../providers/AbstractProviderService.java | 18 +-
.../slider/providers/ProviderService.java | 12 +-
.../server/appmaster/SliderAppMaster.java | 4 +-
.../slider/server/appmaster/state/AppState.java | 134 +++++++++----
.../state/ContainerReleaseSelector.java | 38 ++++
.../MostRecentContainerReleaseSelector.java | 52 +++++
.../appmaster/state/SimpleReleaseSelector.java | 34 ++++
.../TestAppStateContainerFailure.groovy | 166 ----------------
.../appstate/TestAppStateDynamicRoles.groovy | 93 ---------
.../TestAppStateRebuildOnAMRestart.groovy | 124 ------------
.../appstate/TestAppStateRolePlacement.groovy | 99 ----------
.../appstate/TestAppStateRoleRelease.groovy | 82 --------
.../TestContainerResourceAllocations.groovy | 108 -----------
.../model/appstate/TestFlexDynamicRoles.groovy | 189 ------------------
.../TestMockAppStateContainerFailure.groovy | 166 ++++++++++++++++
.../TestMockAppStateDynamicRoles.groovy | 94 +++++++++
.../TestMockAppStateFlexDynamicRoles.groovy | 190 +++++++++++++++++++
.../appstate/TestMockAppStateFlexing.groovy | 126 ++++++++++++
.../TestMockAppStateRMOperations.groovy | 176 +++++++++++++++++
.../TestMockAppStateRebuildOnAMRestart.groovy | 124 ++++++++++++
.../TestMockAppStateRolePlacement.groovy | 99 ++++++++++
.../appstate/TestMockAppStateRoleRelease.groovy | 82 ++++++++
.../TestMockContainerResourceAllocations.groovy | 108 +++++++++++
.../model/appstate/TestMockFlexing.groovy | 128 -------------
.../model/appstate/TestMockRMOperations.groovy | 176 -----------------
.../model/mock/BaseMockAppStateTest.groovy | 2 +-
.../model/mock/MockProviderService.groovy | 56 +++---
.../web/rest/agent/TestAMAgentWebServices.java | 3 +-
.../management/TestAMManagementWebServices.java | 3 +-
30 files changed, 1510 insertions(+), 1234 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java b/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
new file mode 100644
index 0000000..0ccca0f
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import java.io.Serializable;
+import java.util.Comparator;
+
+public class Comparators {
+
+ public static class LongComparator implements Comparator<Long>, Serializable {
+ @Override
+ public int compare(Long o1, Long o2) {
+ long result = o1 - o2;
+ // need to comparisons with a diff greater than integer size
+ if (result < 0 ) {
+ return -1;
+ } else if (result >0) {
+ return 1;
+ }
+ return 0;
+ }
+ }
+
+ /**
+ * Little template class to reverse any comparitor
+ * @param <CompareType> the type that is being compared
+ */
+ public static class ComparatorReverser<CompareType> implements Comparator<CompareType>,
+ Serializable {
+
+ final Comparator<CompareType> instance;
+
+ public ComparatorReverser(Comparator<CompareType> instance) {
+ this.instance = instance;
+ }
+
+ @Override
+ public int compare(CompareType first, CompareType second) {
+ return instance.compare(second, first);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index 7639625..7483a8d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -32,6 +32,8 @@ import org.apache.slider.core.main.ExitCodeProvider;
import org.apache.slider.core.registry.info.RegisteredEndpoint;
import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.server.appmaster.AMViewForProviders;
+import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
+import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
import org.apache.slider.server.services.registry.RegistryViewForProviders;
@@ -318,11 +320,21 @@ public abstract class AbstractProviderService
}
@Override
public void applyInitialRegistryDefinitions(URL unsecureWebAPI,
- URL secureWebAPI,
- ServiceInstanceData registryInstanceData) throws MalformedURLException,
- IOException {
+ URL secureWebAPI,
+ ServiceInstanceData registryInstanceData) throws IOException {
this.amWebAPI = unsecureWebAPI;
this.registryInstanceData = registryInstanceData;
}
+
+ /**
+ * {@inheritDoc}
+ *
+ *
+ * @return The base implementation returns the most recent containers first.
+ */
+ @Override
+ public ContainerReleaseSelector createContainerReleaseSelector() {
+ return new MostRecentContainerReleaseSelector();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index b9fa34c..4278b09 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -33,6 +33,7 @@ import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.core.main.ExitCodeProvider;
import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.server.appmaster.AMViewForProviders;
+import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
import org.apache.slider.server.services.registry.RegistryViewForProviders;
@@ -172,6 +173,13 @@ public interface ProviderService extends ProviderCore, Service,
*/
void applyInitialRegistryDefinitions(URL unsecureWebAPI,
URL secureWebAPI,
- ServiceInstanceData registryInstanceData) throws MalformedURLException,
- IOException;
+ ServiceInstanceData registryInstanceData)
+ throws IOException;
+
+ /**
+ * Create the container release selector for this provider...any policy
+ * can be implemented
+ * @return the selector to use for choosing containers.
+ */
+ ContainerReleaseSelector createContainerReleaseSelector();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index b3650ca..dbf08e1 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -105,6 +105,7 @@ import org.apache.slider.server.appmaster.state.ProviderAppState;
import org.apache.slider.server.appmaster.state.RMOperationHandler;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.apache.slider.server.appmaster.state.SimpleReleaseSelector;
import org.apache.slider.server.appmaster.web.AgentService;
import org.apache.slider.server.appmaster.web.rest.agent.AgentWebApp;
import org.apache.slider.server.appmaster.web.SliderAMWebApp;
@@ -677,7 +678,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
fs.getFileSystem(),
historyDir,
liveContainers,
- appInformation);
+ appInformation,
+ new SimpleReleaseSelector());
// add the AM to the list of nodes in the cluster
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 08d43db..c20c924 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -18,7 +18,9 @@
package org.apache.slider.server.appmaster.state;
+import com.beust.jcommander.internal.Lists;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -63,7 +65,9 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedList;
import java.util.List;
+import java.util.ListIterator;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -251,6 +255,8 @@ public class AppState {
private int failureThreshold = 10;
private String logServerURL = "";
+
+ private ContainerReleaseSelector containerReleaseSelector;
public AppState(AbstractRecordFactory recordFactory) {
this.recordFactory = recordFactory;
@@ -440,22 +446,27 @@ public class AppState {
* @param historyDir directory containing history files
* @param liveContainers list of live containers supplied on an AM restart
* @param applicationInfo
+ * @param releaseSelector
*/
public synchronized void buildInstance(AggregateConf instanceDefinition,
- Configuration appmasterConfig, Configuration publishedProviderConf,
+ Configuration appmasterConfig,
+ Configuration publishedProviderConf,
List<ProviderRole> providerRoles,
FileSystem fs,
Path historyDir,
List<Container> liveContainers,
- Map<String, String> applicationInfo) throws
- BadClusterStateException,
- BadConfigException,
- IOException {
+ Map<String, String> applicationInfo,
+ SimpleReleaseSelector releaseSelector)
+ throws BadClusterStateException, BadConfigException, IOException {
+ Preconditions.checkArgument(instanceDefinition != null);
+ Preconditions.checkArgument(releaseSelector != null);
+
this.publishedProviderConf = publishedProviderConf;
- this.applicationInfo = applicationInfo != null ? applicationInfo
- : new HashMap<String, String>();
+ this.applicationInfo = applicationInfo != null ? applicationInfo
+ : new HashMap<String, String>();
clientProperties = new HashMap<>();
+ containerReleaseSelector = releaseSelector;
Set<String> confKeys = ConfigHelper.sortedConfigKeys(publishedProviderConf);
@@ -465,8 +476,8 @@ public class AppState {
String val = publishedProviderConf.get(key);
clientProperties.put(key, val);
}
-
-
+
+
// set the cluster specification (once its dependency the client properties
// is out the way
@@ -479,15 +490,16 @@ public class AppState {
}
ConfTreeOperations resources =
- instanceDefinition.getResourceOperations();
-
+ instanceDefinition.getResourceOperations();
+
Set<String> roleNames = resources.getComponentNames();
for (String name : roleNames) {
if (!roles.containsKey(name)) {
// this is a new value
log.info("Adding new role {}", name);
MapOperations resComponent = resources.getComponent(name);
- ProviderRole dynamicRole = createDynamicProviderRole(name, resComponent);
+ ProviderRole dynamicRole =
+ createDynamicProviderRole(name, resComponent);
buildRole(dynamicRole);
providerRoles.add(dynamicRole);
}
@@ -498,24 +510,24 @@ public class AppState {
//set the livespan
MapOperations globalInternalOpts =
- instanceDefinition.getInternalOperations().getGlobalOptions();
+ instanceDefinition.getInternalOperations().getGlobalOptions();
startTimeThreshold = globalInternalOpts.getOptionInt(
- OptionKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE,
- OptionKeys.DEFAULT_CONTAINER_FAILURE_SHORTLIFE);
-
+ OptionKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE,
+ OptionKeys.DEFAULT_CONTAINER_FAILURE_SHORTLIFE);
+
failureThreshold = globalInternalOpts.getOptionInt(
- OptionKeys.INTERNAL_CONTAINER_FAILURE_THRESHOLD,
- OptionKeys.DEFAULT_CONTAINER_FAILURE_THRESHOLD);
+ OptionKeys.INTERNAL_CONTAINER_FAILURE_THRESHOLD,
+ OptionKeys.DEFAULT_CONTAINER_FAILURE_THRESHOLD);
initClusterStatus();
// add the roles
roleHistory = new RoleHistory(providerRoles);
roleHistory.onStart(fs, historyDir);
-
+
//rebuild any live containers
rebuildModelFromRestart(liveContainers);
-
+
// any am config options to pick up
logServerURL = appmasterConfig.get(YarnConfiguration.YARN_LOG_SERVER_URL,
@@ -845,6 +857,27 @@ public class AppState {
return nodes;
}
+
+ /**
+ * enum nodes by role ID, from either the active or live node list
+ * @param roleId role the container must be in
+ * @param active flag to indicate "use active list" rather than the smaller
+ * "live" list
+ * @return a list of nodes, may be empty
+ */
+ public synchronized List<RoleInstance> enumNodesWithRoleId(int roleId,
+ boolean active) {
+ List<RoleInstance> nodes = new ArrayList<>();
+ Collection<RoleInstance> allRoleInstances;
+ allRoleInstances = active? activeContainers.values() : liveNodes.values();
+ for (RoleInstance node : allRoleInstances) {
+ if (node.roleId == roleId) {
+ nodes.add(node);
+ }
+ }
+ return nodes;
+ }
+
/**
* Build an instance map.
@@ -911,19 +944,19 @@ public class AppState {
throws SliderInternalStateException {
ContainerId id = container.getId();
//look up the container
- RoleInstance info = getActiveContainer(id);
- if (info == null) {
+ RoleInstance instance = getActiveContainer(id);
+ if (instance == null) {
throw new SliderInternalStateException(
- "No active container with ID " + id.toString());
+ "No active container with ID " + id);
}
//verify that it isn't already released
if (containersBeingReleased.containsKey(id)) {
throw new SliderInternalStateException(
"Container %s already queued for release", id);
}
- info.released = true;
- containersBeingReleased.put(id, info.container);
- RoleStatus role = lookupRoleStatus(info.roleId);
+ instance.released = true;
+ containersBeingReleased.put(id, instance.container);
+ RoleStatus role = lookupRoleStatus(instance.roleId);
role.incReleasing();
roleHistory.onContainerReleaseSubmitted(container);
}
@@ -1229,8 +1262,8 @@ public class AppState {
if (shortLived) {
roleStatus.incStartFailed();
}
-
- if (failedContainer!= null) {
+
+ if (failedContainer != null) {
roleHistory.onFailedContainer(failedContainer, shortLived);
}
@@ -1485,18 +1518,43 @@ public class AppState {
// get the nodes to release
int roleId = role.getKey();
- List<NodeInstance> nodesForRelease =
- roleHistory.findNodesForRelease(roleId, excess);
-
- for (NodeInstance node : nodesForRelease) {
- RoleInstance possible = findRoleInstanceOnHost(node, roleId);
- if (possible == null) {
- throw new SliderInternalStateException(
- "Failed to find a container to release on node %s", node.hostname);
+
+ // enum all active nodes that aren't being released
+ List<RoleInstance> containersToRelease = enumNodesWithRoleId(roleId, true);
+
+ // cut all release-in-progress nodes
+ ListIterator<RoleInstance> li = containersToRelease.listIterator();
+ while (li.hasNext()) {
+ RoleInstance next = li.next();
+ if (next.released) {
+ li.remove();
}
- containerReleaseSubmitted(possible.container);
- operations.add(new ContainerReleaseOperation(possible.getId()));
+ }
+
+ // warn if the desired state can't be reaced
+ if (containersToRelease.size() < excess) {
+ log.warn("Not enough nodes to release...short of {} nodes",
+ containersToRelease.size() - excess);
+ }
+
+ // ask the release selector to sort the targets
+ containersToRelease = containerReleaseSelector.sortCandidates(
+ roleId,
+ containersToRelease,
+ excess);
+
+ //crop to the excess
+ List<RoleInstance> finalCandidates = (excess < containersToRelease.size())
+ ? containersToRelease.subList(0, excess)
+ : containersToRelease;
+
+
+ // then build up a release operation, logging each container as released
+ for (RoleInstance possible : finalCandidates) {
+ log.debug("Targeting for release: {}", possible);
+ containerReleaseSubmitted(possible.container);
+ operations.add(new ContainerReleaseOperation(possible.getId()));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java
new file mode 100644
index 0000000..0cbc134
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import java.util.List;
+
+/**
+ * Interface implemented by anything that must choose containers to release
+ *
+ */
+public interface ContainerReleaseSelector {
+
+ /**
+ * Given a list of candidate containers, return a sorted version of the priority
+ * in which they should be released.
+ * @param candidates candidate list ... everything considered suitable
+ * @return
+ */
+ List<RoleInstance> sortCandidates(int roleId,
+ List<RoleInstance> candidates,
+ int minimumToSelect);
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
new file mode 100644
index 0000000..9d936a1
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import org.apache.slider.common.tools.Comparators;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+/**
+ * Sort the candidate list by the most recent container first.
+ */
+public class MostRecentContainerReleaseSelector implements ContainerReleaseSelector {
+
+ @Override
+ public List<RoleInstance> sortCandidates(int roleId,
+ List<RoleInstance> candidates,
+ int minimumToSelect) {
+ Collections.sort(candidates, new newerThan());
+ return candidates;
+ }
+
+ private static class newerThan implements Comparator<RoleInstance>, Serializable {
+ private final Comparator<Long> innerComparator =
+ new Comparators.ComparatorReverser<>(new Comparators.LongComparator());
+ public int compare(RoleInstance o1, RoleInstance o2) {
+ return innerComparator.compare(o1.createTime, o2.createTime);
+
+ }
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java
new file mode 100644
index 0000000..b7f0e05
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import java.util.List;
+
+/**
+ * Simplest release selector simply returns the list
+ */
+public class SimpleReleaseSelector implements ContainerReleaseSelector {
+
+ @Override
+ public List<RoleInstance> sortCandidates(int roleId,
+ List<RoleInstance> candidates,
+ int minimumToSelect) {
+ return candidates;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateContainerFailure.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateContainerFailure.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateContainerFailure.groovy
deleted file mode 100644
index 9c17763..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateContainerFailure.groovy
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.ContainerId
-import org.apache.slider.core.exceptions.SliderException
-import org.apache.slider.core.exceptions.TriggerClusterTeardownException
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockRoles
-import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
-import org.apache.slider.server.appmaster.state.*
-import org.junit.Test
-
-/**
- * Test that if you have >1 role, the right roles are chosen for release.
- */
-@CompileStatic
-@Slf4j
-class TestAppStateContainerFailure extends BaseMockAppStateTest
- implements MockRoles {
-
- @Override
- String getTestName() {
- return "TestAppStateContainerFailure"
- }
-
- /**
- * Small cluster with multiple containers per node,
- * to guarantee many container allocations on each node
- * @return
- */
- @Override
- MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(8000, 4)
- }
-
- @Test
- public void testShortLivedFail() throws Throwable {
-
- role0Status.desired = 1
- List<RoleInstance> instances = createAndStartNodes()
- assert instances.size() == 1
-
- RoleInstance instance = instances[0]
- long created = instance.createTime
- long started = instance.startTime
- assert created > 0
- assert started >= created
- List<ContainerId> ids = extractContainerIds(instances, 0)
-
- ContainerId cid = ids[0]
- assert appState.isShortLived(instance)
- AppState.NodeCompletionResult result = appState.onCompletedNode(containerStatus(cid, 1))
- assert result.roleInstance != null
- assert result.containerFailed
- RoleStatus status = role0Status
- assert status.failed == 1
- assert status.startFailed == 1
-
- //view the world
- appState.getRoleHistory().dump();
- List<NodeInstance> queue = appState.roleHistory.cloneAvailableList(0)
- assert queue.size() == 0
-
- }
-
- @Test
- public void testLongLivedFail() throws Throwable {
-
- role0Status.desired = 1
- List<RoleInstance> instances = createAndStartNodes()
- assert instances.size() == 1
-
- RoleInstance instance = instances[0]
- instance.startTime = System.currentTimeMillis() - 60 * 60 * 1000;
- assert !appState.isShortLived(instance)
- List<ContainerId> ids = extractContainerIds(instances, 0)
-
- ContainerId cid = ids[0]
- AppState.NodeCompletionResult result = appState.onCompletedNode(
- containerStatus(cid, 1))
- assert result.roleInstance != null
- assert result.containerFailed
- RoleStatus status = role0Status
- assert status.failed == 1
- assert status.startFailed == 0
-
- //view the world
- appState.getRoleHistory().dump();
- List<NodeInstance> queue = appState.roleHistory.cloneAvailableList(0)
- assert queue.size() == 1
-
- }
-
- @Test
- public void testNodeStartFailure() throws Throwable {
-
- role0Status.desired = 1
- List<RoleInstance> instances = createAndSubmitNodes()
- assert instances.size() == 1
-
- RoleInstance instance = instances[0]
-
- List<ContainerId> ids = extractContainerIds(instances, 0)
-
- ContainerId cid = ids[0]
- appState.onNodeManagerContainerStartFailed(cid, new SliderException("oops"))
- RoleStatus status = role0Status
- assert status.failed == 1
- assert status.startFailed == 1
-
-
- RoleHistory history = appState.roleHistory
- history.dump();
- List<NodeInstance> queue = history.cloneAvailableList(0)
- assert queue.size() == 0
-
- NodeInstance ni = history.getOrCreateNodeInstance(instance.container)
- NodeEntry re = ni.get(0)
- assert re.failed == 1
- assert re.startFailed == 1
- }
-
- @Test
- public void testRecurrentStartupFailure() throws Throwable {
-
- role0Status.desired = 1
- try {
- for (int i = 0; i< 100; i++) {
- List<RoleInstance> instances = createAndSubmitNodes()
- assert instances.size() == 1
-
- List<ContainerId> ids = extractContainerIds(instances, 0)
-
- ContainerId cid = ids[0]
- log.info("$i instance $instances[0] $cid")
- assert cid
- appState.onNodeManagerContainerStartFailed(cid, new SliderException("oops"))
- AppState.NodeCompletionResult result = appState.onCompletedNode(containerStatus(cid))
- assert result.containerFailed
- }
- fail("Cluster did not fail from too many startup failures")
- } catch (TriggerClusterTeardownException teardown) {
- log.info("Exception $teardown.exitCode : $teardown")
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy
deleted file mode 100644
index 6d70885..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.conf.Configuration
-import org.apache.slider.api.ResourceKeys
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
-import org.apache.slider.server.appmaster.model.mock.MockRoles
-import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
-import org.apache.slider.server.appmaster.state.AbstractRMOperation
-import org.apache.slider.server.appmaster.state.AppState
-import org.apache.slider.server.appmaster.state.RoleInstance
-import org.junit.Test
-
-/**
- * Test that if you have >1 role, the right roles are chosen for release.
- */
-@CompileStatic
-@Slf4j
-class TestAppStateDynamicRoles extends BaseMockAppStateTest
- implements MockRoles {
-
- @Override
- String getTestName() {
- return "TestAppStateDynamicRoles"
- }
-
- /**
- * Small cluster with multiple containers per node,
- * to guarantee many container allocations on each node
- * @return
- */
- @Override
- MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(4, 4)
- }
-
- @Override
- void initApp() {
- super.initApp()
- appState = new AppState(new MockRecordFactory())
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
-
- def instance = factory.newInstanceDefinition(0,0,0)
-
- def opts = [
- (ResourceKeys.COMPONENT_INSTANCES): "1",
- (ResourceKeys.COMPONENT_PRIORITY): "4",
- ]
-
- instance.resourceOperations.components["dynamic"]= opts
-
-
- appState.buildInstance(
- instance,
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- null,
- null)
- }
-
- @Test
- public void testAllocateReleaseRealloc() throws Throwable {
-
- List<RoleInstance> instances = createAndStartNodes()
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- appState.getRoleHistory().dump();
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy
deleted file mode 100644
index 8d7ff4f..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.yarn.api.records.Container
-import org.apache.slider.api.StatusKeys
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
-import org.apache.slider.server.appmaster.model.mock.MockRoles
-import org.apache.slider.server.appmaster.state.*
-import org.junit.Test
-
-/**
- * Test that if you have >1 role, the right roles are chosen for release.
- */
-@CompileStatic
-@Slf4j
-class TestAppStateRebuildOnAMRestart extends BaseMockAppStateTest
- implements MockRoles {
-
- @Override
- String getTestName() {
- return "TestAppStateRebuildOnAMRestart"
- }
-
- @Test
- public void testRebuild() throws Throwable {
-
- int r0 = 1
- int r1 = 2
- int r2 = 3
- role0Status.desired = r0
- role1Status.desired = r1
- role2Status.desired = r2
- List<RoleInstance> instances = createAndStartNodes()
-
- int clusterSize = r0 + r1 + r2
- assert instances.size() == clusterSize
-
- //clone the list
- List<RoleInstance> cloned = [];
- List<Container> containers = []
- instances.each { RoleInstance elt ->
- cloned.add(elt.clone() as RoleInstance)
- containers.add(elt.container)
- }
- NodeMap nodemap = appState.roleHistory.cloneNodemap()
-
- // now destroy the app state
- appState = new AppState(new MockRecordFactory())
-
- //and rebuild
- appState.buildInstance(
- factory.newInstanceDefinition(r0, r1, r2),
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- containers,
- null)
-
- assert appState.getStartedCountainerCount() == clusterSize
-
- appState.getRoleHistory().dump();
-
- //check that the app state direct structures match
- List<RoleInstance> r0live = appState.enumLiveNodesInRole(ROLE0)
- List<RoleInstance> r1live = appState.enumLiveNodesInRole(ROLE1)
- List<RoleInstance> r2live = appState.enumLiveNodesInRole(ROLE2)
-
- assert r0 == r0live.size()
- assert r1 == r1live.size()
- assert r2 == r2live.size()
-
- //now examine the role history
- NodeMap newNodemap = appState.roleHistory.cloneNodemap()
-
- for (NodeInstance nodeInstance : newNodemap.values()) {
- String hostname = nodeInstance.hostname
- NodeInstance orig = nodemap[hostname]
- assertNotNull("Null entry in original nodemap for " + hostname, orig)
-
- for (int i = 0; i < ROLE_COUNT; i++) {
-
- assert (nodeInstance.getActiveRoleInstances(i) ==
- orig.getActiveRoleInstances(i))
- NodeEntry origRE = orig.getOrCreate(i)
- NodeEntry newRE = nodeInstance.getOrCreate(i)
- assert origRE.live == newRE.live
- assert newRE.starting == 0
- }
- }
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 0
-
- def status = appState.getClusterStatus()
- // verify the AM restart container count was set
- String restarted = status.getInfo(
- StatusKeys.INFO_CONTAINERS_AM_RESTART)
- assert restarted != null;
- //and that the count == 1 master + the region servers
- assert Integer.parseInt(restarted) == containers.size()
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRolePlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRolePlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRolePlacement.groovy
deleted file mode 100644
index fba1ea0..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRolePlacement.groovy
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.Container
-import org.apache.hadoop.yarn.client.api.AMRMClient
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockRoles
-import org.apache.slider.server.appmaster.state.*
-import org.junit.Test
-
-import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole
-
-/**
- * Test that the app state lets you ask for nodes, get a specific host,
- * release it and then get that one back again.
- */
-@CompileStatic
-@Slf4j
-class TestAppStateRolePlacement extends BaseMockAppStateTest
- implements MockRoles {
-
- @Override
- String getTestName() {
- return "TestAppStateRolePlacement"
- }
-
-
- @Test
- public void testAllocateReleaseRealloc() throws Throwable {
- role0Status.desired = 1
-
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
- AMRMClient.ContainerRequest request = operation.request
- Container allocated = engine.allocateContainer(request)
- List<ContainerAssignment> assignments = [];
- List<AbstractRMOperation> operations = []
- appState.onContainersAllocated([(Container)allocated], assignments, operations)
- assert operations.size() == 0
- assert assignments.size() == 1
- ContainerAssignment assigned = assignments[0]
- Container container = assigned.container
- assert container.id == allocated.id
- int roleId = assigned.role.priority
- assert roleId == extractRole(request.priority)
- assert assigned.role.name == ROLE0
- String containerHostname = RoleHistoryUtils.hostnameOf(container);
- RoleInstance ri = roleInstance(assigned)
- //tell the app it arrived
- appState.containerStartSubmitted(container, ri);
- assert appState.onNodeManagerContainerStarted(container.id)
- assert role0Status.started == 1
- ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 0
-
- //now it is surplus
- role0Status.desired = 0
- ops = appState.reviewRequestAndReleaseNodes()
- ContainerReleaseOperation release = (ContainerReleaseOperation) ops[0]
-
- assert release.containerId == container.id
- engine.execute(ops)
- assert appState.onCompletedNode(containerStatus(container)).roleInstance
-
- //view the world
- appState.getRoleHistory().dump();
-
- //now ask for a new one
- role0Status.desired = 1
- ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 1
- operation = (ContainerRequestOperation) ops[0]
- AMRMClient.ContainerRequest request2 = operation.request
- assert request2 != null
- assert request2.nodes[0] == containerHostname
- engine.execute(ops)
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRoleRelease.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRoleRelease.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRoleRelease.groovy
deleted file mode 100644
index f087a30..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRoleRelease.groovy
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.ContainerId
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockRoles
-import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
-import org.apache.slider.server.appmaster.state.AbstractRMOperation
-import org.apache.slider.server.appmaster.state.RoleInstance
-import org.junit.Test
-
-/**
- * Test that if you have >1 role, the right roles are chosen for release.
- */
-@CompileStatic
-@Slf4j
-class TestAppStateRoleRelease extends BaseMockAppStateTest
- implements MockRoles {
-
- @Override
- String getTestName() {
- return "TestAppStateRolePlacement"
- }
-
- /**
- * Small cluster with multiple containers per node,
- * to guarantee many container allocations on each node
- * @return
- */
- @Override
- MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(4, 4)
- }
-
- @Test
- public void testAllocateReleaseRealloc() throws Throwable {
- /**
- * Allocate to all nodes
- */
- role0Status.desired = 6
- role1Status.desired = 5
- role2Status.desired = 4
- List<RoleInstance> instances = createAndStartNodes()
- assert instances.size() == 15
-
- //now it is surplus
- role0Status.desired = 0
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
-
- List<ContainerId> released = []
- engine.execute(ops, released)
- List<ContainerId> ids = extractContainerIds(instances, 0)
- released.each { ContainerId cid ->
- assert appState.onCompletedNode(containerStatus(cid)).roleInstance
- assert ids.contains(cid)
- }
-
- //view the world
- appState.getRoleHistory().dump();
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestContainerResourceAllocations.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestContainerResourceAllocations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestContainerResourceAllocations.groovy
deleted file mode 100644
index a0b1100..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestContainerResourceAllocations.groovy
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.Resource
-import org.apache.slider.api.ResourceKeys
-import org.apache.slider.core.conf.ConfTree
-import org.apache.slider.core.conf.ConfTreeOperations
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockRoles
-import org.apache.slider.server.appmaster.state.AbstractRMOperation
-import org.apache.slider.server.appmaster.state.ContainerRequestOperation
-import org.junit.Test
-
-/**
- * Test the container resource allocation logic
- */
-@CompileStatic
-@Slf4j
-class TestContainerResourceAllocations extends BaseMockAppStateTest {
-
- @Override
- String getTestName() {
- "TestContainerResourceAllocations"
- }
-
- @Test
- public void testNormalAllocations() throws Throwable {
- ConfTree clusterSpec = factory.newConfTree(1, 0, 0)
- ConfTreeOperations cto = new ConfTreeOperations(clusterSpec)
-
- cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_MEMORY, 512)
- cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_CORES, 2)
- appState.updateResourceDefinitions(clusterSpec)
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 1
- ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
- Resource requirements = operation.request.capability
- assert requirements.memory == 512
- assert requirements.virtualCores == 2
- }
-
- @Test
- public void testMaxMemAllocations() throws Throwable {
- ConfTree clusterSpec = factory.newConfTree(1, 0, 0)
- ConfTreeOperations cto = new ConfTreeOperations(clusterSpec)
-
- cto.setComponentOpt(MockRoles.ROLE0, ResourceKeys.YARN_MEMORY,
- ResourceKeys.YARN_RESOURCE_MAX)
- cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_CORES, 2)
- appState.updateResourceDefinitions(clusterSpec)
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 1
- ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
- Resource requirements = operation.request.capability
- assert requirements.memory == RM_MAX_RAM
- assert requirements.virtualCores == 2
- }
-
- @Test
- public void testMaxCoreAllocations() throws Throwable {
- ConfTree clusterSpec = factory.newConfTree(1, 0, 0)
- ConfTreeOperations cto = new ConfTreeOperations(clusterSpec)
- cto.setRoleOpt(MockRoles.ROLE0, ResourceKeys.YARN_MEMORY,
- 512)
- cto.setComponentOpt(MockRoles.ROLE0, ResourceKeys.YARN_CORES,
- ResourceKeys.YARN_RESOURCE_MAX)
- appState.updateResourceDefinitions(clusterSpec)
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 1
- ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
- Resource requirements = operation.request.capability
- assert requirements.memory == 512
- assert requirements.virtualCores == RM_MAX_CORES
- }
-
- @Test
- public void testMaxDefaultAllocations() throws Throwable {
-
- ConfTree clusterSpec = factory.newConfTree(1, 0, 0)
- appState.updateResourceDefinitions(clusterSpec)
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 1
- ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
- Resource requirements = operation.request.capability
- assert requirements.memory == ResourceKeys.DEF_YARN_MEMORY
- assert requirements.virtualCores == ResourceKeys.DEF_YARN_CORES
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy
deleted file mode 100644
index 911880a..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
-import org.apache.slider.api.ResourceKeys
-import org.apache.slider.core.conf.ConfTreeOperations
-import org.apache.slider.core.exceptions.BadConfigException
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
-import org.apache.slider.server.appmaster.model.mock.MockRoles
-import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
-import org.apache.slider.server.appmaster.state.AppState
-import org.apache.slider.server.avro.RoleHistoryWriter
-import org.junit.Test
-
-/**
- * Test that if you have >1 role, the right roles are chosen for release.
- */
-@CompileStatic
-@Slf4j
-class TestFlexDynamicRoles extends BaseMockAppStateTest
- implements MockRoles {
-
- @Override
- String getTestName() {
- return "TestAppStateDynamicRoles"
- }
-
- /**
- * Small cluster with multiple containers per node,
- * to guarantee many container allocations on each node
- * @return
- */
- @Override
- MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(4, 4)
- }
-
- @Override
- void initApp() {
- super.initApp()
- appState = new AppState(new MockRecordFactory())
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
-
- def instance = factory.newInstanceDefinition(0, 0, 0)
-
- def opts = [
- (ResourceKeys.COMPONENT_INSTANCES): "1",
- (ResourceKeys.COMPONENT_PRIORITY): "6",
- ]
-
- instance.resourceOperations.components["dynamic"] = opts
-
-
- appState.buildInstance(instance,
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- null, null)
- }
-
-
- private ConfTreeOperations init() {
- createAndStartNodes();
- def resources = appState.instanceDefinition.resources;
- return new ConfTreeOperations(resources)
- }
-
- @Test
- public void testDynamicFlexAddRole() throws Throwable {
- def cd = init()
- def opts = [
- (ResourceKeys.COMPONENT_INSTANCES): "1",
- (ResourceKeys.COMPONENT_PRIORITY): "7",
- ]
-
- cd.components["role4"] = opts
- appState.updateResourceDefinitions(cd.confTree);
- createAndStartNodes();
- dumpClusterDescription("updated CD", appState.getClusterStatus())
- appState.lookupRoleStatus(7)
- appState.lookupRoleStatus(6)
- //gaps are still there
- try {
- assert null == appState.lookupRoleStatus(5)
- } catch (RuntimeException expected) {
- }
- }
-
- @Test
- public void testDynamicFlexAddRoleConflictingPriority() throws Throwable {
- def cd = init()
- def opts = [
- (ResourceKeys.COMPONENT_INSTANCES): "1",
- (ResourceKeys.COMPONENT_PRIORITY): "6",
- ]
-
- cd.components["role4"] = opts
- try {
- appState.updateResourceDefinitions(cd.confTree);
- dumpClusterDescription("updated CD", appState.getClusterStatus())
- fail("Expected an exception")
- } catch (BadConfigException expected) {
- }
- }
-
- @Test
- public void testDynamicFlexDropRole() throws Throwable {
- def cd = init()
- cd.components.remove("dynamic");
- appState.updateResourceDefinitions(cd.confTree);
-
- def getCD = appState.getClusterStatus()
- dumpClusterDescription("updated CD", getCD)
- //status is retained for future
- appState.lookupRoleStatus(6)
- }
-
-
- @Test
- public void testHistorySaveFlexLoad() throws Throwable {
- def cd = init()
- def roleHistory = appState.roleHistory
- Path history = roleHistory.saveHistory(0x0001)
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- def opts = [
- (ResourceKeys.COMPONENT_INSTANCES): "1",
- (ResourceKeys.COMPONENT_PRIORITY): "7",
- ]
-
- cd.components["role4"] = opts
- appState.updateResourceDefinitions(cd.confTree);
- createAndStartNodes();
- historyWriter.read(fs, history, appState.roleHistory)
- }
-
- @Test
- public void testHistoryFlexSaveLoad() throws Throwable {
- def cd = init()
- def opts = [
- (ResourceKeys.COMPONENT_INSTANCES): "1",
- (ResourceKeys.COMPONENT_PRIORITY): "7",
- ]
-
- cd.components["role4"] = opts
- appState.updateResourceDefinitions(cd.confTree);
- createAndStartNodes();
- RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- def roleHistory = appState.roleHistory
- Path history = roleHistory.saveHistory(0x0002)
- //now reset the app state
- def historyWorkDir2 = new File("target/history" + testName + "-0002")
- def historyPath2 = new Path(historyWorkDir2.toURI())
- appState = new AppState(new MockRecordFactory())
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
- appState.buildInstance(
- factory.newInstanceDefinition(0, 0, 0),
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath2,
- null, null)
- historyWriter.read(fs, history, appState.roleHistory)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
new file mode 100644
index 0000000..3a287bd
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.slider.core.exceptions.SliderException
+import org.apache.slider.core.exceptions.TriggerClusterTeardownException
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.state.*
+import org.junit.Test
+
+/**
+ * Test that if you have >1 role, the right roles are chosen for release.
+ */
+@CompileStatic
+@Slf4j
+class TestMockAppStateContainerFailure extends BaseMockAppStateTest
+ implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestAppStateContainerFailure"
+ }
+
+ /**
+ * Small cluster with multiple containers per node,
+ * to guarantee many container allocations on each node
+ * @return
+ */
+ @Override
+ MockYarnEngine createYarnEngine() {
+ return new MockYarnEngine(8000, 4)
+ }
+
+ @Test
+ public void testShortLivedFail() throws Throwable {
+
+ role0Status.desired = 1
+ List<RoleInstance> instances = createAndStartNodes()
+ assert instances.size() == 1
+
+ RoleInstance instance = instances[0]
+ long created = instance.createTime
+ long started = instance.startTime
+ assert created > 0
+ assert started >= created
+ List<ContainerId> ids = extractContainerIds(instances, 0)
+
+ ContainerId cid = ids[0]
+ assert appState.isShortLived(instance)
+ AppState.NodeCompletionResult result = appState.onCompletedNode(containerStatus(cid, 1))
+ assert result.roleInstance != null
+ assert result.containerFailed
+ RoleStatus status = role0Status
+ assert status.failed == 1
+ assert status.startFailed == 1
+
+ //view the world
+ appState.getRoleHistory().dump();
+ List<NodeInstance> queue = appState.roleHistory.cloneAvailableList(0)
+ assert queue.size() == 0
+
+ }
+
+ @Test
+ public void testLongLivedFail() throws Throwable {
+
+ role0Status.desired = 1
+ List<RoleInstance> instances = createAndStartNodes()
+ assert instances.size() == 1
+
+ RoleInstance instance = instances[0]
+ instance.startTime = System.currentTimeMillis() - 60 * 60 * 1000;
+ assert !appState.isShortLived(instance)
+ List<ContainerId> ids = extractContainerIds(instances, 0)
+
+ ContainerId cid = ids[0]
+ AppState.NodeCompletionResult result = appState.onCompletedNode(
+ containerStatus(cid, 1))
+ assert result.roleInstance != null
+ assert result.containerFailed
+ RoleStatus status = role0Status
+ assert status.failed == 1
+ assert status.startFailed == 0
+
+ //view the world
+ appState.getRoleHistory().dump();
+ List<NodeInstance> queue = appState.roleHistory.cloneAvailableList(0)
+ assert queue.size() == 1
+
+ }
+
+ @Test
+ public void testNodeStartFailure() throws Throwable {
+
+ role0Status.desired = 1
+ List<RoleInstance> instances = createAndSubmitNodes()
+ assert instances.size() == 1
+
+ RoleInstance instance = instances[0]
+
+ List<ContainerId> ids = extractContainerIds(instances, 0)
+
+ ContainerId cid = ids[0]
+ appState.onNodeManagerContainerStartFailed(cid, new SliderException("oops"))
+ RoleStatus status = role0Status
+ assert status.failed == 1
+ assert status.startFailed == 1
+
+
+ RoleHistory history = appState.roleHistory
+ history.dump();
+ List<NodeInstance> queue = history.cloneAvailableList(0)
+ assert queue.size() == 0
+
+ NodeInstance ni = history.getOrCreateNodeInstance(instance.container)
+ NodeEntry re = ni.get(0)
+ assert re.failed == 1
+ assert re.startFailed == 1
+ }
+
+ @Test
+ public void testRecurrentStartupFailure() throws Throwable {
+
+ role0Status.desired = 1
+ try {
+ for (int i = 0; i< 100; i++) {
+ List<RoleInstance> instances = createAndSubmitNodes()
+ assert instances.size() == 1
+
+ List<ContainerId> ids = extractContainerIds(instances, 0)
+
+ ContainerId cid = ids[0]
+ log.info("$i instance $instances[0] $cid")
+ assert cid
+ appState.onNodeManagerContainerStartFailed(cid, new SliderException("oops"))
+ AppState.NodeCompletionResult result = appState.onCompletedNode(containerStatus(cid))
+ assert result.containerFailed
+ }
+ fail("Cluster did not fail from too many startup failures")
+ } catch (TriggerClusterTeardownException teardown) {
+ log.info("Exception $teardown.exitCode : $teardown")
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
new file mode 100644
index 0000000..a204bea
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.state.AbstractRMOperation
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
+import org.junit.Test
+
+/**
+ * Test that if you have >1 role, the right roles are chosen for release.
+ */
+@CompileStatic
+@Slf4j
+class TestMockAppStateDynamicRoles extends BaseMockAppStateTest
+ implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestAppStateDynamicRoles"
+ }
+
+ /**
+ * Small cluster with multiple containers per node,
+ * to guarantee many container allocations on each node
+ * @return
+ */
+ @Override
+ MockYarnEngine createYarnEngine() {
+ return new MockYarnEngine(4, 4)
+ }
+
+ @Override
+ void initApp() {
+ super.initApp()
+ appState = new AppState(new MockRecordFactory())
+ appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
+
+ def instance = factory.newInstanceDefinition(0,0,0)
+
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PRIORITY): "4",
+ ]
+
+ instance.resourceOperations.components["dynamic"]= opts
+
+
+ appState.buildInstance(
+ instance,
+ new Configuration(),
+ new Configuration(false),
+ factory.ROLES,
+ fs,
+ historyPath,
+ null,
+ null, new SimpleReleaseSelector())
+ }
+
+ @Test
+ public void testAllocateReleaseRealloc() throws Throwable {
+
+ List<RoleInstance> instances = createAndStartNodes()
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ appState.getRoleHistory().dump();
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
new file mode 100644
index 0000000..aa330f8
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.Path
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.core.conf.ConfTreeOperations
+import org.apache.slider.core.exceptions.BadConfigException
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
+import org.apache.slider.server.avro.RoleHistoryWriter
+import org.junit.Test
+
+/**
+ * Test that if you have >1 role, the right roles are chosen for release.
+ */
+@CompileStatic
+@Slf4j
+class TestMockAppStateFlexDynamicRoles extends BaseMockAppStateTest
+ implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestAppStateDynamicRoles"
+ }
+
+ /**
+ * Small cluster with multiple containers per node,
+ * to guarantee many container allocations on each node
+ * @return
+ */
+ @Override
+ MockYarnEngine createYarnEngine() {
+ return new MockYarnEngine(4, 4)
+ }
+
+ @Override
+ void initApp() {
+ super.initApp()
+ appState = new AppState(new MockRecordFactory())
+ appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
+
+ def instance = factory.newInstanceDefinition(0, 0, 0)
+
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PRIORITY): "6",
+ ]
+
+ instance.resourceOperations.components["dynamic"] = opts
+
+
+ appState.buildInstance(instance,
+ new Configuration(),
+ new Configuration(false),
+ factory.ROLES,
+ fs,
+ historyPath,
+ null, null, new SimpleReleaseSelector())
+ }
+
+
+ private ConfTreeOperations init() {
+ createAndStartNodes();
+ def resources = appState.instanceDefinition.resources;
+ return new ConfTreeOperations(resources)
+ }
+
+ @Test
+ public void testDynamicFlexAddRole() throws Throwable {
+ def cd = init()
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PRIORITY): "7",
+ ]
+
+ cd.components["role4"] = opts
+ appState.updateResourceDefinitions(cd.confTree);
+ createAndStartNodes();
+ dumpClusterDescription("updated CD", appState.getClusterStatus())
+ appState.lookupRoleStatus(7)
+ appState.lookupRoleStatus(6)
+ //gaps are still there
+ try {
+ assert null == appState.lookupRoleStatus(5)
+ } catch (RuntimeException expected) {
+ }
+ }
+
+ @Test
+ public void testDynamicFlexAddRoleConflictingPriority() throws Throwable {
+ def cd = init()
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PRIORITY): "6",
+ ]
+
+ cd.components["role4"] = opts
+ try {
+ appState.updateResourceDefinitions(cd.confTree);
+ dumpClusterDescription("updated CD", appState.getClusterStatus())
+ fail("Expected an exception")
+ } catch (BadConfigException expected) {
+ }
+ }
+
+ @Test
+ public void testDynamicFlexDropRole() throws Throwable {
+ def cd = init()
+ cd.components.remove("dynamic");
+ appState.updateResourceDefinitions(cd.confTree);
+
+ def getCD = appState.getClusterStatus()
+ dumpClusterDescription("updated CD", getCD)
+ //status is retained for future
+ appState.lookupRoleStatus(6)
+ }
+
+
+ @Test
+ public void testHistorySaveFlexLoad() throws Throwable {
+ def cd = init()
+ def roleHistory = appState.roleHistory
+ Path history = roleHistory.saveHistory(0x0001)
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PRIORITY): "7",
+ ]
+
+ cd.components["role4"] = opts
+ appState.updateResourceDefinitions(cd.confTree);
+ createAndStartNodes();
+ historyWriter.read(fs, history, appState.roleHistory)
+ }
+
+ @Test
+ public void testHistoryFlexSaveLoad() throws Throwable {
+ def cd = init()
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PRIORITY): "7",
+ ]
+
+ cd.components["role4"] = opts
+ appState.updateResourceDefinitions(cd.confTree);
+ createAndStartNodes();
+ RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ def roleHistory = appState.roleHistory
+ Path history = roleHistory.saveHistory(0x0002)
+ //now reset the app state
+ def historyWorkDir2 = new File("target/history" + testName + "-0002")
+ def historyPath2 = new Path(historyWorkDir2.toURI())
+ appState = new AppState(new MockRecordFactory())
+ appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
+ appState.buildInstance(
+ factory.newInstanceDefinition(0, 0, 0),
+ new Configuration(),
+ new Configuration(false),
+ factory.ROLES,
+ fs,
+ historyPath2,
+ null, null, new SimpleReleaseSelector())
+ historyWriter.read(fs, history, appState.roleHistory)
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5ce953ca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
new file mode 100644
index 0000000..6be158e
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.state.AbstractRMOperation
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.ContainerAssignment
+import org.apache.slider.server.appmaster.state.ContainerReleaseOperation
+import org.apache.slider.server.appmaster.state.ContainerRequestOperation
+import org.apache.slider.server.appmaster.state.RMOperationHandler
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.junit.Test
+
+import static org.apache.slider.server.appmaster.state.ContainerPriority.buildPriority
+import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole
+
+@Slf4j
+class TestMockAppStateFlexing extends BaseMockAppStateTest implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestMockFlexing"
+ }
+
+ @Test
+ public void testFlexDuringLaunchPhase() throws Throwable {
+ role0Status.desired = 1
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ List<Container> allocations = engine.execute(ops)
+ List<ContainerAssignment> assignments = [];
+ List<AbstractRMOperation> releases = []
+ appState.onContainersAllocated(allocations, assignments, releases)
+ assert assignments.size() == 1
+ ContainerAssignment assigned = assignments[0]
+ Container target = assigned.container
+ RoleInstance ri = roleInstance(assigned)
+
+ ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.empty
+
+ //now this is the start point.
+ appState.containerStartSubmitted(target, ri);
+
+ ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.empty
+
+ RoleInstance ri2 = appState.innerOnNodeManagerContainerStarted(target.id)
+ }
+
+ @Test
+ public void testFlexBeforeAllocationPhase() throws Throwable {
+ role0Status.desired = 1
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ assert !ops.empty
+ List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes()
+ assert ops2.empty
+ }
+
+
+ @Test
+ public void testFlexDownTwice() throws Throwable {
+ int r0 = 6
+ int r1 = 0
+ int r2 = 0
+ role0Status.desired = r0
+ role1Status.desired = r1
+ role2Status.desired = r2
+ List<RoleInstance> instances = createAndStartNodes()
+
+ int clusterSize = r0 + r1 + r2
+ assert instances.size() == clusterSize
+ log.info("shrinking cluster")
+ r0 = 4
+ role0Status.desired = r0
+ List<AppState.NodeCompletionResult> completionResults = []
+ instances = createStartAndStopNodes(completionResults)
+ assert instances.size() == 0
+ // assert two nodes were released
+ assert completionResults.size() == 2
+
+ // no-op review
+ completionResults = []
+ instances = createStartAndStopNodes(completionResults)
+ assert instances.size() == 0
+ // assert two nodes were released
+ assert completionResults.size() == 0
+
+
+ // now shrink again
+ role0Status.desired = r0 = 1
+ completionResults = []
+ instances = createStartAndStopNodes(completionResults)
+ assert instances.size() == 0
+ // assert two nodes were released
+ assert completionResults.size() == 3
+
+ }
+
+
+}
[04/50] [abbrv] git commit: SLIDER-270 test to replicate the failure
condition in the slider-hbase-provider
Posted by st...@apache.org.
SLIDER-270 test to replicate the failure condition in the slider-hbase-provider
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/34c00e86
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/34c00e86
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/34c00e86
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 34c00e86ddbf1c4ec8c800ff68be15a10752c14a
Parents: 28babb1
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 5 18:10:59 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 5 18:10:59 2014 +0100
----------------------------------------------------------------------
.../minicluster/HBaseMiniClusterTestBase.groovy | 149 ++++++++++++-------
.../flexing/TestClusterFlex2To5.groovy | 2 +-
.../flexing/TestClusterFlexDownMultiple.groovy | 58 ++++++++
3 files changed, 151 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34c00e86/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
index 3f49771..c9f0a4f 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.client.HConnection
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.ClusterNode
import org.apache.slider.api.ResourceKeys
-import org.apache.slider.providers.hbase.HBaseKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.hbase.HBaseTestUtils
@@ -214,9 +213,7 @@ public abstract class HBaseMiniClusterTestBase extends YarnZKMiniClusterTestBase
(ROLE_MASTER): masters,
(ROLE_WORKER): workers,
];
- extraArgs << ARG_RES_COMP_OPT << ROLE_MASTER << ResourceKeys.YARN_MEMORY << YRAM
- extraArgs << ARG_RES_COMP_OPT << ROLE_WORKER << ResourceKeys.YARN_MEMORY << YRAM
- extraArgs << ARG_PROVIDER << PROVIDER_HBASE;
+ hbaseArgs(extraArgs)
return createCluster(clustername,
roles,
@@ -226,6 +223,15 @@ public abstract class HBaseMiniClusterTestBase extends YarnZKMiniClusterTestBase
[:])
}
+ public List<String> hbaseArgs(List<String> extraArgs) {
+ extraArgs << ARG_RES_COMP_OPT << ROLE_MASTER << ResourceKeys.YARN_MEMORY <<
+ YRAM
+ extraArgs << ARG_RES_COMP_OPT << ROLE_WORKER << ResourceKeys.YARN_MEMORY <<
+ YRAM
+ extraArgs << ARG_PROVIDER << PROVIDER_HBASE;
+ return extraArgs;
+ }
+
/**
* Create an AM without a master
* @param clustername AM name
@@ -241,9 +247,7 @@ public abstract class HBaseMiniClusterTestBase extends YarnZKMiniClusterTestBase
];
return createCluster(clustername,
roles,
- [
- ARG_PROVIDER, PROVIDER_HBASE
- ],
+ hbaseArgs([]),
deleteExistingData,
blockUntilRunning,
[:])
@@ -278,62 +282,93 @@ public abstract class HBaseMiniClusterTestBase extends YarnZKMiniClusterTestBase
int masters,
int masterFlexTarget,
int workers,
- int flexTarget,
+ int workerFlexTarget,
boolean testHBaseAfter) {
- createMiniCluster(clustername, configuration,
- 1,
- true);
+ SliderClient sliderClient = startHBaseCluster(clustername, masters, workers)
+
+ //now flex
+ return flexCluster(
+ sliderClient,
+ clustername,
+ masterFlexTarget,
+ workerFlexTarget,
+ testHBaseAfter)
+
+ }
+
+ public SliderClient startHBaseCluster(
+ String clustername,
+ int masters,
+ int workers) {
+ clustername = createMiniCluster(clustername, configuration,
+ 1,
+ true);
//now launch the cluster
SliderClient sliderClient;
ServiceLauncher<SliderClient> launcher = createCluster(clustername,
- [
- (ROLE_MASTER): masters,
- (ROLE_WORKER): workers,
- ],
- [
- ARG_RES_COMP_OPT , ROLE_MASTER, ResourceKeys.YARN_MEMORY, YRAM,
- ARG_RES_COMP_OPT , ROLE_WORKER, ResourceKeys.YARN_MEMORY, YRAM,
- ARG_PROVIDER , PROVIDER_HBASE
- ],
- true,
- true,
- [:]);
+ [
+ (ROLE_MASTER): masters,
+ (ROLE_WORKER): workers,
+ ],
+ hbaseArgs([]),
+ true,
+ true,
+ [:]);
sliderClient = launcher.service;
- try {
- basicHBaseClusterStartupSequence(sliderClient);
-
- describe("Waiting for initial worker count of $workers");
-
- //verify the #of roles is as expected
- //get the hbase status
- waitForWorkerInstanceCount(sliderClient, workers, hbaseClusterStartupToLiveTime);
- waitForSliderMasterCount(sliderClient, masters, hbaseClusterStartupToLiveTime);
-
- log.info("Slider worker count at $workers, waiting for region servers to match");
- waitForHBaseRegionServerCount(sliderClient, clustername, workers, hbaseClusterStartupToLiveTime);
-
- //now flex
- describe("Flexing masters:$masters -> $masterFlexTarget ; workers $workers -> $flexTarget");
- boolean flexed;
- flexed = 0 == sliderClient.flex(clustername,
- [
- (ROLE_WORKER): flexTarget,
- (ROLE_MASTER): masterFlexTarget
- ]
- );
- waitForWorkerInstanceCount(sliderClient, flexTarget, hbaseClusterStartupToLiveTime);
- waitForSliderMasterCount(sliderClient, masterFlexTarget,
- hbaseClusterStartupToLiveTime);
-
- if (testHBaseAfter) {
- waitForHBaseRegionServerCount(sliderClient, clustername, flexTarget,
- hbaseClusterStartupToLiveTime);
- }
- return flexed;
- } finally {
- maybeStopCluster(sliderClient, null, "end of flex test run");
- }
+ basicHBaseClusterStartupSequence(sliderClient);
+
+ describe("Waiting for initial worker count of $workers");
+
+ //verify the #of roles is as expected
+ //get the hbase status
+ waitForWorkerInstanceCount(
+ sliderClient,
+ workers,
+ hbaseClusterStartupToLiveTime);
+ waitForSliderMasterCount(
+ sliderClient,
+ masters,
+ hbaseClusterStartupToLiveTime);
+
+ log.info(
+ "Slider worker count at $workers, waiting for region servers to match");
+ waitForHBaseRegionServerCount(
+ sliderClient,
+ clustername,
+ workers,
+ hbaseClusterStartupToLiveTime);
+ sliderClient
+ }
+
+ public boolean flexCluster(
+ SliderClient sliderClient,
+ String clustername,
+ int masterFlexTarget,
+ int workerFlexTarget,
+ boolean testHBaseAfter) {
+ int flexTarget
+ describe(
+ "Flexing masters -> $masterFlexTarget ; workers -> ${workerFlexTarget}");
+ boolean flexed;
+ flexed = 0 == sliderClient.flex(clustername,
+ [
+ (ROLE_WORKER): workerFlexTarget,
+ (ROLE_MASTER): masterFlexTarget
+ ]
+ );
+ waitForWorkerInstanceCount(
+ sliderClient,
+ workerFlexTarget,
+ hbaseClusterStartupToLiveTime);
+ waitForSliderMasterCount(sliderClient, masterFlexTarget,
+ hbaseClusterStartupToLiveTime);
+
+ if (testHBaseAfter) {
+ waitForHBaseRegionServerCount(sliderClient, clustername, workerFlexTarget,
+ hbaseClusterStartupToLiveTime);
+ }
+ flexed
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34c00e86/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy
index 680286c..683c02a 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy
@@ -31,7 +31,7 @@ import org.junit.Test
class TestClusterFlex2To5 extends HBaseMiniClusterTestBase {
@Test
- public void testClusterFlex1To5() throws Throwable {
+ public void testClusterFlex2To5() throws Throwable {
assert flexHBaseClusterTestRun(
"",
1,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34c00e86/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
new file mode 100644
index 0000000..43ade73
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.flexing
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.junit.Test
+
+/**
+ * Create a master against the File:// fs
+ */
+@CompileStatic
+@Slf4j
+class TestClusterFlexDownMultiple extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testClusterFlexDownMultiple() throws Throwable {
+ def clusterName = createClusterName();
+ SliderClient sliderClient = startHBaseCluster(clusterName, 1, 4)
+
+ assert flexCluster(
+ sliderClient,
+ clusterName,
+ 1,
+ 3,
+ true)
+
+ assert flexCluster(
+ sliderClient,
+ clusterName,
+ 1,
+ 1,
+ true)
+
+
+
+ }
+
+
+}
[34/50] [abbrv] git commit: Merge branch
'feature/SLIDER-286_AM_to_notify_providers_when_restarted' into develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-286_AM_to_notify_providers_when_restarted' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bd69a4cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bd69a4cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bd69a4cb
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: bd69a4cbb71591fc4304e452a5ce036e4cddc6d5
Parents: 373aed3 11335c1
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 7 17:58:43 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 7 17:58:43 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 4 +-
.../slider/core/launch/AppMasterLauncher.java | 7 +-
.../providers/AbstractProviderService.java | 6 +-
.../slider/providers/ProviderService.java | 14 +-
.../server/appmaster/SliderAppMaster.java | 13 +-
.../agent/AgentMiniClusterTestBase.groovy | 7 +-
.../agent/actions/TestActionExists.groovy | 7 +-
.../slider/agent/actions/TestActionList.groovy | 5 +-
.../agent/actions/TestActionStatus.groovy | 7 +-
.../agent/freezethaw/TestFreezeCommands.groovy | 6 +-
.../TestFreezeThawMasterlessAM.groovy | 5 +-
.../agent/standalone/TestBadAMHeap.groovy | 2 +-
.../agent/standalone/TestBadYarnQueue.groovy | 2 +-
.../standalone/TestDestroyMasterlessAM.groovy | 153 ------------------
.../standalone/TestDestroyStandaloneAM.groovy | 156 +++++++++++++++++++
.../standalone/TestKillMasterlessAM.groovy | 73 ---------
.../standalone/TestKillStandaloneAM.groovy | 73 +++++++++
.../standalone/TestRestartStandaloneAM.groovy | 96 ++++++++++++
.../standalone/TestStandaloneAgentAM.groovy | 6 +-
.../TestStandaloneBadClusterName.groovy | 2 +-
.../standalone/TestStandaloneRegistryAM.groovy | 2 +-
.../model/mock/MockProviderService.groovy | 3 +-
.../apache/slider/test/SliderTestUtils.groovy | 2 +-
.../slider/test/YarnMiniClusterTestBase.groovy | 7 +-
.../publisher/TestAgentProviderService.java | 10 +-
25 files changed, 396 insertions(+), 272 deletions(-)
----------------------------------------------------------------------
[10/50] [abbrv] git commit: SLIDER-271. When a container is releases
the AppMaster should not monitor heartbeats from that container SLIDER-293.
On Windows slider agent should be able to execute commands and not wait for
completion SLIDER-294. Clean up m
Posted by st...@apache.org.
SLIDER-271. When a container is releases the AppMaster should not monitor heartbeats from that container
SLIDER-293. On Windows slider agent should be able to execute commands and not wait for completion
SLIDER-294. Clean up memcached app packages for Windows and Linux
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/405aa51e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/405aa51e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/405aa51e
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 405aa51e203f611a385a0e923fe68eb0d7d86fc0
Parents: 640233d
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Aug 5 18:31:16 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Aug 5 18:51:58 2014 -0700
----------------------------------------------------------------------
app-packages/memcached-win/appConfig.json | 6 +-
app-packages/memcached-win/metainfo.xml | 7 +-
.../memcached-win/package/scripts/memcached.py | 2 +-
.../memcached-win/package/scripts/params.py | 1 +
app-packages/memcached/appConfig.json | 6 +-
app-packages/memcached/metainfo.xml | 1 -
.../core/providers/windows/system.py | 54 ++++++++++++++-
.../providers/agent/AgentProviderService.java | 71 +++++++++++---------
.../agent/TestAgentProviderService.java | 17 ++++-
9 files changed, 121 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/405aa51e/app-packages/memcached-win/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/appConfig.json b/app-packages/memcached-win/appConfig.json
index 8c00660..63835ec 100644
--- a/app-packages/memcached-win/appConfig.json
+++ b/app-packages/memcached-win/appConfig.json
@@ -5,12 +5,16 @@
"global": {
"application.def": "/slider/jmemcached-1.0.0.zip",
"java_home": "C:\\java",
+
"site.global.app_user": "hadoop",
"site.global.app_root": "${AGENT_WORK_ROOT}\\app\\install",
+
"site.global.additional_cp": "C:\\hdp\\hadoop-2.4.0.2.1.3.0-1990\\share\\hadoop\\common\\lib\\*",
"site.global.xmx_val": "256m",
"site.global.xms_val": "128m",
- "site.global.memory_val": "200M"
+ "site.global.memory_val": "200M",
+ "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
+
},
"components": {
"slider-appmaster": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/405aa51e/app-packages/memcached-win/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/metainfo.xml b/app-packages/memcached-win/metainfo.xml
index dc2d67b..d056c0a 100644
--- a/app-packages/memcached-win/metainfo.xml
+++ b/app-packages/memcached-win/metainfo.xml
@@ -28,10 +28,15 @@
<component>
<name>MEMCACHED</name>
<category>MASTER</category>
+ <exports>
+ <export>
+ <name>host_port</name>
+ <value>${THIS_HOST}:${site.global.listen_port}</value>
+ </export>
+ </exports>
<commandScript>
<script>scripts/memcached.py</script>
<scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
</commandScript>
</component>
</components>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/405aa51e/app-packages/memcached-win/package/scripts/memcached.py
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/package/scripts/memcached.py b/app-packages/memcached-win/package/scripts/memcached.py
index 222a62c..1bb05dc 100644
--- a/app-packages/memcached-win/package/scripts/memcached.py
+++ b/app-packages/memcached-win/package/scripts/memcached.py
@@ -34,7 +34,7 @@ class Memcached(Script):
import params
env.set_params(params)
self.configure(env)
- process_cmd = format("{java64_home}\\bin\\java -Xmx{xmx_val} -Xms{xms_val} -classpath {app_root}\\*;{additional_cp} com.thimbleware.jmemcached.Main --memory={memory_val}")
+ process_cmd = format("{java64_home}\\bin\\java -Xmx{xmx_val} -Xms{xms_val} -classpath {app_root}\\*;{additional_cp} com.thimbleware.jmemcached.Main --memory={memory_val} --port={port}")
Execute(process_cmd,
user=params.app_user,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/405aa51e/app-packages/memcached-win/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/package/scripts/params.py b/app-packages/memcached-win/package/scripts/params.py
index a2fba8b..f8e64da 100644
--- a/app-packages/memcached-win/package/scripts/params.py
+++ b/app-packages/memcached-win/package/scripts/params.py
@@ -30,3 +30,4 @@ additional_cp = config['configurations']['global']['additional_cp']
xmx_val = config['configurations']['global']['xmx_val']
xms_val = config['configurations']['global']['xms_val']
memory_val = config['configurations']['global']['memory_val']
+port = config['configurations']['global']['listen_port']
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/405aa51e/app-packages/memcached/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached/appConfig.json b/app-packages/memcached/appConfig.json
index 78cb486..b179723 100644
--- a/app-packages/memcached/appConfig.json
+++ b/app-packages/memcached/appConfig.json
@@ -3,15 +3,17 @@
"metadata": {
},
"global": {
- "application.def": "/slider/jmemcached-1.0.0.zip",
+ "application.def": "package/jmemcached-1.0.0.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
+
"site.global.app_user": "yarn",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/jmemcached-1.0.0",
+
"site.global.additional_cp": "/usr/lib/hadoop/lib/*",
"site.global.xmx_val": "256m",
"site.global.xms_val": "128m",
"site.global.memory_val": "200M",
- "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DEFAULT_11211}{DO_NOT_PROPAGATE}"
+ "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
},
"components": {
"slider-appmaster": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/405aa51e/app-packages/memcached/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/memcached/metainfo.xml b/app-packages/memcached/metainfo.xml
index 08a504e..525816e 100644
--- a/app-packages/memcached/metainfo.xml
+++ b/app-packages/memcached/metainfo.xml
@@ -37,7 +37,6 @@
<commandScript>
<script>scripts/memcached.py</script>
<scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
</commandScript>
</component>
</components>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/405aa51e/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
index 1e8142e..fa29d7d 100644
--- a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
+++ b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
@@ -23,15 +23,26 @@ from resource_management.core.providers import Provider
from resource_management.core.logger import Logger
from resource_management.core.base import Fail
from resource_management.core import ExecuteTimeoutException
+from multiprocessing import Queue
import time
import os
import subprocess
+import shutil
def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish=True, timeout=None):
# TODO implement timeout, logoutput, wait_for_finish
+ Logger.info("Executing %s" % (command))
proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
cwd=cwd, env=env, shell=False)
+ if not wait_for_finish:
+ return None, None
+
+ if timeout:
+ q = Queue()
+ t = threading.Timer( timeout, on_timeout, [proc, q] )
+ t.start()
+
out = proc.communicate()[0].strip()
code = proc.returncode
if logoutput and out:
@@ -83,7 +94,8 @@ class FileProvider(Provider):
if content:
fp.write(content)
- _set_file_acl(self.resource.path, self.resource.owner, self.resource.mode)
+ if self.resource.owner and self.resource.mode:
+ _set_file_acl(self.resource.path, self.resource.owner, self.resource.mode)
def action_delete(self):
path = self.resource.path
@@ -133,10 +145,46 @@ class ExecuteProvider(Provider):
time.sleep(self.resource.try_sleep)
except ExecuteTimeoutException:
err_msg = ("Execution of '%s' was killed due timeout after %d seconds") % (
- self.resource.command, self.resource.timeout)
+ self.resource.command, self.resource.timeout)
if self.resource.on_timeout:
Logger.info("Executing '%s'. Reason: %s" % (self.resource.on_timeout, err_msg))
_call_command(self.resource.on_timeout)
else:
- raise Fail(err_msg)
\ No newline at end of file
+ raise Fail(err_msg)
+
+
+class DirectoryProvider(Provider):
+ def action_create(self):
+ path = DirectoryProvider._trim_uri(self.resource.path)
+ if not os.path.exists(path):
+ Logger.info("Creating directory %s" % self.resource)
+ if self.resource.recursive:
+ os.makedirs(path)
+ else:
+ dirname = os.path.dirname(path)
+ if not os.path.isdir(dirname):
+ raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname))
+
+ os.mkdir(path)
+
+ if not os.path.isdir(path):
+ raise Fail("Applying %s failed, file %s already exists" % (self.resource, path))
+
+ if self.resource.owner and self.resource.mode:
+ _set_file_acl(path, self.resource.owner, self.resource.mode)
+
+ def action_delete(self):
+ path = self.resource.path
+ if os.path.exists(path):
+ if not os.path.isdir(path):
+ raise Fail("Applying %s failed, %s is not a directory" % (self.resource, path))
+
+ Logger.info("Removing directory %s and all its content" % self.resource)
+ shutil.rmtree(path)
+
+ @staticmethod
+ def _trim_uri(file_uri):
+ if file_uri.startswith("file:///"):
+ return file_uri[8:]
+ return file_uri
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/405aa51e/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 682efba..6cfadf3 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -116,24 +116,26 @@ public class AgentProviderService extends AbstractProviderService implements
private static final String DO_NOT_PROPAGATE_TAG = "{DO_NOT_PROPAGATE}";
private static final int MAX_LOG_ENTRIES = 20;
private static final int DEFAULT_HEARTBEAT_MONITOR_INTERVAL = 60 * 1000;
+
private final Object syncLock = new Object();
- private final Map<String, Map<String, String>> allocatedPorts = new ConcurrentHashMap<>();
private int heartbeatMonitorInterval = 0;
private AgentClientProvider clientProvider;
- private Map<String, ComponentInstanceState> componentStatuses = new ConcurrentHashMap<>();
private AtomicInteger taskId = new AtomicInteger(0);
private volatile Metainfo metainfo = null;
private ComponentCommandOrder commandOrder = null;
private HeartbeatMonitor monitor;
+ private Boolean canAnyMasterPublish = null;
+ private AgentLaunchParameter agentLaunchParameter = null;
+
+ private Map<String, ComponentInstanceState> componentStatuses = new ConcurrentHashMap<>();
+ private Map<String, Map<String, String>> componentInstanceData = new ConcurrentHashMap();
+ private final Map<String, Map<String, String>> allocatedPorts = new ConcurrentHashMap<>();
private Map<String, String> workFolders =
Collections.synchronizedMap(new LinkedHashMap<String, String>(MAX_LOG_ENTRIES, 0.75f, false) {
protected boolean removeEldestEntry(Map.Entry eldest) {
return size() > MAX_LOG_ENTRIES;
}
});
- private Map<String, Map<String, String>> componentInstanceData = new ConcurrentHashMap();
- private Boolean canAnyMasterPublish = null;
- private AgentLaunchParameter agentLaunchParameter = null;
/**
* Create an instance of AgentProviderService
@@ -286,11 +288,11 @@ public class AgentProviderService extends AbstractProviderService implements
launcher.addCommand(operation.build());
// initialize the component instance state
- componentStatuses.put(label,
- new ComponentInstanceState(
- role,
- container.getId().toString(),
- getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME)));
+ getComponentStatuses().put(label,
+ new ComponentInstanceState(
+ role,
+ container.getId().toString(),
+ getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME)));
}
/**
@@ -329,12 +331,13 @@ public class AgentProviderService extends AbstractProviderService implements
public RegistrationResponse handleRegistration(Register registration) {
RegistrationResponse response = new RegistrationResponse();
String label = registration.getHostname();
- if (componentStatuses.containsKey(label)) {
+ if (getComponentStatuses().containsKey(label)) {
response.setResponseStatus(RegistrationStatus.OK);
- componentStatuses.get(label).setLastHeartbeat(System.currentTimeMillis());
+ getComponentStatuses().get(label).setLastHeartbeat(System.currentTimeMillis());
} else {
response.setResponseStatus(RegistrationStatus.FAILED);
response.setLog("Label not recognized.");
+ log.warn("Received registration request from unknown label {}", label);
}
return response;
}
@@ -362,12 +365,12 @@ public class AgentProviderService extends AbstractProviderService implements
return response;
}
- if (!componentStatuses.containsKey(label)) {
+ if (!getComponentStatuses().containsKey(label)) {
return response;
}
Boolean isMaster = isMaster(roleName);
- ComponentInstanceState componentStatus = componentStatuses.get(label);
+ ComponentInstanceState componentStatus = getComponentStatuses().get(label);
componentStatus.setLastHeartbeat(System.currentTimeMillis());
// If no Master can explicitly publish then publish if its a master
// Otherwise, wait till the master that can publish is ready
@@ -405,7 +408,7 @@ public class AgentProviderService extends AbstractProviderService implements
if (id < waitForCount) {
log.info("Waiting until heartbeat count {}. Current val: {}", waitForCount, id);
- componentStatuses.put(roleName, componentStatus);
+ getComponentStatuses().put(roleName, componentStatus);
return response;
}
@@ -418,7 +421,7 @@ public class AgentProviderService extends AbstractProviderService implements
componentStatus.commandIssued(command);
} else if (command == Command.START) {
// check against dependencies
- boolean canExecute = commandOrder.canExecute(roleName, command, componentStatuses.values());
+ boolean canExecute = commandOrder.canExecute(roleName, command, getComponentStatuses().values());
if (canExecute) {
log.info("Starting {} on {}.", roleName, containerId);
addStartCommand(roleName, containerId, response, scriptPath);
@@ -474,17 +477,21 @@ public class AgentProviderService extends AbstractProviderService implements
public void notifyContainerCompleted(ContainerId containerId) {
if (containerId != null) {
String containerIdStr = containerId.toString();
- if (this.componentInstanceData.containsKey(containerIdStr)) {
- synchronized (this.componentInstanceData) {
- this.componentInstanceData.remove(containerIdStr);
- }
+ if (getComponentInstanceData().containsKey(containerIdStr)) {
+ getComponentInstanceData().remove(containerIdStr);
log.info("Removing container specific data for {}", containerIdStr);
publishComponentInstanceData();
}
if (this.allocatedPorts.containsKey(containerIdStr)) {
- synchronized (this.allocatedPorts) {
- this.allocatedPorts.remove(containerIdStr);
+ this.allocatedPorts.remove(containerIdStr);
+ }
+
+ synchronized (this.componentStatuses) {
+ for (String label : getComponentStatuses().keySet()) {
+ if (label.startsWith(containerIdStr)) {
+ getComponentStatuses().remove(label);
+ }
}
}
}
@@ -589,7 +596,7 @@ public class AgentProviderService extends AbstractProviderService implements
* @return if release is requested successfully
*/
protected boolean releaseContainer(String label) {
- componentStatuses.remove(label);
+ getComponentStatuses().remove(label);
try {
getAppMaster().refreshContainer(getContainerId(label), true);
} catch (SliderException e) {
@@ -640,7 +647,7 @@ public class AgentProviderService extends AbstractProviderService implements
boolean hasExportGroups = exportGroups != null && !exportGroups.isEmpty();
Set<String> exportedConfigs = new HashSet();
String exportedConfigsStr = application.getExportedConfigs();
- boolean exportedConfigSpecified = exportedConfigsStr != null;
+ boolean exportedAllConfigs = exportedConfigsStr == null || exportedConfigsStr.isEmpty();
if (application.getExportedConfigs() != null && application.getExportedConfigs().length() > 0) {
for (String exportedConfig : application.getExportedConfigs().split(",")) {
if (exportedConfig.trim().length() > 0) {
@@ -653,8 +660,8 @@ public class AgentProviderService extends AbstractProviderService implements
log.info("Status report: " + status.toString());
if (status.getConfigs() != null) {
for (String key : status.getConfigs().keySet()) {
- if ((exportedConfigSpecified && exportedConfigs.contains(key)) ||
- !exportedConfigSpecified) {
+ if ((!exportedAllConfigs && exportedConfigs.contains(key)) ||
+ exportedAllConfigs) {
Map<String, String> configs = status.getConfigs().get(key);
publishApplicationInstanceData(key, key, configs.entrySet());
}
@@ -745,13 +752,13 @@ public class AgentProviderService extends AbstractProviderService implements
if (toPublish.size() > 0) {
Map<String, String> perContainerData = null;
- if (!this.componentInstanceData.containsKey(containerId)) {
+ if (!getComponentInstanceData().containsKey(containerId)) {
perContainerData = new ConcurrentHashMap<>();
} else {
- perContainerData = this.componentInstanceData.get(containerId);
+ perContainerData = getComponentInstanceData().get(containerId);
}
perContainerData.putAll(toPublish);
- this.componentInstanceData.put(containerId, perContainerData);
+ getComponentInstanceData().put(containerId, perContainerData);
publishComponentInstanceData();
}
}
@@ -759,10 +766,10 @@ public class AgentProviderService extends AbstractProviderService implements
private void publishComponentInstanceData() {
Map<String, String> dataToPublish = new HashMap<>();
synchronized (this.componentInstanceData) {
- for (String container : this.componentInstanceData.keySet()) {
- for (String prop : this.componentInstanceData.get(container).keySet()) {
+ for (String container : getComponentInstanceData().keySet()) {
+ for (String prop : getComponentInstanceData().get(container).keySet()) {
dataToPublish.put(
- container + "." + prop, this.componentInstanceData.get(container).get(prop));
+ container + "." + prop, getComponentInstanceData().get(container).get(prop));
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/405aa51e/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 4b2e253..922de68 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -820,20 +820,31 @@ public class TestAgentProviderService {
mockAps.getComponentInstanceData().put("cid2", new HashMap<String, String>());
mockAps.getComponentInstanceData().put(id, new HashMap<String, String>());
+ mockAps.getComponentStatuses().put("cid2_HM", new ComponentInstanceState("HM", "cid2", "aid"));
+ mockAps.getComponentStatuses().put(id + "_HM", new ComponentInstanceState("HM", id, "aid"));
+
Assert.assertNotNull(mockAps.getComponentInstanceData().get(id));
Assert.assertNotNull(mockAps.getComponentInstanceData().get("cid2"));
+ Assert.assertNotNull(mockAps.getComponentStatuses().get(id + "_HM"));
+ Assert.assertNotNull(mockAps.getComponentStatuses().get("cid2_HM"));
+
Assert.assertEquals(mockAps.getAllocatedPorts().size(), 1);
Assert.assertEquals(mockAps.getAllocatedPorts(id).size(), 1);
Assert.assertEquals(mockAps.getAllocatedPorts("cid2").size(), 1);
- mockAps.notifyContainerCompleted(new MyContainerId(1));
- Assert.assertNull(mockAps.getComponentInstanceData().get(id));
- Assert.assertNotNull(mockAps.getComponentInstanceData().get("cid2"));
+ // Make the call
+ mockAps.notifyContainerCompleted(new MyContainerId(1));
Assert.assertEquals(mockAps.getAllocatedPorts().size(), 1);
Assert.assertEquals(mockAps.getAllocatedPorts(id).size(), 0);
Assert.assertEquals(mockAps.getAllocatedPorts("cid2").size(), 1);
+
+ Assert.assertNull(mockAps.getComponentInstanceData().get(id));
+ Assert.assertNotNull(mockAps.getComponentInstanceData().get("cid2"));
+
+ Assert.assertNull(mockAps.getComponentStatuses().get(id + "_HM"));
+ Assert.assertNotNull(mockAps.getComponentStatuses().get("cid2_HM"));
}
@Test
[18/50] [abbrv] git commit: SLIDER-276 : AgentProvider releases nodes
that the AM has already been released after detecting heartbeat failure
Posted by st...@apache.org.
SLIDER-276 : AgentProvider releases nodes that the AM has already been released after detecting heartbeat failure
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/32fd3f56
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/32fd3f56
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/32fd3f56
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 32fd3f56a785e29b55c0560c6256e50bf07215b8
Parents: d68be5a
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 6 14:41:00 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 6 14:41:00 2014 +0100
----------------------------------------------------------------------
.../providers/agent/AgentProviderService.java | 34 ++---
.../providers/agent/ComponentCommandOrder.java | 8 +-
.../providers/agent/ComponentInstanceState.java | 80 +++++++---
.../providers/agent/HeartbeatMonitor.java | 66 +++++---
.../server/appmaster/AMViewForProviders.java | 21 ++-
.../server/appmaster/SliderAppMaster.java | 27 ++--
.../slider/server/appmaster/state/AppState.java | 6 +-
.../appmaster/model/mock/MockContainer.groovy | 2 +-
.../agent/TestAgentProviderService.java | 151 +++----------------
.../agent/TestComponentCommandOrder.java | 17 ++-
.../agent/TestComponentInstanceState.java | 124 ++++++++-------
.../providers/agent/TestHeartbeatMonitor.java | 120 ++++++++++-----
12 files changed, 337 insertions(+), 319 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/32fd3f56/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 6cfadf3..4d54509 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -58,6 +58,7 @@ import org.apache.slider.providers.agent.application.metadata.ExportGroup;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.providers.agent.application.metadata.OSPackage;
import org.apache.slider.providers.agent.application.metadata.OSSpecific;
+import org.apache.slider.server.appmaster.AMViewForProviders;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
@@ -127,10 +128,10 @@ public class AgentProviderService extends AbstractProviderService implements
private Boolean canAnyMasterPublish = null;
private AgentLaunchParameter agentLaunchParameter = null;
- private Map<String, ComponentInstanceState> componentStatuses = new ConcurrentHashMap<>();
- private Map<String, Map<String, String>> componentInstanceData = new ConcurrentHashMap();
+ private final Map<String, ComponentInstanceState> componentStatuses = new ConcurrentHashMap<>();
+ private final Map<String, Map<String, String>> componentInstanceData = new ConcurrentHashMap<>();
private final Map<String, Map<String, String>> allocatedPorts = new ConcurrentHashMap<>();
- private Map<String, String> workFolders =
+ private final Map<String, String> workFolders =
Collections.synchronizedMap(new LinkedHashMap<String, String>(MAX_LOG_ENTRIES, 0.75f, false) {
protected boolean removeEldestEntry(Map.Entry eldest) {
return size() > MAX_LOG_ENTRIES;
@@ -291,7 +292,7 @@ public class AgentProviderService extends AbstractProviderService implements
getComponentStatuses().put(label,
new ComponentInstanceState(
role,
- container.getId().toString(),
+ container.getId(),
getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME)));
}
@@ -333,7 +334,7 @@ public class AgentProviderService extends AbstractProviderService implements
String label = registration.getHostname();
if (getComponentStatuses().containsKey(label)) {
response.setResponseStatus(RegistrationStatus.OK);
- getComponentStatuses().get(label).setLastHeartbeat(System.currentTimeMillis());
+ getComponentStatuses().get(label).heartbeat(System.currentTimeMillis());
} else {
response.setResponseStatus(RegistrationStatus.FAILED);
response.setLog("Label not recognized.");
@@ -371,7 +372,7 @@ public class AgentProviderService extends AbstractProviderService implements
Boolean isMaster = isMaster(roleName);
ComponentInstanceState componentStatus = getComponentStatuses().get(label);
- componentStatus.setLastHeartbeat(System.currentTimeMillis());
+ componentStatus.heartbeat(System.currentTimeMillis());
// If no Master can explicitly publish then publish if its a master
// Otherwise, wait till the master that can publish is ready
if (isMaster &&
@@ -591,20 +592,18 @@ public class AgentProviderService extends AbstractProviderService implements
/**
* Lost heartbeat from the container - release it and ask for a replacement
*
+ *
* @param label
+ * @param containerId
*
- * @return if release is requested successfully
+ * @return outcome of the operation
*/
- protected boolean releaseContainer(String label) {
+ protected AMViewForProviders.ContainerLossReportOutcomes lostContainer(
+ String label,
+ ContainerId containerId) throws
+ SliderException {
getComponentStatuses().remove(label);
- try {
- getAppMaster().refreshContainer(getContainerId(label), true);
- } catch (SliderException e) {
- log.info("Error while requesting container release for {}. Message: {}", label, e.getMessage());
- return false;
- }
-
- return true;
+ return getAppMaster().providerLostContainer(containerId);
}
/**
@@ -1059,7 +1058,8 @@ public class AgentProviderService extends AbstractProviderService implements
if (!this.allocatedPorts.containsKey(containerId)) {
synchronized (this.allocatedPorts) {
if (!this.allocatedPorts.containsKey(containerId)) {
- this.allocatedPorts.put(containerId, new ConcurrentHashMap());
+ this.allocatedPorts.put(containerId,
+ new ConcurrentHashMap<String, String>());
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/32fd3f56/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
index 0dce4bb..a23cc38 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
@@ -126,18 +126,18 @@ public class ComponentCommandOrder {
for (ComponentState stateToMatch : required) {
for (ComponentInstanceState currState : currentStates) {
log.debug("Checking schedule {} {} against dependency {} is {}",
- component, command, currState.getCompName(), currState.getState());
- if (currState.getCompName().equals(stateToMatch.componentName)) {
+ component, command, currState.getComponentName(), currState.getState());
+ if (currState.getComponentName().equals(stateToMatch.componentName)) {
if (currState.getState() != stateToMatch.state) {
if (stateToMatch.state == State.STARTED) {
log.info("Cannot schedule {} {} as dependency {} is {}",
- component, command, currState.getCompName(), currState.getState());
+ component, command, currState.getComponentName(), currState.getState());
canExecute = false;
} else {
//state is INSTALLED
if (currState.getState() != State.STARTING && currState.getState() != State.STARTED) {
log.info("Cannot schedule {} {} as dependency {} is {}",
- component, command, currState.getCompName(), currState.getState());
+ component, command, currState.getComponentName(), currState.getState());
canExecute = false;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/32fd3f56/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
index 60a6f82..f7f8bf4 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
@@ -19,6 +19,8 @@
package org.apache.slider.providers.agent;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,8 +32,9 @@ public class ComponentInstanceState {
private static String INVALID_TRANSITION_ERROR =
"Result {0} for command {1} is not expected for component {2} in state {3}.";
- private final String compName;
- private final String containerId;
+ private final String componentName;
+ private final ContainerId containerId;
+ private final String containerIdAsString;
private final String applicationId;
private State state = State.INIT;
private State targetState = State.STARTED;
@@ -40,18 +43,19 @@ public class ComponentInstanceState {
private long lastHeartbeat = 0;
private ContainerState containerState;
- public ComponentInstanceState(String compName,
- String containerId,
- String applicationId) {
- this.compName = compName;
+ public ComponentInstanceState(String componentName,
+ ContainerId containerId,
+ String applicationId) {
+ this.componentName = componentName;
this.containerId = containerId;
+ this.containerIdAsString = containerId.toString();
this.applicationId = applicationId;
this.containerState = ContainerState.INIT;
this.lastHeartbeat = System.currentTimeMillis();
}
- public String getCompName() {
- return compName;
+ public String getComponentName() {
+ return componentName;
}
public Boolean getConfigReported() {
@@ -74,20 +78,32 @@ public class ComponentInstanceState {
return lastHeartbeat;
}
- public void setLastHeartbeat(long lastHeartbeat) {
- this.lastHeartbeat = lastHeartbeat;
- if(this.containerState == ContainerState.UNHEALTHY ||
- this.containerState == ContainerState.INIT) {
- this.containerState = ContainerState.HEALTHY;
+ /**
+ * Update the heartbeat, and change container state
+ * to mark as healthy if appropriate
+ * @param heartbeatTime last time the heartbeat was seen
+ * @return the current container state
+ */
+ public ContainerState heartbeat(long heartbeatTime) {
+ this.lastHeartbeat = heartbeatTime;
+ if(containerState == ContainerState.UNHEALTHY ||
+ containerState == ContainerState.INIT) {
+ containerState = ContainerState.HEALTHY;
}
+ return containerState;
+ }
+
+
+ public ContainerId getContainerId() {
+ return containerId;
}
public void commandIssued(Command command) {
Command expected = getNextCommand();
if (expected != command) {
- throw new IllegalArgumentException("Command " + command + " is not allowed is state " + state);
+ throw new IllegalArgumentException("Command " + command + " is not allowed in state " + state);
}
- this.state = this.state.getNextState(command);
+ state = state.getNextState(command);
}
public void applyCommandResult(CommandResult result, Command command) {
@@ -101,12 +117,12 @@ public class ComponentInstanceState {
} else if (result == CommandResult.COMPLETED) {
failuresSeen = 0;
}
- this.state = this.state.getNextState(result);
+ state = state.getNextState(result);
} catch (IllegalArgumentException e) {
String message = String.format(INVALID_TRANSITION_ERROR,
result.toString(),
command.toString(),
- compName,
+ componentName,
state.toString());
log.warn(message);
throw new IllegalStateException(message);
@@ -114,8 +130,8 @@ public class ComponentInstanceState {
}
public boolean hasPendingCommand() {
- if (this.state.canIssueCommands() &&
- this.state != this.targetState &&
+ if (state.canIssueCommands() &&
+ state != targetState &&
failuresSeen < MAX_FAILURE_TOLERATED) {
return true;
}
@@ -144,8 +160,8 @@ public class ComponentInstanceState {
public int hashCode() {
int hashCode = 1;
- hashCode = hashCode ^ (compName != null ? compName.hashCode() : 0);
- hashCode = hashCode ^ (containerId != null ? containerId.hashCode() : 0);
+ hashCode = hashCode ^ (componentName != null ? componentName.hashCode() : 0);
+ hashCode = hashCode ^ (containerIdAsString != null ? containerIdAsString.hashCode() : 0);
hashCode = hashCode ^ (applicationId != null ? applicationId.hashCode() : 0);
return hashCode;
}
@@ -158,13 +174,13 @@ public class ComponentInstanceState {
ComponentInstanceState that = (ComponentInstanceState) o;
- if (this.compName != null ?
- !this.compName.equals(that.compName) : this.compName != null) {
+ if (this.componentName != null ?
+ !this.componentName.equals(that.componentName) : this.componentName != null) {
return false;
}
- if (this.containerId != null ?
- !this.containerId.equals(that.containerId) : this.containerId != null) {
+ if (this.containerIdAsString != null ?
+ !this.containerIdAsString.equals(that.containerIdAsString) : this.containerIdAsString != null) {
return false;
}
@@ -175,4 +191,18 @@ public class ComponentInstanceState {
return true;
}
+
+ @Override
+ public String toString() {
+ final StringBuilder sb =
+ new StringBuilder("ComponentInstanceState{");
+ sb.append("containerIdAsString='").append(containerIdAsString).append('\'');
+ sb.append(", state=").append(state);
+ sb.append(", failuresSeen=").append(failuresSeen);
+ sb.append(", lastHeartbeat=").append(lastHeartbeat);
+ sb.append(", containerState=").append(containerState);
+ sb.append(", componentName='").append(componentName).append('\'');
+ sb.append('}');
+ return sb.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/32fd3f56/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java b/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java
index 3aeff66..cb22d2e 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java
@@ -17,6 +17,9 @@
*/
package org.apache.slider.providers.agent;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.slider.core.exceptions.SliderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,10 +56,7 @@ public class HeartbeatMonitor implements Runnable {
}
public boolean isAlive() {
- if (monitorThread != null) {
- return monitorThread.isAlive();
- }
- return false;
+ return monitorThread != null && monitorThread.isAlive();
}
@Override
@@ -66,7 +66,7 @@ public class HeartbeatMonitor implements Runnable {
log.debug("Putting monitor to sleep for " + threadWakeupInterval + " " +
"milliseconds");
Thread.sleep(threadWakeupInterval);
- doWork();
+ doWork(System.currentTimeMillis());
} catch (InterruptedException ex) {
log.warn("Scheduler thread is interrupted going to stop", ex);
shouldRun = false;
@@ -83,32 +83,52 @@ public class HeartbeatMonitor implements Runnable {
* received in last check interval they are marked as UNHEALTHY. INIT is when the agent is started but it did not
* communicate at all. HEALTHY being the AM has received heartbeats. After an interval as UNHEALTHY the container is
* declared unavailable
+ * @param now current time in milliseconds ... tests can set this explicitly
*/
- private void doWork() {
+ @VisibleForTesting
+ public void doWork(long now) {
Map<String, ComponentInstanceState> componentStatuses = provider.getComponentStatuses();
if (componentStatuses != null) {
for (String containerLabel : componentStatuses.keySet()) {
ComponentInstanceState componentInstanceState = componentStatuses.get(containerLabel);
- long timeSinceLastHeartbeat = System.currentTimeMillis() - componentInstanceState.getLastHeartbeat();
+ long timeSinceLastHeartbeat = now - componentInstanceState.getLastHeartbeat();
if (timeSinceLastHeartbeat > threadWakeupInterval) {
- if (componentInstanceState.getContainerState() == ContainerState.HEALTHY ||
- componentInstanceState.getContainerState() == ContainerState.INIT) {
- componentInstanceState.setContainerState(ContainerState.UNHEALTHY);
- log.warn("Component {} marked UNHEALTHY. Last heartbeat received at {} approx. {} ms. back.",
- containerLabel, componentInstanceState.getLastHeartbeat(),
- timeSinceLastHeartbeat);
- continue;
- }
- if (componentInstanceState.getContainerState() == ContainerState.UNHEALTHY
- && timeSinceLastHeartbeat > threadWakeupInterval * 2) {
- componentInstanceState.setContainerState(ContainerState.HEARTBEAT_LOST);
- log.warn("Component {} marked HEARTBEAT_LOST. Last heartbeat received at {} approx. {} ms. back.",
- containerLabel, componentInstanceState.getLastHeartbeat(),
- timeSinceLastHeartbeat);
- this.provider.releaseContainer(containerLabel);
- continue;
+ switch (componentInstanceState.getContainerState()) {
+ case INIT:
+ case HEALTHY:
+ componentInstanceState.setContainerState(ContainerState.UNHEALTHY);
+ log.warn(
+ "Component {} marked UNHEALTHY. Last heartbeat received at {} approx. {} ms. back.",
+ componentInstanceState,
+ componentInstanceState.getLastHeartbeat(),
+ timeSinceLastHeartbeat);
+ break;
+ case UNHEALTHY:
+ if (timeSinceLastHeartbeat > threadWakeupInterval * 2) {
+ componentInstanceState.setContainerState(
+ ContainerState.HEARTBEAT_LOST);
+ log.warn(
+ "Component {} marked HEARTBEAT_LOST. Last heartbeat received at {} approx. {} ms. back.",
+ componentInstanceState, componentInstanceState.getLastHeartbeat(),
+ timeSinceLastHeartbeat);
+ ContainerId containerId =
+ componentInstanceState.getContainerId();
+ try {
+ provider.lostContainer(containerLabel, containerId);
+ } catch (SliderException e) {
+ log.info(
+ "Error while requesting container release for {}. Message: {}",
+ containerId, e, e);
+ }
+ }
+ break;
+ case HEARTBEAT_LOST:
+ // unexpected case
+ log.warn("Heartbeat from lost component: {}", componentInstanceState);
+ break;
}
+
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/32fd3f56/slider-core/src/main/java/org/apache/slider/server/appmaster/AMViewForProviders.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/AMViewForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/AMViewForProviders.java
index 287035f..f426551 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/AMViewForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/AMViewForProviders.java
@@ -18,10 +18,29 @@
package org.apache.slider.server.appmaster;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.slider.core.exceptions.SliderException;
/** Operations available to a provider from AppMaster */
public interface AMViewForProviders {
+
+ /**
+ * Outcomes from container loss
+ */
+ enum ContainerLossReportOutcomes {
+ /**
+ * The container doesn't exist...either it wasn't in use or it
+ * has been released
+ */
+ CONTAINER_NOT_IN_USE,
+
+ /**
+ * The container is known about and a review has been initated
+ */
+ CONTAINER_LOST_REVIEW_INITIATED,
+ }
+
/** Provider can ask AppMaster to release a specific container */
- void refreshContainer(String containerId, boolean newHostIfPossible) throws SliderException;
+ ContainerLossReportOutcomes providerLostContainer(ContainerId containerId) throws SliderException;
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/32fd3f56/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 96da98d..b3650ca 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -1431,22 +1431,27 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
/* =================================================================== */
/**
- * Refreshes the container by releasing it and having it reallocated
- *
- * @param containerId id of the container to release
- * @param newHostIfPossible allocate the replacement container on a new host
+ * report container loss. If this isn't already known about, react
*
+ * @param containerId id of the container which has failed
* @throws SliderException
*/
- public void refreshContainer(String containerId, boolean newHostIfPossible)
+ public synchronized ContainerLossReportOutcomes providerLostContainer(
+ ContainerId containerId)
throws SliderException {
- log.info(
- "Refreshing container {} per provider request.",
+ log.info("containerLostContactWithProvider: container {} lost",
containerId);
- rmOperationHandler.execute(appState.releaseContainer(containerId));
-
- // ask for more containers if needed
- reviewRequestAndReleaseNodes();
+ RoleInstance activeContainer = appState.getActiveContainer(containerId);
+ if (activeContainer != null) {
+ rmOperationHandler.execute(appState.releaseContainer(containerId));
+ // ask for more containers if needed
+ log.info("Container released; triggering review");
+ reviewRequestAndReleaseNodes();
+ return ContainerLossReportOutcomes.CONTAINER_LOST_REVIEW_INITIATED;
+ } else {
+ log.info("Container not in active set - ignoring");
+ return ContainerLossReportOutcomes.CONTAINER_NOT_IN_USE;
+ }
}
/* =================================================================== */
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/32fd3f56/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index db74ea8..08d43db 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1217,7 +1217,7 @@ public class AppState {
//build the failure message
if (failedContainer != null) {
String completedLogsUrl = getLogsURLForContainer(failedContainer);
- message = String.format("Failure %s on host %s: %s" +
+ message = String.format("Failure %s on host %s: %s",
roleInstance.getContainerId(),
failedContainer.getNodeId().getHost(),
completedLogsUrl);
@@ -1510,12 +1510,12 @@ public class AppState {
* @return
* @throws SliderInternalStateException
*/
- public List<AbstractRMOperation> releaseContainer(String containerId)
+ public List<AbstractRMOperation> releaseContainer(ContainerId containerId)
throws SliderInternalStateException {
List<AbstractRMOperation> operations = new ArrayList<>();
List<RoleInstance> activeRoleInstances = cloneActiveContainerList();
for (RoleInstance role : activeRoleInstances) {
- if (role.container.getId().toString().equals(containerId)) {
+ if (role.container.getId().equals(containerId)) {
containerReleaseSubmitted(role.container);
operations.add(new ContainerReleaseOperation(role.getId()));
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/32fd3f56/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainer.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainer.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainer.groovy
index 25bee36..3eba7c4 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainer.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainer.groovy
@@ -20,7 +20,7 @@ package org.apache.slider.server.appmaster.model.mock
import org.apache.hadoop.yarn.api.records.*
-class MockContainer extends Container{
+class MockContainer extends Container {
ContainerId id;
NodeId nodeId
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/32fd3f56/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 922de68..d16603f 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -21,7 +21,6 @@ package org.apache.slider.providers.agent;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FilterFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
@@ -85,6 +84,7 @@ import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.createNiceMock;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
+import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyCollection;
import static org.mockito.Matchers.anyString;
@@ -315,15 +315,15 @@ public class TestAgentProviderService {
ClusterDescription cd = new ClusterDescription();
cd.status = new HashMap<String, Object>();
Map<String, Map<String, ClusterNode>> roleMap = new HashMap<>();
- ClusterNode cn1 = new ClusterNode(new MyContainerId(1));
+ ClusterNode cn1 = new ClusterNode(new MockContainerId(1));
cn1.host = "FIRST_HOST";
Map<String, ClusterNode> map1 = new HashMap<>();
map1.put("FIRST_CONTAINER", cn1);
- ClusterNode cn2 = new ClusterNode(new MyContainerId(2));
+ ClusterNode cn2 = new ClusterNode(new MockContainerId(2));
cn2.host = "SECOND_HOST";
Map<String, ClusterNode> map2 = new HashMap<>();
map2.put("SECOND_CONTAINER", cn2);
- ClusterNode cn3 = new ClusterNode(new MyContainerId(3));
+ ClusterNode cn3 = new ClusterNode(new MockContainerId(3));
cn3.host = "THIRD_HOST";
map2.put("THIRD_CONTAINER", cn3);
@@ -416,12 +416,13 @@ public class TestAgentProviderService {
Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<>();
Map<String, ClusterNode> container = new HashMap<>();
- ClusterNode cn1 = new ClusterNode(new MyContainerId(1));
+ ClusterNode cn1 = new ClusterNode(new MockContainerId(1));
cn1.host = "HOST1";
container.put("cid1", cn1);
roleClusterNodeMap.put("HBASE_MASTER", container);
- ComponentInstanceState componentStatus = new ComponentInstanceState("HBASE_MASTER", "aid", "cid");
+ ComponentInstanceState componentStatus = new ComponentInstanceState("HBASE_MASTER",
+ new MockContainerId(1), "cid");
AgentProviderService mockAps = Mockito.spy(aps);
doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
doReturn(metainfo).when(mockAps).getMetainfo();
@@ -572,7 +573,7 @@ public class TestAgentProviderService {
}
@Test
- public void testOrchastratedAppStart() throws IOException {
+ public void testOrchestratedAppStart() throws IOException {
// App has two components HBASE_MASTER and HBASE_REGIONSERVER
// Start of HBASE_RS depends on the start of HBASE_MASTER
InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
@@ -811,8 +812,9 @@ public class TestAgentProviderService {
AgentProviderService mockAps = Mockito.spy(aps);
doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
- ContainerId cid = new MyContainerId(1);
+ ContainerId cid = new MockContainerId(1);
String id = cid.toString();
+ ContainerId cid2 = new MockContainerId(2);
mockAps.getAllocatedPorts().put("a", "100");
mockAps.getAllocatedPorts(id).put("b", "101");
mockAps.getAllocatedPorts("cid2").put("c", "102");
@@ -820,8 +822,8 @@ public class TestAgentProviderService {
mockAps.getComponentInstanceData().put("cid2", new HashMap<String, String>());
mockAps.getComponentInstanceData().put(id, new HashMap<String, String>());
- mockAps.getComponentStatuses().put("cid2_HM", new ComponentInstanceState("HM", "cid2", "aid"));
- mockAps.getComponentStatuses().put(id + "_HM", new ComponentInstanceState("HM", id, "aid"));
+ mockAps.getComponentStatuses().put("cid2_HM", new ComponentInstanceState("HM", cid2, "aid"));
+ mockAps.getComponentStatuses().put(id + "_HM", new ComponentInstanceState("HM", cid, "aid"));
Assert.assertNotNull(mockAps.getComponentInstanceData().get(id));
Assert.assertNotNull(mockAps.getComponentInstanceData().get("cid2"));
@@ -834,7 +836,7 @@ public class TestAgentProviderService {
Assert.assertEquals(mockAps.getAllocatedPorts("cid2").size(), 1);
// Make the call
- mockAps.notifyContainerCompleted(new MyContainerId(1));
+ mockAps.notifyContainerCompleted(new MockContainerId(1));
Assert.assertEquals(mockAps.getAllocatedPorts().size(), 1);
Assert.assertEquals(mockAps.getAllocatedPorts(id).size(), 0);
@@ -875,7 +877,7 @@ public class TestAgentProviderService {
Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<>();
Map<String, ClusterNode> container = new HashMap<>();
- ClusterNode cn1 = new ClusterNode(new MyContainerId(1));
+ ClusterNode cn1 = new ClusterNode(new MockContainerId(1));
cn1.host = "HOST1";
container.put("cid1", cn1);
roleClusterNodeMap.put("HBASE_MASTER", container);
@@ -922,7 +924,7 @@ public class TestAgentProviderService {
Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<>();
Map<String, ClusterNode> container = new HashMap<>();
- ClusterNode cn1 = new ClusterNode(new MyContainerId(1));
+ ClusterNode cn1 = new ClusterNode(new MockContainerId(1));
cn1.host = "HOST1";
container.put("cid1", cn1);
roleClusterNodeMap.put("HBASE_MASTER", container);
@@ -941,124 +943,11 @@ public class TestAgentProviderService {
Assert.assertTrue(hbr.getExecutionCommands().get(0).getConfigurations().containsKey("hbase-site"));
Map<String, String> hbaseSiteConf = hbr.getExecutionCommands().get(0).getConfigurations().get("hbase-site");
Assert.assertTrue(hbaseSiteConf.containsKey("a.port"));
- Assert.assertTrue(hbaseSiteConf.get("a.port").equals("10023"));
- Assert.assertTrue(hbaseSiteConf.get("b.port").equals("10024"));
- Assert.assertTrue(hbaseSiteConf.get("random.port").equals("10025"));
- Assert.assertTrue(hbaseSiteConf.get("random2.port").equals("${HBASE_MASTER.ALLOCATED_PORT}"));
+ Assert.assertEquals("10023", hbaseSiteConf.get("a.port"));
+ Assert.assertEquals("10024", hbaseSiteConf.get("b.port"));
+ Assert.assertEquals("10025", hbaseSiteConf.get("random.port"));
+ assertEquals("${HBASE_MASTER.ALLOCATED_PORT}",
+ hbaseSiteConf.get("random2.port"));
}
- private static class MyContainer extends Container {
-
- ContainerId cid = null;
-
- @Override
- public ContainerId getId() {
- return this.cid;
- }
-
- @Override
- public void setId(ContainerId containerId) {
- this.cid = containerId;
- }
-
- @Override
- public NodeId getNodeId() {
- return null; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public void setNodeId(NodeId nodeId) {
- //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public String getNodeHttpAddress() {
- return null; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public void setNodeHttpAddress(String s) {
- //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public Resource getResource() {
- return null; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public void setResource(Resource resource) {
- //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public Priority getPriority() {
- return null; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public void setPriority(Priority priority) {
- //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public Token getContainerToken() {
- return null; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public void setContainerToken(Token token) {
- //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public int compareTo(Container o) {
- return 0; //To change body of implemented methods use File | Settings | File Templates.
- }
- }
-
- private static class MyContainerId extends ContainerId {
- int id;
-
- private MyContainerId(int id) {
- this.id = id;
- }
-
- @Override
- public ApplicationAttemptId getApplicationAttemptId() {
- return null; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- protected void setApplicationAttemptId(ApplicationAttemptId applicationAttemptId) {
- //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public int getId() {
- return id; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- protected void setId(int i) {
- //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- protected void build() {
- //To change body of implemented methods use File | Settings | File Templates.
- }
-
- @Override
- public int hashCode() {
- return this.id;
- }
-
- @Override
- public String toString() {
- return "MyContainerId{" +
- "id=" + id +
- '}';
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/32fd3f56/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentCommandOrder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentCommandOrder.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentCommandOrder.java
index 3ef1839..c123fbb 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentCommandOrder.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentCommandOrder.java
@@ -19,6 +19,7 @@
package org.apache.slider.providers.agent;
import org.apache.slider.providers.agent.application.metadata.CommandOrder;
+import org.apache.slider.server.appmaster.model.mock.MockContainerId;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
@@ -29,6 +30,7 @@ import java.util.Arrays;
public class TestComponentCommandOrder {
protected static final Logger log =
LoggerFactory.getLogger(TestComponentCommandOrder.class);
+ private final MockContainerId containerId = new MockContainerId(1);
@Test
public void testComponentCommandOrder() throws Exception {
@@ -43,11 +45,12 @@ public class TestComponentCommandOrder {
co3.setRequires("C-STARTED,D-STARTED,E-INSTALLED");
ComponentCommandOrder cco = new ComponentCommandOrder(Arrays.asList(co1, co2, co3));
- ComponentInstanceState cisB = new ComponentInstanceState("B", "cid", "aid");
- ComponentInstanceState cisC = new ComponentInstanceState("C", "cid", "aid");
- ComponentInstanceState cisD = new ComponentInstanceState("D", "cid", "aid");
- ComponentInstanceState cisE = new ComponentInstanceState("E", "cid", "aid");
- ComponentInstanceState cisE2 = new ComponentInstanceState("E", "cid", "aid");
+ ComponentInstanceState cisB = new ComponentInstanceState("B",
+ containerId, "aid");
+ ComponentInstanceState cisC = new ComponentInstanceState("C", containerId, "aid");
+ ComponentInstanceState cisD = new ComponentInstanceState("D", containerId, "aid");
+ ComponentInstanceState cisE = new ComponentInstanceState("E", containerId, "aid");
+ ComponentInstanceState cisE2 = new ComponentInstanceState("E", containerId, "aid");
cisB.setState(State.STARTED);
cisC.setState(State.INSTALLED);
Assert.assertTrue(cco.canExecute("A", Command.START, Arrays.asList(cisB)));
@@ -92,8 +95,8 @@ public class TestComponentCommandOrder {
co.setCommand(" A-START");
co.setRequires("B-STARTED , C-STARTED");
- ComponentInstanceState cisB = new ComponentInstanceState("B", "cid", "aid");
- ComponentInstanceState cisC = new ComponentInstanceState("C", "cid", "aid");
+ ComponentInstanceState cisB = new ComponentInstanceState("B", containerId, "aid");
+ ComponentInstanceState cisC = new ComponentInstanceState("C", containerId, "aid");
cisB.setState(State.STARTED);
cisC.setState(State.STARTED);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/32fd3f56/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentInstanceState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentInstanceState.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentInstanceState.java
index be9f178..e1f3d02 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentInstanceState.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentInstanceState.java
@@ -19,6 +19,7 @@
package org.apache.slider.providers.agent;
import junit.framework.TestCase;
+import org.apache.slider.server.appmaster.model.mock.MockContainerId;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -29,6 +30,7 @@ public class TestComponentInstanceState {
private State[] states = new State[]{
State.INIT, State.INSTALLING, State.INSTALLED,
State.STARTING, State.STARTED, State.INSTALL_FAILED};
+ private final MockContainerId containerId = new MockContainerId(1);
@Test
public void testValidateSupportedCommands() {
@@ -67,97 +69,107 @@ public class TestComponentInstanceState {
@Test
public void testGetNextStateBasedOnCommand() {
- for (int index = 0; index < states.length; index++) {
- TestCase.assertEquals(states[index], states[index].getNextState(Command.NOP));
+ for (State state : states) {
+ TestCase.assertEquals(state, state.getNextState(Command.NOP));
}
TestCase.assertEquals(State.INSTALLING, State.INIT.getNextState(Command.INSTALL));
TestCase.assertEquals(State.INSTALLING, State.INSTALL_FAILED.getNextState(Command.INSTALL));
- expectExceptionOnGetNextForCommand(IllegalArgumentException.class, State.INSTALLED, Command.INSTALL);
- expectExceptionOnGetNextForCommand(IllegalArgumentException.class, State.STARTING, Command.INSTALL);
- expectExceptionOnGetNextForCommand(IllegalArgumentException.class, State.STARTED, Command.INSTALL);
+ expectIllegalArgumentException(State.INSTALLED, Command.INSTALL);
+ expectIllegalArgumentException(State.STARTING, Command.INSTALL);
+ expectIllegalArgumentException(State.STARTED, Command.INSTALL);
TestCase.assertEquals(State.STARTING, State.INSTALLED.getNextState(Command.START));
- expectExceptionOnGetNextForCommand(IllegalArgumentException.class, State.INIT, Command.START);
- expectExceptionOnGetNextForCommand(IllegalArgumentException.class, State.INSTALL_FAILED, Command.START);
- expectExceptionOnGetNextForCommand(IllegalArgumentException.class, State.STARTING, Command.START);
- expectExceptionOnGetNextForCommand(IllegalArgumentException.class, State.INSTALLING, Command.START);
- expectExceptionOnGetNextForCommand(IllegalArgumentException.class, State.STARTED, Command.START);
+ expectIllegalArgumentException(State.INIT, Command.START);
+ expectIllegalArgumentException(State.INSTALL_FAILED, Command.START);
+ expectIllegalArgumentException(State.STARTING, Command.START);
+ expectIllegalArgumentException(State.INSTALLING, Command.START);
+ expectIllegalArgumentException(State.STARTED, Command.START);
+ }
+
+ protected void expectIllegalArgumentException(State state, Command command) {
+ expectExceptionOnGetNextForCommand(IllegalArgumentException.class,
+ state, command);
}
@Test
public void validateStateTransitionNormal() {
- ComponentInstanceState componentInstanceState = new ComponentInstanceState("HBASE_MASTER", "CID_001", "AID_001");
- TestCase.assertEquals(State.INIT, componentInstanceState.getState());
+ ComponentInstanceState componentInstanceState = new ComponentInstanceState("HBASE_MASTER", containerId, "AID_001");
+ assertInState(State.INIT, componentInstanceState);
TestCase.assertEquals(true, componentInstanceState.hasPendingCommand());
TestCase.assertEquals(Command.INSTALL, componentInstanceState.getNextCommand());
- TestCase.assertEquals(State.INIT, componentInstanceState.getState());
+ assertInState(State.INIT, componentInstanceState);
componentInstanceState.commandIssued(Command.INSTALL);
- TestCase.assertEquals(State.INSTALLING, componentInstanceState.getState());
+ assertInState(State.INSTALLING, componentInstanceState);
componentInstanceState.applyCommandResult(CommandResult.IN_PROGRESS, Command.INSTALL);
- TestCase.assertEquals(State.INSTALLING, componentInstanceState.getState());
+ assertInState(State.INSTALLING, componentInstanceState);
componentInstanceState.applyCommandResult(CommandResult.COMPLETED, Command.INSTALL);
- TestCase.assertEquals(State.INSTALLED, componentInstanceState.getState());
+ assertInState(State.INSTALLED, componentInstanceState);
TestCase.assertEquals(Command.START, componentInstanceState.getNextCommand());
componentInstanceState.commandIssued(Command.START);
- TestCase.assertEquals(State.STARTING, componentInstanceState.getState());
+ assertInState(State.STARTING, componentInstanceState);
componentInstanceState.applyCommandResult(CommandResult.IN_PROGRESS, Command.START);
- TestCase.assertEquals(State.STARTING, componentInstanceState.getState());
+ assertInState(State.STARTING, componentInstanceState);
componentInstanceState.applyCommandResult(CommandResult.COMPLETED, Command.START);
- TestCase.assertEquals(State.STARTED, componentInstanceState.getState());
+ assertInState(State.STARTED, componentInstanceState);
+ }
+
+ protected void assertInState(State state,
+ ComponentInstanceState componentInstanceState) {
+ TestCase.assertEquals(state, componentInstanceState.getState());
}
@Test
public void validateStateTransitionScenario2() {
- ComponentInstanceState componentInstanceState = new ComponentInstanceState("HBASE_MASTER", "CID_001", "AID_001");
- TestCase.assertEquals(State.INIT, componentInstanceState.getState());
+ ComponentInstanceState componentInstanceState = new ComponentInstanceState("HBASE_MASTER", containerId, "AID_001");
+ assertInState(State.INIT, componentInstanceState);
TestCase.assertEquals(true, componentInstanceState.hasPendingCommand());
TestCase.assertEquals(Command.INSTALL, componentInstanceState.getNextCommand());
- TestCase.assertEquals(State.INIT, componentInstanceState.getState());
+ assertInState(State.INIT, componentInstanceState);
componentInstanceState.commandIssued(Command.INSTALL);
- TestCase.assertEquals(State.INSTALLING, componentInstanceState.getState());
+ assertInState(State.INSTALLING, componentInstanceState);
componentInstanceState.applyCommandResult(CommandResult.FAILED, Command.INSTALL);
- TestCase.assertEquals(State.INSTALL_FAILED, componentInstanceState.getState());
+ assertInState(State.INSTALL_FAILED, componentInstanceState);
componentInstanceState.commandIssued(Command.INSTALL);
- TestCase.assertEquals(State.INSTALLING, componentInstanceState.getState());
+ assertInState(State.INSTALLING, componentInstanceState);
componentInstanceState.applyCommandResult(CommandResult.COMPLETED, Command.INSTALL);
- TestCase.assertEquals(State.INSTALLED, componentInstanceState.getState());
+ assertInState(State.INSTALLED, componentInstanceState);
TestCase.assertEquals(Command.START, componentInstanceState.getNextCommand());
componentInstanceState.commandIssued(Command.START);
- TestCase.assertEquals(State.STARTING, componentInstanceState.getState());
+ assertInState(State.STARTING, componentInstanceState);
componentInstanceState.applyCommandResult(CommandResult.FAILED, Command.START);
- TestCase.assertEquals(State.INSTALLED, componentInstanceState.getState());
+ assertInState(State.INSTALLED, componentInstanceState);
componentInstanceState.commandIssued(Command.START);
componentInstanceState.applyCommandResult(CommandResult.COMPLETED, Command.START);
- TestCase.assertEquals(State.STARTED, componentInstanceState.getState());
+ assertInState(State.STARTED, componentInstanceState);
}
@Test
public void tolerateMaxFailures() {
- ComponentInstanceState componentInstanceState = new ComponentInstanceState("HBASE_MASTER", "CID_001", "AID_001");
- TestCase.assertEquals(State.INIT, componentInstanceState.getState());
+ ComponentInstanceState componentInstanceState = new ComponentInstanceState("HBASE_MASTER", containerId, "AID_001");
+ assertInState(State.INIT, componentInstanceState);
TestCase.assertEquals(true, componentInstanceState.hasPendingCommand());
TestCase.assertEquals(Command.INSTALL, componentInstanceState.getNextCommand());
- TestCase.assertEquals(State.INIT, componentInstanceState.getState());
+ assertInState(State.INIT, componentInstanceState);
componentInstanceState.commandIssued(Command.INSTALL);
- TestCase.assertEquals(State.INSTALLING, componentInstanceState.getState());
+ assertInState(State.INSTALLING, componentInstanceState);
componentInstanceState.applyCommandResult(CommandResult.FAILED, Command.INSTALL);
- TestCase.assertEquals(State.INSTALL_FAILED, componentInstanceState.getState());
+ assertInState(State.INSTALL_FAILED, componentInstanceState);
componentInstanceState.commandIssued(Command.INSTALL);
- TestCase.assertEquals(State.INSTALLING, componentInstanceState.getState());
+ assertInState(State.INSTALLING, componentInstanceState);
componentInstanceState.applyCommandResult(CommandResult.FAILED, Command.INSTALL);
- TestCase.assertEquals(State.INSTALL_FAILED, componentInstanceState.getState());
+ assertInState(State.INSTALL_FAILED, componentInstanceState);
componentInstanceState.commandIssued(Command.INSTALL);
- TestCase.assertEquals(State.INSTALLING, componentInstanceState.getState());
+ assertInState(State.INSTALLING, componentInstanceState);
componentInstanceState.applyCommandResult(CommandResult.FAILED, Command.INSTALL);
- TestCase.assertEquals(State.INSTALL_FAILED, componentInstanceState.getState());
+ assertInState(State.INSTALL_FAILED, componentInstanceState);
try {
componentInstanceState.commandIssued(Command.INSTALL);
@@ -168,41 +180,41 @@ public class TestComponentInstanceState {
@Test
public void tolerateFewFailureThenReset() {
- ComponentInstanceState componentInstanceState = new ComponentInstanceState("HBASE_MASTER", "CID_001", "AID_001");
- TestCase.assertEquals(State.INIT, componentInstanceState.getState());
+ ComponentInstanceState componentInstanceState = new ComponentInstanceState("HBASE_MASTER", containerId, "AID_001");
+ assertInState(State.INIT, componentInstanceState);
TestCase.assertEquals(true, componentInstanceState.hasPendingCommand());
TestCase.assertEquals(Command.INSTALL, componentInstanceState.getNextCommand());
- TestCase.assertEquals(State.INIT, componentInstanceState.getState());
+ assertInState(State.INIT, componentInstanceState);
componentInstanceState.commandIssued(Command.INSTALL);
- TestCase.assertEquals(State.INSTALLING, componentInstanceState.getState());
+ assertInState(State.INSTALLING, componentInstanceState);
componentInstanceState.applyCommandResult(CommandResult.FAILED, Command.INSTALL);
- TestCase.assertEquals(State.INSTALL_FAILED, componentInstanceState.getState());
+ assertInState(State.INSTALL_FAILED, componentInstanceState);
componentInstanceState.commandIssued(Command.INSTALL);
- TestCase.assertEquals(State.INSTALLING, componentInstanceState.getState());
+ assertInState(State.INSTALLING, componentInstanceState);
componentInstanceState.applyCommandResult(CommandResult.FAILED, Command.INSTALL);
- TestCase.assertEquals(State.INSTALL_FAILED, componentInstanceState.getState());
+ assertInState(State.INSTALL_FAILED, componentInstanceState);
componentInstanceState.commandIssued(Command.INSTALL);
- TestCase.assertEquals(State.INSTALLING, componentInstanceState.getState());
+ assertInState(State.INSTALLING, componentInstanceState);
componentInstanceState.applyCommandResult(CommandResult.COMPLETED, Command.INSTALL);
- TestCase.assertEquals(State.INSTALLED, componentInstanceState.getState());
+ assertInState(State.INSTALLED, componentInstanceState);
componentInstanceState.commandIssued(Command.START);
- TestCase.assertEquals(State.STARTING, componentInstanceState.getState());
+ assertInState(State.STARTING, componentInstanceState);
componentInstanceState.applyCommandResult(CommandResult.FAILED, Command.START);
- TestCase.assertEquals(State.INSTALLED, componentInstanceState.getState());
+ assertInState(State.INSTALLED, componentInstanceState);
componentInstanceState.commandIssued(Command.START);
- TestCase.assertEquals(State.STARTING, componentInstanceState.getState());
+ assertInState(State.STARTING, componentInstanceState);
componentInstanceState.applyCommandResult(CommandResult.FAILED, Command.START);
- TestCase.assertEquals(State.INSTALLED, componentInstanceState.getState());
+ assertInState(State.INSTALLED, componentInstanceState);
componentInstanceState.commandIssued(Command.START);
- TestCase.assertEquals(State.STARTING, componentInstanceState.getState());
+ assertInState(State.STARTING, componentInstanceState);
componentInstanceState.applyCommandResult(CommandResult.FAILED, Command.START);
- TestCase.assertEquals(State.INSTALLED, componentInstanceState.getState());
+ assertInState(State.INSTALLED, componentInstanceState);
try {
componentInstanceState.commandIssued(Command.START);
@@ -213,7 +225,7 @@ public class TestComponentInstanceState {
@Test
public void testBadTransitions() {
- ComponentInstanceState componentInstanceState = new ComponentInstanceState("HBASE_MASTER", "CID_001", "AID_001");
+ ComponentInstanceState componentInstanceState = new ComponentInstanceState("HBASE_MASTER", containerId, "AID_001");
try {
componentInstanceState.commandIssued(Command.START);
@@ -268,7 +280,7 @@ public class TestComponentInstanceState {
TestCase.fail("Must fail");
} catch (Exception e) {
if (!expected.isInstance(e)) {
- TestCase.fail("Unexpected exception " + e.getClass());
+ throw e;
}
}
}
@@ -280,7 +292,7 @@ public class TestComponentInstanceState {
TestCase.fail("Must fail");
} catch (Exception e) {
if (!expected.isInstance(e)) {
- TestCase.fail("Unexpected exception " + e.getClass());
+ throw e;
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/32fd3f56/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
index c2cfafd..b16b145 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestHeartbeatMonitor.java
@@ -16,6 +16,9 @@
*/
package org.apache.slider.providers.agent;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.slider.server.appmaster.AMViewForProviders;
+import org.apache.slider.server.appmaster.model.mock.MockContainerId;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
@@ -55,9 +58,11 @@ public class TestHeartbeatMonitor {
HeartbeatMonitor hbm = new HeartbeatMonitor(provider, 500);
Assert.assertFalse(hbm.isAlive());
Map<String, ComponentInstanceState> statuses = new HashMap<>();
- ComponentInstanceState state = new ComponentInstanceState("HBASE_MASTER", "Cid", "Aid");
+ ContainerId container1 = new MockContainerId(1);
+ ComponentInstanceState state = new ComponentInstanceState("HBASE_MASTER",
+ container1, "Aid");
state.setState(State.STARTED);
- state.setLastHeartbeat(System.currentTimeMillis());
+ state.heartbeat(System.currentTimeMillis());
statuses.put("label_1", state);
expect(provider.getComponentStatuses()).andReturn(statuses).anyTimes();
replay(provider);
@@ -72,65 +77,100 @@ public class TestHeartbeatMonitor {
@Test
public void testHeartbeatMonitorWithUnhealthyAndThenLost() throws Exception {
AgentProviderService provider = createNiceMock(AgentProviderService.class);
- HeartbeatMonitor hbm = new HeartbeatMonitor(provider, 2 * 1000);
- Assert.assertFalse(hbm.isAlive());
+ long now = 100000;
+ int wakeupInterval = 2 * 1000;
+
Map<String, ComponentInstanceState> statuses = new HashMap<>();
- ComponentInstanceState masterState = new ComponentInstanceState("HBASE_MASTER", "Cid1", "Aid1");
- masterState.setState(State.STARTED);
- masterState.setLastHeartbeat(System.currentTimeMillis());
- statuses.put("Aid1_Cid1_HBASE_MASTER", masterState);
+ ContainerId masterContainer = new MockContainerId(1);
+ ContainerId slaveContainer = new MockContainerId(2);
+ ComponentInstanceState masterState = new ComponentInstanceState("HBASE_MASTER",
+ masterContainer, "Aid1");
+ String masterLabel = "Aid1_Cid1_HBASE_MASTER";
+ statuses.put(masterLabel, masterState);
+
+ ComponentInstanceState slaveState = new ComponentInstanceState("HBASE_REGIONSERVER",
+ slaveContainer, "Aid1");
+ String slaveLabel = "Aid1_Cid2_HBASE_REGIONSERVER";
+ statuses.put(slaveLabel, slaveState);
- ComponentInstanceState slaveState = new ComponentInstanceState("HBASE_REGIONSERVER", "Cid2", "Aid1");
+ masterState.setState(State.STARTED);
+ masterState.heartbeat(now);
slaveState.setState(State.STARTED);
- slaveState.setLastHeartbeat(System.currentTimeMillis());
- statuses.put("Aid1_Cid2_HBASE_REGIONSERVER", slaveState);
-
+ slaveState.heartbeat(now);
expect(provider.getComponentStatuses()).andReturn(statuses).anyTimes();
- expect(provider.releaseContainer("Aid1_Cid2_HBASE_REGIONSERVER")).andReturn(true).once();
+ expect(provider.lostContainer(slaveLabel, slaveContainer)).andReturn(
+ AMViewForProviders.ContainerLossReportOutcomes.CONTAINER_LOST_REVIEW_INITIATED).once();
+ // expect the second iteration to report no container any more
+ expect(provider.lostContainer(slaveLabel, slaveContainer)).andReturn(
+ AMViewForProviders.ContainerLossReportOutcomes.CONTAINER_NOT_IN_USE).once();
replay(provider);
- hbm.start();
- Thread.sleep(1 * 1000);
+
+ HeartbeatMonitor heartbeatMonitor = new HeartbeatMonitor(provider,
+ wakeupInterval);
+ Assert.assertFalse(heartbeatMonitor.isAlive());
+ now += wakeupInterval;
+ masterState.setState(State.STARTED);
+ masterState.heartbeat(now);
+
+ slaveState.setState(State.STARTED);
// just dial back by at least 2 sec but no more than 4
- slaveState.setLastHeartbeat(System.currentTimeMillis() - (2 * 1000 + 100));
- masterState.setLastHeartbeat(System.currentTimeMillis());
+ slaveState.heartbeat(now - (wakeupInterval + 100));
- Thread.sleep(1 * 1000 + 500);
- masterState.setLastHeartbeat(System.currentTimeMillis());
- log.info("Slave container state {}", slaveState.getContainerState());
- Assert.assertEquals(ContainerState.HEALTHY, masterState.getContainerState());
- Assert.assertEquals(ContainerState.UNHEALTHY, slaveState.getContainerState());
+ assertInState(ContainerState.HEALTHY, masterState, now);
+ assertInState(ContainerState.HEALTHY, slaveState, now);
+
+ //tick #1
+ heartbeatMonitor.doWork(now);
- Thread.sleep(1 * 1000);
- // some lost heartbeats are ignored (e.g. ~ 1 sec)
- masterState.setLastHeartbeat(System.currentTimeMillis() - 1 * 1000);
+ assertInState(ContainerState.HEALTHY, masterState, now);
+ assertInState(ContainerState.UNHEALTHY, slaveState, now);
- Thread.sleep(1 * 1000 + 500);
+ // heartbeat from the master
+ masterState.heartbeat(now + 1500);
- log.info("Slave container state {}", slaveState.getContainerState());
- Assert.assertEquals(ContainerState.HEALTHY, masterState.getContainerState());
- Assert.assertEquals(ContainerState.HEARTBEAT_LOST, slaveState.getContainerState());
- hbm.shutdown();
+ // tick #2
+ now += wakeupInterval;
+ heartbeatMonitor.doWork(now);
+
+ assertInState(ContainerState.HEALTHY, masterState, now);
+ assertInState(ContainerState.HEARTBEAT_LOST, slaveState, now);
+ }
+
+ protected void assertInState(ContainerState expectedState,
+ ComponentInstanceState componentInstanceState, long now) {
+ ContainerState actualState = componentInstanceState.getContainerState();
+ if (!expectedState.equals(actualState)) {
+ // mismatch
+ Assert.fail(String.format("at [%06d] Expected component state %s " +
+ "but found state %s in in component %s",
+ now, expectedState, actualState, componentInstanceState));
+ }
}
@Test
public void testHeartbeatTransitions() {
- ComponentInstanceState slaveState = new ComponentInstanceState("HBASE_REGIONSERVER", "Cid2", "Aid1");
+ ContainerId container2 = new MockContainerId(2);
+ ComponentInstanceState slaveState = new ComponentInstanceState("HBASE_REGIONSERVER",
+ container2, "Aid1");
slaveState.setState(State.STARTED);
- Assert.assertEquals(ContainerState.INIT, slaveState.getContainerState());
- slaveState.setLastHeartbeat(System.currentTimeMillis());
- Assert.assertEquals(ContainerState.HEALTHY, slaveState.getContainerState());
+ long lastHeartbeat = System.currentTimeMillis();
+ assertInState(ContainerState.INIT, slaveState, 0);
+ slaveState.heartbeat(lastHeartbeat);
+ assertInState(ContainerState.HEALTHY, slaveState, lastHeartbeat);
slaveState.setContainerState(ContainerState.UNHEALTHY);
- Assert.assertEquals(ContainerState.UNHEALTHY, slaveState.getContainerState());
- slaveState.setLastHeartbeat(System.currentTimeMillis());
- Assert.assertEquals(ContainerState.HEALTHY, slaveState.getContainerState());
+ lastHeartbeat = System.currentTimeMillis();
+ assertInState(ContainerState.UNHEALTHY, slaveState, lastHeartbeat);
+ slaveState.heartbeat(lastHeartbeat);
+ assertInState(ContainerState.HEALTHY, slaveState, lastHeartbeat);
slaveState.setContainerState(ContainerState.HEARTBEAT_LOST);
- Assert.assertEquals(ContainerState.HEARTBEAT_LOST, slaveState.getContainerState());
- slaveState.setLastHeartbeat(System.currentTimeMillis());
- Assert.assertEquals(ContainerState.HEARTBEAT_LOST, slaveState.getContainerState());
+ assertInState(ContainerState.HEARTBEAT_LOST, slaveState, lastHeartbeat);
+ lastHeartbeat = System.currentTimeMillis();
+ slaveState.heartbeat(lastHeartbeat);
+ assertInState(ContainerState.HEARTBEAT_LOST, slaveState, lastHeartbeat);
}
}
[32/50] [abbrv] git commit: SLIDER-286 AM restart count is now set to
0 in AppMasterLauncher/Slider client,
so AMs get default cluster value. The value is logged at debug level
Posted by st...@apache.org.
SLIDER-286 AM restart count is now set to 0 in AppMasterLauncher/Slider client, so AMs get default cluster value. The value is logged at debug level
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1350491f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1350491f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1350491f
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 1350491fd487987b458da75dc571c07c1ae76770
Parents: 56083ec
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 7 17:36:45 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 7 17:36:45 2014 +0100
----------------------------------------------------------------------
.../src/main/java/org/apache/slider/client/SliderClient.java | 4 ++--
.../java/org/apache/slider/core/launch/AppMasterLauncher.java | 7 +++++--
2 files changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1350491f/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index ff2e865..e1d083c 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -833,8 +833,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// set the application name;
amLauncher.setKeepContainersOverRestarts(true);
- amLauncher.setMaxAppAttempts(config.getInt(KEY_AM_RESTART_LIMIT,
- DEFAULT_AM_RESTART_LIMIT));
+ int maxAppAttempts = config.getInt(KEY_AM_RESTART_LIMIT, 0);
+ amLauncher.setMaxAppAttempts(maxAppAttempts);
sliderFileSystem.purgeAppInstanceTempFiles(clustername);
Path tempPath = sliderFileSystem.createAppInstanceTempPath(
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1350491f/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
index dc4a886..bd8a0a5 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
@@ -50,7 +50,7 @@ public class AppMasterLauncher extends AbstractLauncher {
private final ApplicationSubmissionContext submissionContext;
private final ApplicationId appId;
private final boolean secureCluster;
- private int maxAppAttempts = 2;
+ private int maxAppAttempts = 0;
private boolean keepContainersOverRestarts = true;
private String queue = YarnConfiguration.DEFAULT_QUEUE_NAME;
private int priority = 1;
@@ -174,7 +174,10 @@ public class AppMasterLauncher extends AbstractLauncher {
submissionContext.setKeepContainersAcrossApplicationAttempts(true);
}
- submissionContext.setMaxAppAttempts(maxAppAttempts);
+ if (maxAppAttempts > 0) {
+ log.debug("Setting max AM attempts to {}", maxAppAttempts);
+ submissionContext.setMaxAppAttempts(maxAppAttempts);
+ }
if (secureCluster) {
addSecurityTokens();
[48/50] [abbrv] git commit: SLIDER-302 rm StartContainerAction
Posted by st...@apache.org.
SLIDER-302 rm StartContainerAction
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/323a80f8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/323a80f8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/323a80f8
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 323a80f8f92c9d3b2daba89189aa503b6ec20093
Parents: e464bba
Author: Steve Loughran <st...@apache.org>
Authored: Sat Aug 9 12:37:33 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Sat Aug 9 12:37:47 2014 +0100
----------------------------------------------------------------------
.../appmaster/actions/StartContainerAction.java | 41 --------------------
1 file changed, 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/323a80f8/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/StartContainerAction.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/StartContainerAction.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/StartContainerAction.java
deleted file mode 100644
index 95ecbe8..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/StartContainerAction.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-
-/**
- * Start a container
- */
-public class StartContainerAction {
-
- private final Container container;
- private final ContainerLaunchContext ctx;
- private final RoleInstance instance;
-
- public StartContainerAction(Container container,
- ContainerLaunchContext ctx,
- RoleInstance instance) {
- this.container = container;
- this.ctx = ctx;
- this.instance = instance;
- }
-}
[09/50] [abbrv] git commit: SLIDER-270 assertion checking has found
the location of the problem
Posted by st...@apache.org.
SLIDER-270 assertion checking has found the location of the problem
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/df963f24
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/df963f24
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/df963f24
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: df963f24b474f33d00719f91d8b50e83a8f55b6f
Parents: 291a82e
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 5 21:35:28 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 5 21:35:54 2014 +0100
----------------------------------------------------------------------
.../slider/server/appmaster/state/AppState.java | 25 ++++++++++++++------
.../server/appmaster/state/RoleHistory.java | 6 ++---
.../TestRoleHistoryContainerEvents.groovy | 2 +-
.../src/test/resources/log4j.properties | 1 +
4 files changed, 23 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/df963f24/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 74e91a0..2b17c3a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1172,7 +1172,7 @@ public class AppState {
RoleInstance roleInstance;
if (containersBeingReleased.containsKey(containerId)) {
- log.info("Container was queued for release");
+ log.info("Container was queued for release : {}", containerId);
Container container = containersBeingReleased.remove(containerId);
RoleStatus roleStatus = lookupRoleStatus(container);
int releasing = roleStatus.decReleasing();
@@ -1183,7 +1183,7 @@ public class AppState {
actual,
releasing,
completedCount);
- roleHistory.onReleaseCompleted(container);
+ roleHistory.onReleaseCompleted(container, true);
} else if (surplusNodes.remove(containerId)) {
//its a surplus one being purged
@@ -1251,18 +1251,28 @@ public class AppState {
//record the complete node's details; this pulls it from the livenode set
//remove the node
ContainerId id = status.getContainerId();
+ log.info("Removing node ID {}", id);
RoleInstance node = getLiveNodes().remove(id);
- if (node == null) {
- log.warn("Received notification of completion of unknown node {}", id);
- completionOfNodeNotInLiveListEvent.incrementAndGet();
-
- } else {
+ if (node != null) {
node.state = ClusterDescription.STATE_DESTROYED;
node.exitCode = status.getExitStatus();
node.diagnostics = status.getDiagnostics();
getCompletedNodes().put(id, node);
result.roleInstance = node;
+ } else {
+ // not in the list
+ log.warn("Received notification of completion of unknown node {}", id);
+ completionOfNodeNotInLiveListEvent.incrementAndGet();
+
}
+ // finally, verify the node doesn't exist any more
+ assert !containersBeingReleased.containsKey(
+ containerId) : "container in release queue";
+ assert !getLiveNodes().containsKey(
+ containerId) : " container in live nodes";
+ assert getActiveContainer(containerId) ==
+ null : "Container still in active container list";
+
return result;
}
@@ -1422,6 +1432,7 @@ public class AppState {
* @throws SliderInternalStateException if the operation reveals that
* the internal state of the application is inconsistent.
*/
+ @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
public List<AbstractRMOperation> reviewOneRole(RoleStatus role)
throws SliderInternalStateException, TriggerClusterTeardownException {
List<AbstractRMOperation> operations = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/df963f24/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 0cd2b39..de7e9d8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -29,7 +29,6 @@ import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.server.avro.RoleHistoryHeader;
import org.apache.slider.server.avro.RoleHistoryWriter;
-import org.mortbay.log.Log;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -659,10 +658,11 @@ public class RoleHistory {
/**
* App state notified of a container completed
* @param container completed container
+ * @param wasReleased
* @return true if the node was queued
*/
- public boolean onReleaseCompleted(Container container) {
- return markContainerFinished(container, true, false);
+ public boolean onReleaseCompleted(Container container, boolean wasReleased) {
+ return markContainerFinished(container, wasReleased, false);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/df963f24/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
index 795b48f..1a71aed 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
@@ -158,7 +158,7 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
assert roleEntry.active == 0
// release completed
- roleHistory.onReleaseCompleted(container)
+ roleHistory.onReleaseCompleted(container, true)
assert roleEntry.releasing == 0
assert roleEntry.live == 0
assert roleEntry.active == 0
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/df963f24/slider-providers/hbase/slider-hbase-provider/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/resources/log4j.properties b/slider-providers/hbase/slider-hbase-provider/src/test/resources/log4j.properties
index a552a55..1cc355a 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/resources/log4j.properties
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/resources/log4j.properties
@@ -57,3 +57,4 @@ log4j.logger.org.apache.hadoop.security.token.delegation=WARN
log4j.logger.org.apache.hadoop.yarn.util.AbstractLivelinessMonitor=WARN
log4j.logger.org.apache.hadoop.yarn.server.nodemanager.security=WARN
log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMNMInfo=WARN
+log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager=WARN
\ No newline at end of file