You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/05/28 04:21:22 UTC
[17/44] git commit: Misc issues,
support for explicit publisher component, add hostname to heartbeat,
publish log folders
Misc issues, support for explicit publisher component, add hostname to heartbeat, publish log folders
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4fd70e0a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4fd70e0a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4fd70e0a
Branch: refs/heads/master
Commit: 4fd70e0af24636d8d30d7d4614fe5e573663232e
Parents: ab246e2
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu May 22 17:24:06 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu May 22 21:31:18 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo-v1_5/metainfo.xml | 1 +
app-packages/hbase-v0_96/README.txt | 2 +-
app-packages/hbase-v0_96/appConfig.json | 2 +-
app-packages/storm-v0_91/README.txt | 16 ++++
app-packages/storm-v0_91/appConfig.json | 16 ++--
app-packages/storm-v0_91/metainfo.xml | 21 ++++++
.../storm-v0_91/package/scripts/params.py | 4 +-
.../src/main/python/agent/ActionQueue.py | 2 +
slider-agent/src/main/python/agent/Constants.py | 3 +
.../python/agent/CustomServiceOrchestrator.py | 7 ++
slider-agent/src/main/python/agent/Heartbeat.py | 4 +-
.../src/test/python/agent/TestActionQueue.py | 3 +-
.../src/test/python/agent/TestHeartbeat.py | 5 +-
.../providers/agent/AgentProviderService.java | 69 ++++++++++++++++-
.../agent/application/metadata/Component.java | 10 +++
.../application/metadata/MetainfoParser.java | 1 +
.../appmaster/web/rest/agent/CommandReport.java | 17 ++++-
.../appmaster/web/rest/agent/HeartBeat.java | 17 ++++-
.../agent/TestAgentProviderService.java | 78 ++++++++++++++++++++
19 files changed, 255 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fd70e0a/app-packages/accumulo-v1_5/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/accumulo-v1_5/metainfo.xml b/app-packages/accumulo-v1_5/metainfo.xml
index 09ab9c4..0224b92 100644
--- a/app-packages/accumulo-v1_5/metainfo.xml
+++ b/app-packages/accumulo-v1_5/metainfo.xml
@@ -73,6 +73,7 @@
<component>
<name>ACCUMULO_MONITOR</name>
<category>MASTER</category>
+ <publishConfig>true</publishConfig>
<commandScript>
<script>scripts/accumulo_monitor.py</script>
<scriptType>PYTHON</scriptType>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fd70e0a/app-packages/hbase-v0_96/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/README.txt b/app-packages/hbase-v0_96/README.txt
index d7d8ff7..4fc089d 100644
--- a/app-packages/hbase-v0_96/README.txt
+++ b/app-packages/hbase-v0_96/README.txt
@@ -1,6 +1,6 @@
How to create a Slider package?
-Replace the placeholder tarball for Accumulo.
+Replace the placeholder tarball for HBase.
cp ~/Downloads/hbase-0.96.1-hadoop2-bin.tar.gz package/files/
rm package/files/hbase-0.96.1-hadoop2-bin.tar.gz.REPLACE
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fd70e0a/app-packages/hbase-v0_96/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/appConfig.json b/app-packages/hbase-v0_96/appConfig.json
index e25adbb..fd884cb 100644
--- a/app-packages/hbase-v0_96/appConfig.json
+++ b/app-packages/hbase-v0_96/appConfig.json
@@ -18,7 +18,7 @@
"site.global.user_group": "hadoop",
"site.global.security_enabled": "false",
"site.global.ganglia_server_host": "${NN_HOST}",
- "site.global.ganglia_server_port": "8663",
+ "site.global.ganglia_server_port": "8667",
"site.global.ganglia_server_id": "Application1",
"site.hbase-site.hbase.hstore.flush.retries.number": "120",
"site.hbase-site.hbase.client.keyvalue.maxsize": "10485760",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fd70e0a/app-packages/storm-v0_91/README.txt
----------------------------------------------------------------------
diff --git a/app-packages/storm-v0_91/README.txt b/app-packages/storm-v0_91/README.txt
new file mode 100644
index 0000000..7cc6117
--- /dev/null
+++ b/app-packages/storm-v0_91/README.txt
@@ -0,0 +1,16 @@
+How to create a Slider package?
+
+Replace the placeholder tarball for Storm.
+ cp ~/Downloads/apache-storm-0.9.1.2.1.1.0-237.tar.gz package/files/
+ rm package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
+
+Create a zip package at the root of the package (<slider enlistment>/app-packages/storm-v0_91/)
+ zip -r storm_v091.zip .
+
+Verify the content using
+ unzip -l "$@" storm_v091.zip
+
+While appConfig.json and resources.json are not required for the package they work
+well as the default configuration for Slider apps. So its advisable that when you
+create an application package for Slider, include sample/default resources.json and
+appConfig.json for a minimal Yarn cluster.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fd70e0a/app-packages/storm-v0_91/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/storm-v0_91/appConfig.json b/app-packages/storm-v0_91/appConfig.json
index 7bdfd93..4251dac 100644
--- a/app-packages/storm-v0_91/appConfig.json
+++ b/app-packages/storm-v0_91/appConfig.json
@@ -12,6 +12,10 @@
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237",
"site.global.user_group": "hadoop",
"site.global.security_enabled": "false",
+ "site.global.ganglia_server_host": "${NN_HOST}",
+ "site.global.ganglia_server_id": "Application2",
+ "site.global.rest_api_port": "${STORM_REST_API.ALLOCATED_PORT}",
+ "site.global.rest_api_admin_port": "${STORM_REST_API.ALLOCATED_PORT}",
"site.storm-site.topology.tuple.serializer": "backtype.storm.serialization.types.ListDelegateSerializer",
"site.storm-site.topology.workers": "1",
"site.storm-site.drpc.worker.threads": "64",
@@ -38,8 +42,8 @@
"site.storm-site.storm.local.mode.zmq": "false",
"site.storm-site.topology.max.task.parallelism": "null",
"site.storm-site.storm.zookeeper.port": "2181",
- "site.storm-site.nimbus.childopts": "-Xmx1024m -Djava.security.auth.login.config=/etc/storm/storm_jaas.conf -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host={0},port=8649,wireformat31x=true,mode=multicast,config=/tmp/container002/work/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
- "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host={0},port=8650,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
+ "site.storm-site.nimbus.childopts": "-Xmx1024m -Djava.security.auth.login.config=/etc/storm/storm_jaas.conf -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host={0},port=8669,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Nimbus_JVM",
+ "site.storm-site.worker.childopts": "-Xmx768m -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host={0},port=8669,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Worker_%ID%_JVM",
"site.storm-site.drpc.queue.size": "128",
"site.storm-site.storm.zookeeper.retry.times": "5",
"site.storm-site.nimbus.monitor.freq.secs": "10",
@@ -62,7 +66,7 @@
"site.storm-site.logviewer.appender.name": "A1",
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
- "site.storm-site.supervisor.slots.ports": "[6700, 6701]",
+ "site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}, ${SUPERVISOR.ALLOCATED_PORT}]",
"site.storm-site.nimbus.file.copy.expiration.secs": "600",
"site.storm-site.supervisor.monitor.frequency.secs": "3",
"site.storm-site.transactional.zookeeper.servers": "null",
@@ -77,14 +81,14 @@
"site.storm-site.topology.executor.receive.buffer.size": "1024",
"site.storm-site.topology.stats.sample.rate": "0.05",
"site.storm-site.topology.fall.back.on.java.serialization": "true",
- "site.storm-site.supervisor.childopts": "-Xmx256m -Djava.security.auth.login.config=/etc/storm/storm_jaas.conf -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=56431 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host={0},port=8650,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "site.storm-site.supervisor.childopts": "-Xmx256m -Djava.security.auth.login.config=/etc/storm/storm_jaas.conf -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=${SUPERVISOR.ALLOCATED_PORT} -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host={0},port=8669,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
"site.storm-site.topology.enable.message.timeouts": "true",
"site.storm-site.storm.messaging.netty.max_wait_ms": "1000",
"site.storm-site.nimbus.topology.validator": "backtype.storm.nimbus.DefaultTopologyValidator",
"site.storm-site.nimbus.supervisor.timeout.secs": "60",
"site.storm-site.topology.disruptor.wait.strategy": "com.lmax.disruptor.BlockingWaitStrategy",
"site.storm-site.nimbus.inbox.jar.expiration.secs": "3600",
- "site.storm-site.drpc.port": "3772",
+ "site.storm-site.drpc.port": "${DRPC_SERVER.ALLOCATED_PORT}",
"site.storm-site.topology.kryo.factory": "backtype.storm.serialization.DefaultKryoFactory",
"site.storm-site.storm.zookeeper.retry.interval": "1000",
"site.storm-site.storm.messaging.netty.max_retries": "30",
@@ -99,7 +103,7 @@
"site.storm-site.topology.trident.batch.emit.interval.millis": "500",
"site.storm-site.topology.builtin.metrics.bucket.size.secs": "60",
"site.storm-site.storm.thrift.transport": "backtype.storm.security.auth.SimpleTransportPlugin",
- "site.storm-site.logviewer.port": "8000",
+ "site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}",
"site.storm-site.topology.debug": "false"
},
"components": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fd70e0a/app-packages/storm-v0_91/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm-v0_91/metainfo.xml b/app-packages/storm-v0_91/metainfo.xml
index 5c77eb5..31950ab 100644
--- a/app-packages/storm-v0_91/metainfo.xml
+++ b/app-packages/storm-v0_91/metainfo.xml
@@ -24,6 +24,26 @@
<comment>Apache Hadoop Stream processing framework</comment>
<version>0.9.1.2.1</version>
+ <exportGroups>
+ <exportGroup>
+ <name>QuickLinks</name>
+ <exports>
+ <export>
+ <name>org.apache.slider.jmx</name>
+ <value>http://${STORM_REST_API_HOST}:${site.global.rest_api_port}/api/cluster/summary</value>
+ </export>
+ <export>
+ <name>org.apache.slider.monitor</name>
+ <value>http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}</value>
+ </export>
+ <export>
+ <name>org.apache.slider.metrics</name>
+ <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value>
+ </export>
+ </exports>
+ </exportGroup>
+ </exportGroups>
+
<commandOrders>
<commandOrder>
<command>SUPERVISOR-START</command>
@@ -78,6 +98,7 @@
<component>
<name>STORM_UI_SERVER</name>
<category>MASTER</category>
+ <publishConfig>true</publishConfig>
<commandScript>
<script>scripts/ui_server.py</script>
<scriptType>PYTHON</scriptType>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fd70e0a/app-packages/storm-v0_91/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/storm-v0_91/package/scripts/params.py b/app-packages/storm-v0_91/package/scripts/params.py
index 784069d..23e9112 100644
--- a/app-packages/storm-v0_91/package/scripts/params.py
+++ b/app-packages/storm-v0_91/package/scripts/params.py
@@ -35,8 +35,8 @@ java64_home = config['hostLevelParams']['java_home']
nimbus_host = config['configurations']['storm-site']['nimbus.host']
nimbus_port = config['configurations']['storm-site']['nimbus.thrift.port']
nimbus_host = config['configurations']['storm-site']['nimbus.host']
-rest_api_port = "8745"
-rest_api_admin_port = "8746"
+rest_api_port = config['configurations']['global']['rest_api_port']
+rest_api_admin_port = config['configurations']['global']['rest_api_admin_port']
rest_api_conf_file = format("{conf_dir}/config.yaml")
rest_lib_dir = format("{app_root}/contrib/storm-rest")
storm_bin = format("{app_root}/bin/storm")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fd70e0a/slider-agent/src/main/python/agent/ActionQueue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/ActionQueue.py b/slider-agent/src/main/python/agent/ActionQueue.py
index b37c94b..d4d8bc2 100644
--- a/slider-agent/src/main/python/agent/ActionQueue.py
+++ b/slider-agent/src/main/python/agent/ActionQueue.py
@@ -168,6 +168,8 @@ class ActionQueue(threading.Thread):
roleResult['configurationTags'] = command['configurationTags']
if Constants.ALLOCATED_PORTS in commandresult:
roleResult['allocatedPorts'] = commandresult[Constants.ALLOCATED_PORTS]
+ if Constants.FOLDERS in commandresult:
+ roleResult['folders'] = commandresult[Constants.FOLDERS]
self.commandStatuses.put_command_status(command, roleResult)
# Store action result to agent response queue
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fd70e0a/slider-agent/src/main/python/agent/Constants.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Constants.py b/slider-agent/src/main/python/agent/Constants.py
index b49bda3..b937cd2 100644
--- a/slider-agent/src/main/python/agent/Constants.py
+++ b/slider-agent/src/main/python/agent/Constants.py
@@ -24,3 +24,6 @@ Constants used by Slider Agent
EXIT_CODE = "exitcode"
ALLOCATED_PORTS = "allocated_ports"
+FOLDERS = "folders"
+AGENT_WORK_ROOT = "AGENT_WORK_ROOT"
+AGENT_LOG_ROOT = "AGENT_LOG_ROOT"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fd70e0a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 20e55df..328084d 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -125,6 +125,13 @@ class CustomServiceOrchestrator():
if Constants.EXIT_CODE in ret and ret[Constants.EXIT_CODE] == 0:
ret[Constants.ALLOCATED_PORTS] = allocated_port
+
+ # Irrespective of the outcome report the folder paths
+ if command_name == 'INSTALL':
+ ret[Constants.FOLDERS] = {
+ Constants.AGENT_LOG_ROOT : self.config.getLogPath(),
+ Constants.AGENT_WORK_ROOT : self.config.getWorkRootPath()
+ }
return ret
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fd70e0a/slider-agent/src/main/python/agent/Heartbeat.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Heartbeat.py b/slider-agent/src/main/python/agent/Heartbeat.py
index dd1e8ab..8192348 100644
--- a/slider-agent/src/main/python/agent/Heartbeat.py
+++ b/slider-agent/src/main/python/agent/Heartbeat.py
@@ -22,6 +22,7 @@ import json
import logging
import time
from pprint import pformat
+import hostname
from ActionQueue import ActionQueue
import AgentConfig
@@ -46,7 +47,8 @@ class Heartbeat:
heartbeat = {'responseId': int(id),
'timestamp': timestamp,
'hostname': self.config.getLabel(),
- 'nodeStatus': nodeStatus
+ 'nodeStatus': nodeStatus,
+ 'fqdn': hostname.public_hostname()
}
commandsInProgress = False
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fd70e0a/slider-agent/src/test/python/agent/TestActionQueue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestActionQueue.py b/slider-agent/src/test/python/agent/TestActionQueue.py
index 3dad2de..2e1e4cf 100644
--- a/slider-agent/src/test/python/agent/TestActionQueue.py
+++ b/slider-agent/src/test/python/agent/TestActionQueue.py
@@ -363,7 +363,8 @@ class TestActionQueue(TestCase):
'taskId': 3,
'structuredOut': '',
'exitcode': 0,
- 'allocatedPorts': {}}
+ 'allocatedPorts': {},
+ 'folders': {'AGENT_LOG_ROOT': tempdir, 'AGENT_WORK_ROOT': tempdir}}
self.assertEqual(len(report['reports']), 1)
self.assertEqual(report['reports'][0], expected)
self.assertTrue(os.path.isfile(configname))
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fd70e0a/slider-agent/src/test/python/agent/TestHeartbeat.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestHeartbeat.py b/slider-agent/src/test/python/agent/TestHeartbeat.py
index 9ce8a80..b60c14c 100644
--- a/slider-agent/src/test/python/agent/TestHeartbeat.py
+++ b/slider-agent/src/test/python/agent/TestHeartbeat.py
@@ -64,7 +64,7 @@ class TestHeartbeat(TestCase):
self.assertEquals(result['nodeStatus']['cause'], "NONE")
self.assertEquals(result['nodeStatus']['status'], "HEALTHY")
# result may or may NOT have an agentEnv structure in it
- self.assertEquals((len(result) is 4) or (len(result) is 5), True)
+ self.assertEquals((len(result) is 5) or (len(result) is 6), True)
self.assertEquals(not heartbeat.reports, True,
"Heartbeat should not contain task in progress")
@@ -132,10 +132,11 @@ class TestHeartbeat(TestCase):
hb = heartbeat.build({}, 10)
hb['hostname'] = 'hostname'
hb['timestamp'] = 'timestamp'
+ hb['fqdn'] = 'fqdn'
expected = {'nodeStatus':
{'status': 'HEALTHY',
'cause': 'NONE'},
- 'timestamp': 'timestamp', 'hostname': 'hostname',
+ 'timestamp': 'timestamp', 'hostname': 'hostname', 'fqdn': 'fqdn',
'responseId': 10, 'reports': [
{'status': 'IN_PROGRESS', 'roleCommand': u'INSTALL',
'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1',
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fd70e0a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index f4eec25..6d3d0e1 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -78,8 +78,10 @@ import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -102,6 +104,8 @@ public class AgentProviderService extends AbstractProviderService implements
private static final String LABEL_MAKER = "___";
private static final String CONTAINER_ID = "container_id";
private static final String GLOBAL_CONFIG_TAG = "global";
+ private static final String LOG_FOLDERS_TAG = "LogFolders";
+ private static final int MAX_LOG_ENTRIES = 20;
private final Object syncLock = new Object();
private final Map<String, String> allocatedPorts = new ConcurrentHashMap<>();
private AgentClientProvider clientProvider;
@@ -109,6 +113,13 @@ public class AgentProviderService extends AbstractProviderService implements
private AtomicInteger taskId = new AtomicInteger(0);
private volatile Metainfo metainfo = null;
private ComponentCommandOrder commandOrder = null;
+ private Map<String, String> workFolders =
+ Collections.synchronizedMap(new LinkedHashMap<String, String>(MAX_LOG_ENTRIES, 0.75f, false) {
+ protected boolean removeEldestEntry(Map.Entry eldest) {
+ return size() > MAX_LOG_ENTRIES;
+ }
+ });
+ private Boolean canAnyMasterPublish = null;
public AgentProviderService() {
super("AgentProviderService");
@@ -364,6 +375,7 @@ public class AgentProviderService extends AbstractProviderService implements
String label = heartBeat.getHostname();
String roleName = getRoleName(label);
+
String containerId = getContainerId(label);
StateAccessForProviders accessor = getAmState();
String scriptPath = getScriptPathFromMetainfo(roleName);
@@ -379,8 +391,10 @@ public class AgentProviderService extends AbstractProviderService implements
Boolean isMaster = isMaster(roleName);
ComponentInstanceState componentStatus = componentStatuses.get(label);
- // TODO: Currently only process configurations from Master
- if (isMaster) {
+ // If no Master can explicitly publish then publish if its a master
+ // Otherwise, wait till the master that can publish is ready
+ if (isMaster &&
+ (canAnyMasterPublishConfig() == false || canPublishConfig(roleName))) {
processReturnedStatus(heartBeat, componentStatus);
}
@@ -398,6 +412,10 @@ public class AgentProviderService extends AbstractProviderService implements
Command command = getCommand(report.getRoleCommand());
componentStatus.applyCommandResult(result, command);
log.info("Component operation. Status: {}", result);
+
+ if (command == Command.INSTALL && report.getFolders() != null && report.getFolders().size() > 0) {
+ processFolderPaths(report.getFolders(), containerId, heartBeat.getFqdn());
+ }
}
int waitForCount = accessor.getInstanceDefinitionSnapshot().
@@ -443,6 +461,14 @@ public class AgentProviderService extends AbstractProviderService implements
return response;
}
+ private void processFolderPaths(Map<String, String> folders, String containerId, String hostFqdn) {
+ for(String key : folders.keySet()) {
+ workFolders.put(String.format("%s-%s-%s", hostFqdn, containerId, key), folders.get(key));
+ }
+
+ publishComponentConfiguration(LOG_FOLDERS_TAG, LOG_FOLDERS_TAG, (new HashMap<>(this.workFolders)).entrySet());
+ }
+
protected void processReturnedStatus(HeartBeat heartBeat, ComponentInstanceState componentStatus) {
List<ComponentStatus> statuses = heartBeat.getComponentStatus();
if (statuses != null && !statuses.isEmpty()) {
@@ -519,7 +545,7 @@ public class AgentProviderService extends AbstractProviderService implements
return scriptPath;
}
- protected Boolean isMaster(String roleName) {
+ protected boolean isMaster(String roleName) {
List<Service> services = getMetainfo().getServices();
if (services.size() != 1) {
log.error("Malformed app definition: Expect only one service in the metainfo.xml");
@@ -538,6 +564,43 @@ public class AgentProviderService extends AbstractProviderService implements
return false;
}
+ protected boolean canPublishConfig(String roleName) {
+ List<Service> services = getMetainfo().getServices();
+ if (services.size() != 1) {
+ log.error("Malformed app definition: Expect only one service in the metainfo.xml");
+ } else {
+ Service service = services.get(0);
+ for (Component component : service.getComponents()) {
+ if (component.getName().equals(roleName)) {
+ return Boolean.TRUE.toString().equals(component.getPublishConfig());
+ }
+ }
+ }
+ return false;
+ }
+
+ protected boolean canAnyMasterPublishConfig() {
+ if (canAnyMasterPublish == null) {
+ List<Service> services = getMetainfo().getServices();
+ if (services.size() != 1) {
+ log.error("Malformed app definition: Expect only one service in the metainfo.xml");
+ } else {
+ Service service = services.get(0);
+ for (Component component : service.getComponents()) {
+ if (Boolean.TRUE.toString().equals(component.getPublishConfig()) &&
+ component.getCategory().equals("MASTER")) {
+ canAnyMasterPublish = true;
+ }
+ }
+ }
+ }
+
+ if (canAnyMasterPublish == null) {
+ canAnyMasterPublish = false;
+ }
+ return canAnyMasterPublish;
+ }
+
private String getRoleName(String label) {
return label.substring(label.indexOf(LABEL_MAKER) + LABEL_MAKER.length());
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fd70e0a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
index d34a223..01ccb1d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
@@ -22,9 +22,11 @@ package org.apache.slider.providers.agent.application.metadata;
public class Component {
String name;
String category;
+ String publishConfig;
CommandScript commandScript;
public Component() {
+ publishConfig = Boolean.FALSE.toString();
}
public String getName() {
@@ -43,6 +45,14 @@ public class Component {
this.category = category;
}
+ public String getPublishConfig() {
+ return publishConfig;
+ }
+
+ public void setPublishConfig(String publishConfig) {
+ this.publishConfig = publishConfig;
+ }
+
public CommandScript getCommandScript() {
return commandScript;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fd70e0a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
index 554540c..ee5d3d0 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
@@ -55,6 +55,7 @@ public class MetainfoParser {
digester.addObjectCreate("*/component", Component.class);
digester.addBeanPropertySetter("*/component/name");
digester.addBeanPropertySetter("*/component/category");
+ digester.addBeanPropertySetter("*/component/publishConfig");
digester.addSetNext("*/component", "addComponent");
digester.addObjectCreate("*/commandScript", CommandScript.class);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fd70e0a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/CommandReport.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/CommandReport.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/CommandReport.java
index ff5e19a..a37e490 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/CommandReport.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/CommandReport.java
@@ -40,6 +40,7 @@ public class CommandReport {
private String serviceName;
private long taskId;
private String roleCommand;
+ private Map<String, String> folders;
private Map<String, String> allocatedPorts;
private Map<String, Map<String, String>> configurationTags;
@@ -165,7 +166,7 @@ public class CommandReport {
configurationTags = tags;
}
- /** @return the config tags that match this command, or <code>null</code> if none are present */
+ /** @return the allocated ports, or <code>null</code> if none are present */
@JsonProperty("allocatedPorts")
public Map<String, String> getAllocatedPorts() {
return allocatedPorts;
@@ -174,7 +175,19 @@ public class CommandReport {
/** @param ports allocated ports */
@JsonProperty("allocatedPorts")
public void setAllocatedPorts(Map<String, String> ports) {
- allocatedPorts = ports;
+ this.allocatedPorts = ports;
+ }
+
+ /** @return the folders, or <code>null</code> if none are present */
+ @JsonProperty("folders")
+ public Map<String, String> getFolders() {
+ return folders;
+ }
+
+ /** @param folders allocated ports */
+ @JsonProperty("folders")
+ public void setFolders(Map<String, String> folders) {
+ this.folders = folders;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fd70e0a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java
index fa149b8..d3388f5 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java
@@ -43,6 +43,7 @@ public class HeartBeat {
private List<DiskInfo> mounts = new ArrayList<DiskInfo>();
HostStatus nodeStatus;
private AgentEnv agentEnv = null;
+ private String fqdn;
public long getResponseId() {
return responseId;
@@ -56,18 +57,26 @@ public class HeartBeat {
return timestamp;
}
- public String getHostname() {
- return hostname;
- }
-
public void setTimestamp(long timestamp) {
this.timestamp = timestamp;
}
+ public String getHostname() {
+ return hostname;
+ }
+
public void setHostname(String hostname) {
this.hostname = hostname;
}
+ public String getFqdn() {
+ return fqdn;
+ }
+
+ public void setFqdn(String fqdn) {
+ this.fqdn = fqdn;
+ }
+
@JsonProperty("reports")
public List<CommandReport> getReports() {
return this.reports;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4fd70e0a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 11339c1..13093fc 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -168,6 +168,44 @@ public class TestAgentProviderService {
+ " </services>\n"
+ "</metainfo>";
+
+ private static final String metainfo_2_str = "<metainfo>\n"
+ + " <schemaVersion>2.0</schemaVersion>\n"
+ + " <services>\n"
+ + " <service>\n"
+ + " <name>HBASE</name>\n"
+ + " <comment>\n"
+ + " Apache HBase\n"
+ + " </comment>\n"
+ + " <version>0.96.0.2.1.1</version>\n"
+ + " <type>YARN-APP</type>\n"
+ + " <minHadoopVersion>2.1.0</minHadoopVersion>\n"
+ + " <components>\n"
+ + " <component>\n"
+ + " <name>HBASE_MASTER</name>\n"
+ + " <category>MASTER</category>\n"
+ + " <publishConfig>true</publishConfig>\n"
+ + " <minInstanceCount>1</minInstanceCount>\n"
+ + " <maxInstanceCount>2</maxInstanceCount>\n"
+ + " <commandScript>\n"
+ + " <script>scripts/hbase_master.py</script>\n"
+ + " <scriptType>PYTHON</scriptType>\n"
+ + " <timeout>600</timeout>\n"
+ + " </commandScript>\n"
+ + " </component>\n"
+ + " <component>\n"
+ + " <name>HBASE_REGIONSERVER</name>\n"
+ + " <category>SLAVE</category>\n"
+ + " <minInstanceCount>1</minInstanceCount>\n"
+ + " <commandScript>\n"
+ + " <script>scripts/hbase_regionserver.py</script>\n"
+ + " <scriptType>PYTHON</scriptType>\n"
+ + " </commandScript>\n"
+ + " </component>\n"
+ + " </components>\n"
+ + " </service>\n"
+ + " </services>\n"
+ + "</metainfo>";
@Test
public void testRegistration() throws IOException {
@@ -445,6 +483,35 @@ public class TestAgentProviderService {
}
@Test
+ public void testMetaInfoRelatedOperations() throws Exception {
+ InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
+ Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
+ InputStream metainfo_2 = new ByteArrayInputStream(metainfo_2_str.getBytes());
+ Metainfo metainfo2 = new MetainfoParser().parse(metainfo_2);
+ String role_hm = "HBASE_MASTER";
+ String role_hrs = "HBASE_REGIONSERVER";
+
+ AgentProviderService aps = new AgentProviderService();
+ AgentProviderService mockAps = Mockito.spy(aps);
+ doReturn(metainfo).when(mockAps).getMetainfo();
+
+ AgentProviderService mockAps2 = Mockito.spy(aps);
+ doReturn(metainfo2).when(mockAps2).getMetainfo();
+
+ Assert.assertTrue(mockAps.isMaster(role_hm));
+ Assert.assertFalse(mockAps.isMaster(role_hrs));
+ Assert.assertFalse(mockAps.canPublishConfig(role_hm));
+ Assert.assertFalse(mockAps.canPublishConfig(role_hrs));
+ Assert.assertFalse(mockAps.canAnyMasterPublishConfig());
+
+ Assert.assertTrue(mockAps2.isMaster(role_hm));
+ Assert.assertFalse(mockAps2.isMaster(role_hrs));
+ Assert.assertTrue(mockAps2.canPublishConfig(role_hm));
+ Assert.assertFalse(mockAps2.canPublishConfig(role_hrs));
+ Assert.assertTrue(mockAps2.canAnyMasterPublishConfig());
+ }
+
+ @Test
public void testOrchastratedAppStart() throws IOException {
// App has two components HBASE_MASTER and HBASE_REGIONSERVER
// Start of HBASE_RS depends on the start of HBASE_MASTER
@@ -502,6 +569,11 @@ public class TestAgentProviderService {
anyString(),
anyString(),
any(HeartBeatResponse.class));
+ doNothing().when(mockAps).publishComponentConfiguration(
+ anyString(),
+ anyString(),
+ anyCollection());
+
} catch (SliderException e) {
}
@@ -584,6 +656,7 @@ public class TestAgentProviderService {
cr.setRole("HBASE_REGIONSERVER");
cr.setRoleCommand("INSTALL");
cr.setStatus("COMPLETED");
+ cr.setFolders(new HashMap<String, String>() {{put("a", "b");}});
hb.setReports(Arrays.asList(cr));
hbr = mockAps.handleHeartBeat(hb);
Assert.assertEquals(3, hbr.getResponseId());
@@ -662,6 +735,11 @@ public class TestAgentProviderService {
} catch (SliderException | IOException he) {
log.warn(he.getMessage());
}
+
+ Mockito.verify(mockAps, Mockito.times(1)).publishComponentConfiguration(
+ anyString(),
+ anyString(),
+ anyCollection());
}