You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2015/03/19 04:40:08 UTC
[2/5] incubator-slider git commit: SLIDER-663. Make it easy to
develop and deploy application packages that are essentially shell commands
(part-I)
SLIDER-663. Make it easy to develop and deploy application packages that are essentially shell commands (part-I)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/704e8136
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/704e8136
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/704e8136
Branch: refs/heads/develop
Commit: 704e81363a72444f0d27b6fe3ab43d46b7791231
Parents: 38fed99
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Mar 17 17:49:49 2015 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Mar 17 17:49:49 2015 -0700
----------------------------------------------------------------------
.../python/agent/CustomServiceOrchestrator.py | 56 ++++-
.../python/scripts/shell_cmd/basic_installer.py | 55 +++++
.../src/main/python/scripts/shell_cmd/params.py | 29 +++
.../agent/TestCustomServiceOrchestrator.py | 104 ++++++++-
.../org/apache/slider/api/InternalKeys.java | 8 +
.../org/apache/slider/client/SliderClient.java | 22 +-
.../org/apache/slider/common/SliderKeys.java | 12 +-
.../AbstractClusterBuildingActionArgs.java | 31 ++-
.../slider/common/params/AddonArgsDelegate.java | 54 +++++
.../apache/slider/common/params/Arguments.java | 3 +
.../slider/common/tools/CoreFileSystem.java | 27 ++-
.../apache/slider/common/tools/SliderUtils.java | 56 ++++-
.../slider/core/build/InstanceBuilder.java | 6 +
.../core/persist/AppDefinitionPersister.java | 174 +++++++++++++++
.../slider/core/persist/InstancePaths.java | 4 +
.../providers/agent/AgentClientProvider.java | 57 +++--
.../slider/providers/agent/AgentKeys.java | 2 +
.../providers/agent/AgentProviderService.java | 211 ++++++++++++++-----
.../slider/providers/agent/AgentUtils.java | 30 ++-
.../providers/agent/ComponentCommandOrder.java | 4 +-
.../agent/application/metadata/Application.java | 94 +++++++--
.../application/metadata/CommandOrder.java | 9 +-
.../application/metadata/CommandScript.java | 11 +-
.../agent/application/metadata/Component.java | 83 ++++++--
.../application/metadata/ComponentCommand.java | 85 ++++++++
.../agent/application/metadata/ConfigFile.java | 8 +-
.../agent/application/metadata/Export.java | 9 +-
.../agent/application/metadata/ExportGroup.java | 11 +-
.../agent/application/metadata/Metainfo.java | 24 ++-
.../application/metadata/MetainfoParser.java | 62 +++++-
.../agent/application/metadata/OSPackage.java | 9 +-
.../agent/application/metadata/OSSpecific.java | 11 +-
.../agent/application/metadata/Package.java | 60 ++++++
.../agent/application/metadata/Validate.java | 27 +++
.../web/rest/agent/ExecutionCommand.java | 18 +-
.../test_min_pkg/nc_ping_cmd/appConfig.json | 8 +
.../test_min_pkg/nc_ping_cmd/metainfo.json | 28 +++
.../test_min_pkg/nc_ping_cmd/resources.json | 17 ++
.../test_min_pkg/sleep_cmd/metainfo.json | 16 ++
.../test_min_pkg/sleep_cmd/resources.json | 17 ++
.../agent/TestAgentClientProvider2.java | 12 +-
.../agent/TestAgentProviderService.java | 129 +++++++-----
.../metadata/TestMetainfoParser.java | 98 ++++++++-
.../funtest/framework/CommandTestBase.groovy | 80 +++++++
.../funtest/lifecycle/AgentMinSleepIT.groovy | 110 ++++++++++
.../funtest/lifecycle/AgentPingSocketIT.groovy | 142 +++++++++++++
46 files changed, 1907 insertions(+), 216 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 119c926..87ce621 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -76,23 +76,37 @@ class CustomServiceOrchestrator():
override_output_files=True, store_command=False):
allocated_ports = {}
try:
+ py_file_list = []
+ json_path = None
+
script_type = command['commandParams']['script_type']
- script = command['commandParams']['script']
- timeout = int(command['commandParams']['command_timeout'])
task_id = command['taskId']
command_name = command['roleCommand']
- script_path = self.resolve_script_path(self.base_dir, script, script_type)
- script_tuple = (script_path, self.base_dir)
-
tmpstrucoutfile = os.path.realpath(posixpath.join(self.tmp_dir,
"structured-out-{0}.json".format(task_id)))
- if script_type.upper() != self.SCRIPT_TYPE_PYTHON:
- # We don't support anything else yet
+ if script_type.upper() == self.SCRIPT_TYPE_PYTHON:
+ script = command['commandParams']['script']
+ timeout = int(command['commandParams']['command_timeout'])
+ script_path = self.resolve_script_path(self.base_dir, script, script_type)
+ script_tuple = (script_path, self.base_dir)
+ py_file_list = [script_tuple]
+
+ json_path = self.dump_command_to_json(command, allocated_ports, store_command)
+ elif script_type.upper() == "SHELL":
+ timeout = int(command['commandParams']['command_timeout'])
+
+ json_path = self.dump_command_to_json(command, allocated_ports, store_command)
+ script_path = os.path.realpath(posixpath.join(self.config.getWorkRootPath(),
+ "infra", "agent", "slider-agent", "scripts",
+ "shell_cmd", "basic_installer.py"))
+ script_tuple = (script_path, self.base_dir)
+ py_file_list = [script_tuple]
+ else:
+ # We don't support anything else yet
message = "Unknown script type {0}".format(script_type)
raise AgentException(message)
- json_path = self.dump_command_to_json(command, allocated_ports, store_command)
- py_file_list = [script_tuple]
+
# filter None values
filtered_py_file_list = [i for i in py_file_list if i]
logger_level = logging.getLevelName(logger.level)
@@ -239,6 +253,7 @@ class CustomServiceOrchestrator():
os.unlink(file_path)
self.finalize_command(command, store_command, allocated_ports)
+ self.finalize_exec_command(command)
with os.fdopen(os.open(file_path, os.O_WRONLY | os.O_CREAT,
0644), 'w') as f:
@@ -246,6 +261,7 @@ class CustomServiceOrchestrator():
f.write(content)
return file_path
+
"""
patch content
${AGENT_WORK_ROOT} -> AgentConfig.getWorkRootPath()
@@ -291,6 +307,28 @@ class CustomServiceOrchestrator():
pass
"""
+ configurations/global/exec_cmd should be resolved based on the rest of the config
+ {$conf:@//site/global/xmx_val} ==> command['configurations']['global']['xmx_val']
+ """
+ def finalize_exec_command(self, command):
+ variable_format = "{{$conf:@//site/{0}/{1}}}"
+ if 'configurations' in command:
+ if 'global' in command['configurations'] and 'exec_cmd' in command['configurations']['global']:
+ exec_cmd = command['configurations']['global']['exec_cmd']
+ for key in command['configurations']:
+ if len(command['configurations'][key]) > 0:
+ for k, value in command['configurations'][key].items():
+ replaced_key = variable_format.format(key, k)
+ exec_cmd = exec_cmd.replace(replaced_key, value)
+ pass
+ pass
+ pass
+ command['configurations']['global']['exec_cmd'] = exec_cmd
+ pass
+ pass
+
+
+ """
All unallocated ports should be set to 0
Look for "${SOME_COMPONENT_NAME.ALLOCATED_PORT}"
or "${component.ALLOCATED_PORT}{DEFAULT_port}"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-agent/src/main/python/scripts/shell_cmd/basic_installer.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/scripts/shell_cmd/basic_installer.py b/slider-agent/src/main/python/scripts/shell_cmd/basic_installer.py
new file mode 100644
index 0000000..561fd6c
--- /dev/null
+++ b/slider-agent/src/main/python/scripts/shell_cmd/basic_installer.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+class BasicInstaller(Script):
+ def install(self, env):
+ self.install_packages(env)
+ pass
+
+ def configure(self, env):
+ pass
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ process_cmd = format("{cmd}")
+
+ Execute(process_cmd,
+ logoutput=False,
+ wait_for_finish=False,
+ pid_file=params.pid_file,
+ poll_after = 5
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ def status(self, env):
+ import params
+ env.set_params(params)
+ check_process_status(params.pid_file)
+
+if __name__ == "__main__":
+ BasicInstaller().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-agent/src/main/python/scripts/shell_cmd/params.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/scripts/shell_cmd/params.py b/slider-agent/src/main/python/scripts/shell_cmd/params.py
new file mode 100644
index 0000000..9051f43
--- /dev/null
+++ b/slider-agent/src/main/python/scripts/shell_cmd/params.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+# server configurations
+config = Script.get_config()
+
+app_root = config['configurations']['global']['app_root']
+pid_file = config['configurations']['global']['pid_file']
+exec_cmd = config['configurations']['global']['exec_cmd']
+cmd = format(exec_cmd)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index 30c8d7a..aaef3f9 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -23,15 +23,13 @@ import pprint
from unittest import TestCase
import unittest
-import threading
import tempfile
-import time
+import posixpath
import logging
-from threading import Thread
from PythonExecutor import PythonExecutor
from CustomServiceOrchestrator import CustomServiceOrchestrator
-from mock.mock import MagicMock, patch
+from mock.mock import MagicMock, patch, call
import StringIO
import sys
from socket import socket
@@ -438,7 +436,6 @@ class TestCustomServiceOrchestrator(TestCase):
self.assertEqual.__self__.maxDiff = None
self.assertEqual(ret['exitcode'], 0)
self.assertTrue(run_file_mock.called)
- self.assertEqual(orchestrator.stored_command, command)
ret = orchestrator.requestComponentStatus(command_get)
self.assertEqual(ret['configurations'], expected)
@@ -447,6 +444,103 @@ class TestCustomServiceOrchestrator(TestCase):
self.assertEqual(ret['configurations'], expected_specific)
pass
+ @patch("hostname.public_hostname")
+ @patch("os.path.isfile")
+ @patch("os.unlink")
+ @patch.object(PythonExecutor, "run_file")
+ def test_runCommand_with_shell_config(self,
+ run_file_mock,
+ unlink_mock,
+ isfile_mock,
+ hostname_mock):
+ hostname_mock.return_value = "test.hst"
+ isfile_mock.return_value = True
+ command = {
+ 'role': 'MEMCACHED',
+ 'componentName': 'MEMCACHED',
+ 'hostLevelParams': {
+ 'jdk_location': 'some_location'
+ },
+ 'commandParams': {
+ 'script_type': 'SHELL',
+ 'command_timeout': '600'
+ },
+ 'configurations': {
+ "memcached-site": {
+ "memcached.log": "${AGENT_LOG_ROOT}",
+ "memcached.number": "10485760"},
+ "memcached-log4j": {"a": "b"}
+ },
+ 'taskId': '3',
+ 'roleCommand': 'INSTALL',
+ 'commandType': 'EXECUTION_COMMAND',
+ 'commandId': '1-1'
+ }
+
+ command_get = {
+ 'roleCommand': 'GET_CONFIG',
+ 'commandType': 'STATUS_COMMAND'
+ }
+
+ command_get_specific = {
+ 'roleCommand': 'GET_CONFIG',
+ 'commandType': 'STATUS_COMMAND',
+ 'commandParams': {
+ 'config_type': 'memcached-site'
+ }
+ }
+
+ tempdir = tempfile.gettempdir()
+ config = MagicMock()
+ config.get.return_value = "something"
+ config.getResolvedPath.return_value = tempdir
+ config.getWorkRootPath.return_value = tempdir
+ config.getLogPath.return_value = tempdir
+
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
+ # normal run case
+ run_file_mock.return_value = {
+ 'stdout': 'sss',
+ 'stderr': 'eee',
+ 'exitcode': 0,
+ }
+
+ expected = {
+ 'memcached-site': {
+ 'memcached.log': tempdir, 'memcached.number': '10485760'},
+ 'memcached-log4j': {'a': 'b'}}
+
+ expected_specific = {
+ 'memcached-site': {
+ 'memcached.log': tempdir, 'memcached.number': '10485760'},
+ }
+
+ ret = orchestrator.runCommand(command, "out.txt", "err.txt", True, True)
+ self.assertEqual.__self__.maxDiff = None
+ self.assertEqual(ret['exitcode'], 0)
+ self.assertTrue(run_file_mock.called)
+
+ ret = orchestrator.requestComponentStatus(command_get)
+ self.assertEqual(ret['configurations'], expected)
+
+ ret = orchestrator.requestComponentStatus(command_get_specific)
+ self.assertEqual(ret['configurations'], expected_specific)
+
+ script_path = os.path.realpath(posixpath.join(tempdir,
+ "infra", "agent", "slider-agent", "scripts",
+ "shell_cmd", "basic_installer.py"))
+ run_file_mock.assert_has_calls([call(
+ script_path,
+ ['INSTALL', os.path.realpath(posixpath.join(tempdir, 'command-3.json')),
+ os.path.realpath(posixpath.join(tempdir, 'package'))],
+ 'out.txt', 'err.txt', 600,
+ os.path.realpath(posixpath.join(tempdir, 'structured-out-3.json')),
+ 'INFO', True,
+ [('PYTHONPATH', ":".join([os.path.realpath(posixpath.join(tempdir, 'infra', 'agent', 'slider-agent', 'jinja2')),
+ os.path.realpath(posixpath.join(tempdir, 'infra', 'agent', 'slider-agent'))]))])])
+ pass
+
@patch.object(CustomServiceOrchestrator, "runCommand")
def test_requestComponentStatus(self, runCommand_mock):
status_command = {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
index b360fbe..fd17ed1 100644
--- a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
@@ -69,6 +69,14 @@ public interface InternalKeys {
*/
String INTERNAL_DATA_DIR_PATH = "internal.data.dir.path";
/**
+ * where the app def is stored
+ */
+ String INTERNAL_APPDEF_DIR_PATH = "internal.appdef.dir.path";
+ /**
+ * where addons for the app are stored
+ */
+ String INTERNAL_ADDONS_DIR_PATH = "internal.addons.dir.path";
+ /**
* Time in milliseconds to wait after forking any in-AM
* process before attempting to start up the containers: {@value}
*
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 84ed5a4..c23177a 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -20,6 +20,7 @@ package org.apache.slider.client;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.io.Files;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
@@ -133,6 +134,7 @@ import org.apache.slider.core.launch.LaunchedApplication;
import org.apache.slider.core.launch.RunningApplication;
import org.apache.slider.core.launch.SerializedApplicationReport;
import org.apache.slider.core.main.RunService;
+import org.apache.slider.core.persist.AppDefinitionPersister;
import org.apache.slider.core.persist.ApplicationReportSerDeser;
import org.apache.slider.core.persist.ConfPersister;
import org.apache.slider.core.persist.LockAcquireFailedException;
@@ -179,7 +181,6 @@ import java.net.InetSocketAddress;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.nio.charset.Charset;
-import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -1052,7 +1053,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
JSONObject config = null;
if(clientInfo.clientConfig != null) {
try {
- byte[] encoded = Files.readAllBytes(clientInfo.clientConfig.toPath());
+ byte[] encoded = Files.toByteArray(clientInfo.clientConfig);
config = new JSONObject(new String(encoded, Charset.defaultCharset()));
}catch(JSONException jsonEx) {
log.error("Unable to read supplied config", jsonEx);
@@ -1304,6 +1305,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
}
+ AppDefinitionPersister appDefinitionPersister = new AppDefinitionPersister(sliderFileSystem);
+ appDefinitionPersister.processSuppliedDefinitions(clustername, buildInfo, appConf);
+
//get the command line options
ConfTree cmdLineAppOptions = buildInfo.buildAppOptionsConfTree();
ConfTree cmdLineResourceOptions = buildInfo.buildResourceOptionsConfTree();
@@ -1395,19 +1399,25 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// make any substitutions needed at this stage
replaceTokens(appConf.getConfTree(), getUsername(), clustername);
- // providers to validate what there is
- AggregateConf instanceDescription = builder.getInstanceDescription();
- validateInstanceDefinition(sliderAM, instanceDescription, sliderFileSystem);
- validateInstanceDefinition(provider, instanceDescription, sliderFileSystem);
+ // TODO: Refactor the validation code and persistence code
try {
persistInstanceDefinition(overwrite, appconfdir, builder);
+ appDefinitionPersister.persistPackages();
+
} catch (LockAcquireFailedException e) {
log.warn("Failed to get a Lock on {} : {}", builder, e);
throw new BadClusterStateException("Failed to save " + clustername
+ ": " + e);
}
+
+ // providers to validate what there is
+ // TODO: Validation should be done before persistence
+ AggregateConf instanceDescription = builder.getInstanceDescription();
+ validateInstanceDefinition(sliderAM, instanceDescription, sliderFileSystem);
+ validateInstanceDefinition(provider, instanceDescription, sliderFileSystem);
}
+
protected void persistInstanceDefinition(boolean overwrite,
Path appconfdir,
InstanceBuilder builder)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 5cf7022..30ff258 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -168,11 +168,21 @@ public interface SliderKeys extends SliderXmlConfKeys {
String TMP_LOGDIR_PREFIX = "/tmp/slider-";
String TMP_DIR_PREFIX = "tmp";
String AM_DIR_PREFIX = "appmaster";
-
+
+ /**
+ * Store the default app definition, e.g. metainfo file or content of a folder
+ */
+ String APP_DEF_DIR = "appdef";
+ /**
+ * Store additional app defs - co-processors
+ */
+ String ADDONS_DIR = "addons";
+
String SLIDER_JAR = "slider.jar";
String JCOMMANDER_JAR = "jcommander.jar";
String GSON_JAR = "gson.jar";
String AGENT_TAR = "slider-agent.tar.gz";
+ String DEFAULT_APP_PKG = "appPkg.zip";
String DEFAULT_JVM_HEAP = "256M";
int DEFAULT_YARN_MEMORY = 256;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
index 56e01c3..f2e3c61 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
@@ -74,13 +74,6 @@ public abstract class AbstractClusterBuildingActionArgs extends
description = "Provider of the specific cluster application")
public String provider = SliderProviderFactory.DEFAULT_CLUSTER_TYPE;
- /*
-
- @Parameter(names = {ARG_PACKAGE},
- description = "URI to a slider package",
- converter = URIArgumentConverter.class )
- public URI packageURI;
- */
@Parameter(names = {ARG_PACKAGE},
description = "URI to a slider package")
public String packageURI;
@@ -93,6 +86,14 @@ public abstract class AbstractClusterBuildingActionArgs extends
description = "Template application configuration")
public File template;
+ @Parameter(names = {ARG_METAINFO},
+ description = "Application meta info")
+ public File appMetaInfo;
+
+ @Parameter(names = {ARG_APPDEF},
+ description = "Application def (folder or a zip package)")
+ public File appDef;
+
@Parameter(names = {ARG_QUEUE},
description = "Queue to submit the application")
public String queue;
@@ -100,6 +101,9 @@ public abstract class AbstractClusterBuildingActionArgs extends
@ParametersDelegate
public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
+ @ParametersDelegate
+ public AddonArgsDelegate addonDelegate = new AddonArgsDelegate();
+
@ParametersDelegate
public AppAndResouceOptionArgsDelegate optionsDelegate =
@@ -152,6 +156,19 @@ public abstract class AbstractClusterBuildingActionArgs extends
return componentDelegate.getComponentMap();
}
+ @VisibleForTesting
+ public List<String> getAddonTuples() {
+ return addonDelegate.getAddonTuples();
+ }
+
+ /**
+ * Get the list of addons (may be empty, but never null)
+ */
+ public Map<String, String> getAddonMap() throws
+ BadCommandArgumentsException {
+ return addonDelegate.getAddonMap();
+ }
+
public Path getConfdir() {
return confdir;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java
new file mode 100644
index 0000000..65ebc4b
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class AddonArgsDelegate extends AbstractArgsDelegate {
+
+ /**
+ * This is a listing of addon packages
+ */
+ @Parameter(names = {ARG_ADDON},
+ arity = 2,
+ description = "--addon <name> <folder or package>",
+ splitter = DontSplitArguments.class)
+ public List<String> addonTuples = new ArrayList<String>(0);
+
+
+ /**
+ * Get the list of addons (may be empty, but never null)
+ *
+ * @return map of named addons
+ *
+ * @throws BadCommandArgumentsException parse problem
+ */
+ public Map<String, String> getAddonMap() throws BadCommandArgumentsException {
+ return convertTupleListToMap("addon", addonTuples);
+ }
+
+ public List<String> getAddonTuples() {
+ return addonTuples;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index 9164edc..ea393d5 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -113,6 +113,9 @@ public interface Arguments {
String ARG_ZKPATH = "--zkpath";
String ARG_ZKPORT = "--zkport";
String ARG_CONFIG = "--config";
+ String ARG_METAINFO = "--metainfo";
+ String ARG_ADDON = "--addon";
+ String ARG_APPDEF = "--appdef";
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index e835312..3df73df 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -122,6 +122,31 @@ public class CoreFileSystem {
}
/**
+ * Build up the path string for app def folder -no attempt to
+ * create the directory is made
+ *
+ * @param clustername name of the cluster
+ * @return the path for persistent data
+ */
+ public Path buildAppDefDirPath(String clustername) {
+ Path path = buildClusterDirPath(clustername);
+ return new Path(path, SliderKeys.APP_DEF_DIR);
+ }
+
+ /**
+ * Build up the path string for addon folder -no attempt to
+ * create the directory is made
+ *
+ * @param clustername name of the cluster
+ * @return the path for persistent data
+ */
+ public Path buildAddonDirPath(String clustername, String addonId) {
+ Preconditions.checkNotNull(addonId);
+ Path path = buildClusterDirPath(clustername);
+ return new Path(path, SliderKeys.ADDONS_DIR + "/" + addonId);
+ }
+
+ /**
* Build up the path string for package install location -no attempt to
* create the directory is made
*
@@ -568,7 +593,7 @@ public class CoreFileSystem {
/**
* List all application instances persisted for this user, giving the
- * patha. The instance name is the last element in the path
+ * path. The instance name is the last element in the path
* @return a possibly empty map of application instance names to paths
*/
public Map<String, Path> listPersistentInstances() throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index ce52b89..1767d9c 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -69,6 +69,7 @@ import org.slf4j.LoggerFactory;
import java.io.ByteArrayInputStream;
import java.io.File;
+import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
@@ -107,6 +108,8 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
/**
* These are slider-specific Util methods
@@ -1732,7 +1735,58 @@ public final class SliderUtils {
}
/**
- * This wrapps ApplicationReports and generates a string version
+ * Given a source folder create zipped file
+ *
+ * @param srcFolder
+ * @param zipFile
+ *
+ * @throws IOException
+ */
+ public static void zipFolder(File srcFolder, File zipFile) throws IOException {
+ log.info("Zipping folder {} to {}", srcFolder.getAbsolutePath(), zipFile.getAbsolutePath());
+ List<String> files = new ArrayList<>();
+ generateFileList(files, srcFolder, srcFolder, true);
+
+ byte[] buffer = new byte[1024];
+
+ try (FileOutputStream fos = new FileOutputStream(zipFile)) {
+ try (ZipOutputStream zos = new ZipOutputStream(fos)) {
+
+ for (String file : files) {
+ ZipEntry ze = new ZipEntry(file);
+ zos.putNextEntry(ze);
+ try (FileInputStream in = new FileInputStream(srcFolder + File.separator + file)) {
+ int len;
+ while ((len = in.read(buffer)) > 0) {
+ zos.write(buffer, 0, len);
+ }
+ }
+ }
+ }
+ }
+ }
+
+
+ private static void generateFileList(List<String> fileList, File node, File rootFolder, Boolean relative) {
+ if (node.isFile()) {
+ String fileFullPath = node.toString();
+ if (relative) {
+ fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1, fileFullPath.length()));
+ } else {
+ fileList.add(fileFullPath);
+ }
+ }
+
+ if (node.isDirectory()) {
+ String[] subNode = node.list();
+ for (String filename : subNode) {
+ generateFileList(fileList, new File(node, filename), rootFolder, relative);
+ }
+ }
+ }
+
+ /**
+ * This wraps ApplicationReports and generates a string version
* iff the toString() operator is invoked
*/
public static class OnDemandReportStringifier {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
index 6812613..0a8dcdb 100644
--- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
@@ -49,6 +49,8 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Map;
+import static org.apache.slider.api.InternalKeys.INTERNAL_ADDONS_DIR_PATH;
+import static org.apache.slider.api.InternalKeys.INTERNAL_APPDEF_DIR_PATH;
import static org.apache.slider.api.InternalKeys.INTERNAL_QUEUE;
import static org.apache.slider.api.OptionKeys.INTERNAL_AM_TMP_DIR;
import static org.apache.slider.api.OptionKeys.INTERNAL_TMP_DIR;
@@ -142,6 +144,10 @@ public class InstanceBuilder {
instancePaths.generatedConfPath.toUri());
internalOps.set(INTERNAL_DATA_DIR_PATH,
instancePaths.dataPath.toUri());
+ internalOps.set(INTERNAL_APPDEF_DIR_PATH,
+ instancePaths.appDefPath.toUri());
+ internalOps.set(INTERNAL_ADDONS_DIR_PATH,
+ instancePaths.addonsPath.toUri());
internalOps.set(InternalKeys.INTERNAL_PROVIDER_NAME, provider);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
new file mode 100644
index 0000000..8f0f7b0
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.persist;
+
+import com.google.common.io.Files;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.providers.agent.AgentKeys;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Class to prepare and persist app and add-on definitions.
+ *
+ * In this case, the app definition and add-on definitions are auto-inferred from the user input rather than explicit
+ * inclusion of application package in the config.
+ *
+ * Processing an app definition involves one or more of the following: - modify appConfig - package definition into a
+ * temporary folder - upload to HDFS
+ *
+ * This class keeps track of all the required operations and allows them to be invoked by build operation
+ */
+public class AppDefinitionPersister {
+ private static final Logger log =
+ LoggerFactory.getLogger(AppDefinitionPersister.class);
+
+ private final SliderFileSystem sliderFileSystem;
+ private List<AppDefinition> appDefinitions;
+
+ public AppDefinitionPersister(SliderFileSystem sliderFileSystem) {
+ this.sliderFileSystem = sliderFileSystem;
+ appDefinitions = new ArrayList<>();
+ }
+
+
+ /**
+ * Process the application package or folder by copying it to the cluster path
+ *
+ * @param appDefinition details of application package
+ *
+ * @throws BadConfigException
+ * @throws IOException
+ */
+ private void persistDefinitionPackageOrFolder(AppDefinition appDefinition)
+ throws BadConfigException, IOException {
+ if (!appDefinition.appDefPkgOrFolder.canRead()) {
+ throw new BadConfigException("Pkg/Folder cannot be accessed - "
+ + appDefinition.appDefPkgOrFolder.getAbsolutePath());
+ }
+
+ File src = appDefinition.appDefPkgOrFolder;
+ String targetName = appDefinition.appDefPkgOrFolder.getName();
+
+ if (appDefinition.appDefPkgOrFolder.isDirectory()) {
+ log.info("Processing app package/folder {} for {}",
+ appDefinition.appDefPkgOrFolder.getAbsolutePath(),
+ appDefinition.pkgName);
+ File tmpDir = Files.createTempDir();
+ File zipFile = new File(tmpDir.getCanonicalPath(), File.separator + appDefinition.pkgName);
+ SliderUtils.zipFolder(appDefinition.appDefPkgOrFolder, zipFile);
+
+ src = zipFile;
+ targetName = appDefinition.pkgName;
+ }
+
+ sliderFileSystem.getFileSystem().copyFromLocalFile(
+ false,
+ false,
+ new Path(src.toURI()),
+ new Path(appDefinition.targetFolderInFs, targetName));
+ }
+
+ public void persistPackages() throws BadConfigException, IOException {
+ for (AppDefinition appDefinition : appDefinitions) {
+ persistDefinitionPackageOrFolder(appDefinition);
+ }
+ }
+
+ public void processSuppliedDefinitions(String clustername,
+ AbstractClusterBuildingActionArgs buildInfo,
+ ConfTreeOperations appConf)
+ throws BadConfigException, IOException, BadCommandArgumentsException {
+ // if metainfo is provided add to the app instance
+ if (buildInfo.appMetaInfo != null) {
+ File tempDir = Files.createTempDir();
+ File pkgSrcDir = new File(tempDir, "default");
+ pkgSrcDir.mkdirs();
+ Files.copy(buildInfo.appMetaInfo, new File(pkgSrcDir, "metainfo.json"));
+
+ Path appDirPath = sliderFileSystem.buildAppDefDirPath(clustername);
+ log.info("Using default app def path {}", appDirPath.toString());
+
+ appDefinitions.add(new AppDefinition(appDirPath, pkgSrcDir, SliderKeys.DEFAULT_APP_PKG));
+ Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG);
+ appConf.getGlobalOptions().set(AgentKeys.APP_DEF, appDefPath);
+ log.info("Setting app package to {}.", appDefPath);
+ }
+
+ if (buildInfo.appDef != null) {
+ Path appDirPath = sliderFileSystem.buildAppDefDirPath(clustername);
+ appDefinitions.add(new AppDefinition(appDirPath, buildInfo.appDef, SliderKeys.DEFAULT_APP_PKG));
+ Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG);
+ appConf.getGlobalOptions().set(AgentKeys.APP_DEF, appDefPath);
+ log.info("Setting app package to {}.", appDefPath);
+ }
+
+ if (buildInfo.addonDelegate.getAddonMap().size() > 0) {
+ List<String> addons = new ArrayList<String>();
+ Map<String, String> addonMap = buildInfo.addonDelegate.getAddonMap();
+ for (String key : addonMap.keySet()) {
+ Path addonPath = sliderFileSystem.buildAddonDirPath(clustername, key);
+ String addonPkgName = "addon_" + key + ".zip";
+ appDefinitions.add(new AppDefinition(addonPath, buildInfo.appDef, addonPkgName));
+ String addOnKey = AgentKeys.ADDON_PREFIX + key;
+ Path addonPkgPath = new Path(addonPath, addonPkgName);
+ log.info("Setting addon package {} to {}.", addOnKey, addonPkgPath);
+ appConf.getGlobalOptions().set(addOnKey, addonPkgPath);
+ }
+
+ String existingList = appConf.getGlobalOptions().get(AgentKeys.ADDONS);
+ if (SliderUtils.isUnset(existingList)) {
+ existingList = "";
+ }
+ appConf.getGlobalOptions().set(AgentKeys.ADDONS, existingList + StringUtils.join(addons, ","));
+ }
+ }
+
+
+ // Helper class to hold details for the app and addon packages
+ class AppDefinition {
+ // The target folder where the package will be stored
+ Path targetFolderInFs;
+ // The on disk location of the app def package or folder
+ File appDefPkgOrFolder;
+ // Package name
+ String pkgName;
+
+ public AppDefinition(Path targetFolderInFs, File appDefPkgOrFolder, String pkgName) {
+ this.targetFolderInFs = targetFolderInFs;
+ this.appDefPkgOrFolder = appDefPkgOrFolder;
+ this.pkgName = pkgName;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java b/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java
index df1206d..3505ac3 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java
@@ -34,6 +34,8 @@ public class InstancePaths {
public final Path dataPath;
public final Path tmpPath;
public final Path tmpPathAM;
+ public final Path appDefPath;
+ public final Path addonsPath;
public InstancePaths(Path instanceDir) {
this.instanceDir = instanceDir;
@@ -45,6 +47,8 @@ public class InstancePaths {
dataPath = new Path(instanceDir, SliderKeys.DATA_DIR_NAME);
tmpPath = new Path(instanceDir, SliderKeys.TMP_DIR_PREFIX);
tmpPathAM = new Path(tmpPath, SliderKeys.AM_DIR_PREFIX);
+ appDefPath = new Path(tmpPath, SliderKeys.APP_DEF_DIR);
+ addonsPath = new Path(tmpPath, SliderKeys.ADDONS_DIR);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 9da269d..2e981c6 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -152,6 +152,7 @@ public class AgentClientProvider extends AbstractClientProvider
} catch (BadConfigException bce) {
throw new BadConfigException("Application definition must be provided. " + bce.getMessage());
}
+
String appDef = instanceDefinition.getAppConfOperations().
getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
log.info("Validating app definition {}", appDef);
@@ -170,7 +171,7 @@ public class AgentClientProvider extends AbstractClientProvider
metaInfo = AgentUtils.getApplicationMetainfo(fs, appDef);
} catch (IOException ioe) {
// Ignore missing metainfo file for now
- log.info("Missing metainfo.xml {}", ioe.getMessage());
+ log.info("Missing metainfo {}", ioe.getMessage());
}
}
@@ -215,19 +216,31 @@ public class AgentClientProvider extends AbstractClientProvider
"Component %s is not a member of application.", name);
}
- MapOperations componentConfig = resources.getMandatoryComponent(name);
- int count =
- componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES);
- int definedMinCount = componentDef.getMinInstanceCountInt();
- int definedMaxCount = componentDef.getMaxInstanceCountInt();
- if (count < definedMinCount || count > definedMaxCount) {
- throw new BadConfigException("Component %s, %s value %d out of range. "
- + "Expected minimum is %d and maximum is %d",
- name,
- ResourceKeys.COMPONENT_INSTANCES,
- count,
- definedMinCount,
- definedMaxCount);
+ // ensure that intance count is 0 for client components
+ if ("CLIENT".equals(componentDef.getCategory())) {
+ MapOperations componentConfig = resources.getMandatoryComponent(name);
+ int count =
+ componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES);
+ if (count > 0) {
+ throw new BadConfigException("Component %s is of type CLIENT and cannot be instantiated."
+ + " Use \"slider client install ...\" command instead.",
+ name);
+ }
+ } else {
+ MapOperations componentConfig = resources.getMandatoryComponent(name);
+ int count =
+ componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES);
+ int definedMinCount = componentDef.getMinInstanceCountInt();
+ int definedMaxCount = componentDef.getMaxInstanceCountInt();
+ if (count < definedMinCount || count > definedMaxCount) {
+ throw new BadConfigException("Component %s, %s value %d out of range. "
+ + "Expected minimum is %d and maximum is %d",
+ name,
+ ResourceKeys.COMPONENT_INSTANCES,
+ count,
+ definedMinCount,
+ definedMaxCount);
+ }
}
}
}
@@ -327,7 +340,19 @@ public class AgentClientProvider extends AbstractClientProvider
while (offset < size) {
offset += zipInputStream.read(content, offset, size - offset);
}
- metaInfo = new MetainfoParser().parse(new ByteArrayInputStream(content));
+ metaInfo = new MetainfoParser().fromXmlStream(new ByteArrayInputStream(content));
+ }
+ } else if ("metainfo.json".equals(zipEntry.getName())) {
+ int size = (int) zipEntry.getSize();
+ if (size != -1) {
+ log.info("Reading {} of size {}", zipEntry.getName(),
+ zipEntry.getSize());
+ byte[] content = new byte[size];
+ int offset = 0;
+ while (offset < size) {
+ offset += zipInputStream.read(content, offset, size - offset);
+ }
+ metaInfo = new MetainfoParser().fromJsonStream(new ByteArrayInputStream(content));
}
} else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) {
int size = (int) zipEntry.getSize();
@@ -362,7 +387,7 @@ public class AgentClientProvider extends AbstractClientProvider
}
if (metaInfo == null) {
- throw new SliderException("Not a valid app package. Could not read metainfo.xml.");
+ throw new SliderException("Not a valid app package. Could not read metainfo.");
}
expandAgentTar(agentPkgDir);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index 2323f97..963dadc 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -67,6 +67,8 @@ public interface AgentKeys {
String AGENT_MAIN_SCRIPT = "agent/main.py";
String APP_DEF = "application.def";
+ String ADDON_PREFIX = "application.addon.";
+ String ADDONS = "application.addons";
String AGENT_VERSION = "agent.version";
String AGENT_CONF = "agent.conf";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index bdf168e..4eadf13 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.registry.client.types.Endpoint;
import org.apache.hadoop.registry.client.types.ProtocolTypes;
import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.Container;
@@ -38,7 +37,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.slider.api.ClusterDescription;
-import org.apache.slider.api.ClusterDescriptionKeys;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.OptionKeys;
@@ -69,6 +67,7 @@ import org.apache.slider.providers.ProviderUtils;
import org.apache.slider.providers.agent.application.metadata.Application;
import org.apache.slider.providers.agent.application.metadata.CommandScript;
import org.apache.slider.providers.agent.application.metadata.Component;
+import org.apache.slider.providers.agent.application.metadata.ComponentCommand;
import org.apache.slider.providers.agent.application.metadata.ComponentExport;
import org.apache.slider.providers.agent.application.metadata.ConfigFile;
import org.apache.slider.providers.agent.application.metadata.DefaultConfig;
@@ -77,6 +76,7 @@ import org.apache.slider.providers.agent.application.metadata.ExportGroup;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.providers.agent.application.metadata.OSPackage;
import org.apache.slider.providers.agent.application.metadata.OSSpecific;
+import org.apache.slider.providers.agent.application.metadata.Package;
import org.apache.slider.providers.agent.application.metadata.PropertyInfo;
import org.apache.slider.server.appmaster.actions.ProviderReportedContainerLoss;
import org.apache.slider.server.appmaster.actions.RegisterComponentInstance;
@@ -157,7 +157,7 @@ public class AgentProviderService extends AbstractProviderService implements
private int heartbeatMonitorInterval = 0;
private AgentClientProvider clientProvider;
private AtomicInteger taskId = new AtomicInteger(0);
- private volatile Metainfo metainfo = null;
+ private volatile Metainfo metaInfo = null;
private Map<String, DefaultConfig> defaultConfigs = null;
private ComponentCommandOrder commandOrder = null;
private HeartbeatMonitor monitor;
@@ -233,7 +233,7 @@ public class AgentProviderService extends AbstractProviderService implements
Set<String> names = resources.getComponentNames();
names.remove(SliderKeys.COMPONENT_AM);
for (String name : names) {
- Component componentDef = getMetainfo().getApplicationComponent(name);
+ Component componentDef = getMetaInfo().getApplicationComponent(name);
if (componentDef == null) {
throw new BadConfigException(
"Component %s is not a member of application.", name);
@@ -262,21 +262,20 @@ public class AgentProviderService extends AbstractProviderService implements
String appDef = instanceDefinition.getAppConfOperations()
.getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
- if (metainfo == null) {
+ if (metaInfo == null) {
synchronized (syncLock) {
- if (metainfo == null) {
+ if (metaInfo == null) {
readAndSetHeartbeatMonitoringInterval(instanceDefinition);
initializeAgentDebugCommands(instanceDefinition);
- metainfo = getApplicationMetainfo(fileSystem, appDef);
- if (metainfo == null || metainfo.getApplication() == null) {
+ metaInfo = getApplicationMetainfo(fileSystem, appDef);
+ if (metaInfo == null || metaInfo.getApplication() == null) {
log.error("metainfo.xml is unavailable or malformed at {}.", appDef);
throw new SliderException(
"metainfo.xml is required in app package.");
}
- commandOrder = new ComponentCommandOrder(metainfo.getApplication()
- .getCommandOrder());
- defaultConfigs = initializeDefaultConfigs(fileSystem, appDef, metainfo);
+ commandOrder = new ComponentCommandOrder(metaInfo.getApplication().getCommandOrders());
+ defaultConfigs = initializeDefaultConfigs(fileSystem, appDef, metaInfo);
monitor = new HeartbeatMonitor(this, getHeartbeatMonitorInterval());
monitor.start();
}
@@ -717,15 +716,20 @@ public class AgentProviderService extends AbstractProviderService implements
StateAccessForProviders accessor = getAmState();
CommandScript cmdScript = getScriptPathFromMetainfo(roleName);
+ List<ComponentCommand> commands = getMetaInfo().getApplicationComponent(roleName).getCommands();
- if (cmdScript == null || cmdScript.getScript() == null) {
+ if ((cmdScript == null || cmdScript.getScript() == null) && commands.size() == 0) {
log.error("role.script is unavailable for {}. Commands will not be sent.",
roleName);
return response;
}
- String scriptPath = cmdScript.getScript();
- long timeout = cmdScript.getTimeout();
+ String scriptPath = null;
+ long timeout = 600L;
+ if(cmdScript != null) {
+ scriptPath = cmdScript.getScript();
+ timeout = cmdScript.getTimeout();
+ }
if (timeout == 0L) {
timeout = 600L;
@@ -778,15 +782,50 @@ public class AgentProviderService extends AbstractProviderService implements
if (Command.NOP != command) {
if (command == Command.INSTALL) {
log.info("Installing {} on {}.", roleName, containerId);
- addInstallCommand(roleName, containerId, response, scriptPath, timeout);
+ if (scriptPath != null) {
+ addInstallCommand(roleName, containerId, response, scriptPath, null, timeout);
+ } else {
+ // commands
+ ComponentCommand installCmd = null;
+ for (ComponentCommand compCmd : commands) {
+ if (compCmd.getName().equals("INSTALL")) {
+ installCmd = compCmd;
+ }
+ }
+ addInstallCommand(roleName, containerId, response, null, installCmd, timeout);
+ }
componentStatus.commandIssued(command);
} else if (command == Command.START) {
// check against dependencies
boolean canExecute = commandOrder.canExecute(roleName, command, getComponentStatuses().values());
if (canExecute) {
log.info("Starting {} on {}.", roleName, containerId);
- addStartCommand(roleName, containerId, response, scriptPath, timeout, isMarkedAutoRestart(roleName));
- componentStatus.commandIssued(command);
+ if (scriptPath != null) {
+ addStartCommand(roleName,
+ containerId,
+ response,
+ scriptPath,
+ null,
+ null,
+ timeout,
+ isMarkedAutoRestart(roleName));
+ componentStatus.commandIssued(command);
+ } else {
+ ComponentCommand startCmd = null;
+ for (ComponentCommand compCmd : commands) {
+ if (compCmd.getName().equals("START")) {
+ startCmd = compCmd;
+ }
+ }
+ ComponentCommand stopCmd = null;
+ for (ComponentCommand compCmd : commands) {
+ if (compCmd.getName().equals("STOP")) {
+ stopCmd = compCmd;
+ }
+ }
+ addStartCommand(roleName, containerId, response, null, startCmd, stopCmd, timeout, false);
+ componentStatus.commandIssued(command);
+ }
} else {
log.info("Start of {} on {} delayed as dependencies have not started.", roleName, containerId);
}
@@ -858,7 +897,6 @@ public class AgentProviderService extends AbstractProviderService implements
}
}
- // component specific publishes
processAndPublishComponentSpecificData(ports, containerId, fqdn, roleName);
processAndPublishComponentSpecificExports(ports, containerId, fqdn, roleName);
@@ -1012,8 +1050,8 @@ public class AgentProviderService extends AbstractProviderService implements
}
@VisibleForTesting
- protected Metainfo getMetainfo() {
- return this.metainfo;
+ protected Metainfo getMetaInfo() {
+ return this.metaInfo;
}
@VisibleForTesting
@@ -1023,7 +1061,7 @@ public class AgentProviderService extends AbstractProviderService implements
@VisibleForTesting
protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
- String appDef) throws IOException {
+ String appDef) throws IOException, BadConfigException {
return AgentUtils.getApplicationMetainfo(fileSystem, appDef);
}
@@ -1045,7 +1083,7 @@ public class AgentProviderService extends AbstractProviderService implements
*/
protected Map<String, DefaultConfig> initializeDefaultConfigs(SliderFileSystem fileSystem,
String appDef, Metainfo metainfo) throws IOException {
- Map<String, DefaultConfig> defaultConfigMap = new HashMap<String, DefaultConfig>();
+ Map<String, DefaultConfig> defaultConfigMap = new HashMap<>();
if (SliderUtils.isNotEmpty(metainfo.getApplication().getConfigFiles())) {
for (ConfigFile configFile : metainfo.getApplication().getConfigFiles()) {
DefaultConfig config = null;
@@ -1226,7 +1264,7 @@ public class AgentProviderService extends AbstractProviderService implements
log.info("Status report: {}", status.toString());
if (status.getConfigs() != null) {
- Application application = getMetainfo().getApplication();
+ Application application = getMetaInfo().getApplication();
if (canAnyMasterPublishConfig() == false || canPublishConfig(componentName)) {
// If no Master can explicitly publish then publish if its a master
@@ -1370,10 +1408,10 @@ public class AgentProviderService extends AbstractProviderService implements
String hostNamePattern = "${THIS_HOST}";
Map<String, String> toPublish = new HashMap<String, String>();
- Application application = getMetainfo().getApplication();
+ Application application = getMetaInfo().getApplication();
for (Component component : application.getComponents()) {
if (component.getName().equals(componentName)) {
- if (!component.getComponentExports().isEmpty()) {
+ if (component.getComponentExports().size() > 0) {
for (ComponentExport export : component.getComponentExports()) {
String templateToExport = export.getValue();
@@ -1399,7 +1437,7 @@ public class AgentProviderService extends AbstractProviderService implements
}
}
- if (!toPublish.isEmpty()) {
+ if (toPublish.size() > 0) {
Map<String, String> perContainerData = null;
if (!getComponentInstanceData().containsKey(containerId)) {
perContainerData = new ConcurrentHashMap<String, String>();
@@ -1420,8 +1458,8 @@ public class AgentProviderService extends AbstractProviderService implements
String portVarFormat = "${site.%s}";
String hostNamePattern = "${" + compName + "_HOST}";
- List<ExportGroup> appExportGroups = getMetainfo().getApplication().getExportGroups();
- Component component = getMetainfo().getApplicationComponent(compName);
+ List<ExportGroup> appExportGroups = getMetaInfo().getApplication().getExportGroups();
+ Component component = getMetaInfo().getApplicationComponent(compName);
if (component != null && SliderUtils.isSet(component.getCompExports())
&& SliderUtils.isNotEmpty(appExportGroups)) {
@@ -1523,7 +1561,7 @@ public class AgentProviderService extends AbstractProviderService implements
* @return the component entry or null for no match
*/
protected Component getApplicationComponent(String roleName) {
- return getMetainfo().getApplicationComponent(roleName);
+ return getMetaInfo().getApplicationComponent(roleName);
}
/**
@@ -1583,7 +1621,7 @@ public class AgentProviderService extends AbstractProviderService implements
protected boolean isMarkedAutoRestart(String roleName) {
Component component = getApplicationComponent(roleName);
if (component != null) {
- return component.getRequiresAutoRestart();
+ return component.getAutoStartOnFailureBoolean();
}
return false;
}
@@ -1595,7 +1633,7 @@ public class AgentProviderService extends AbstractProviderService implements
*/
protected boolean canAnyMasterPublishConfig() {
if (canAnyMasterPublish == null) {
- Application application = getMetainfo().getApplication();
+ Application application = getMetaInfo().getApplication();
if (application == null) {
log.error("Malformed app definition: Expect application as root element in the metainfo.xml");
} else {
@@ -1637,6 +1675,7 @@ public class AgentProviderService extends AbstractProviderService implements
String containerId,
HeartBeatResponse response,
String scriptPath,
+ ComponentCommand compCmd,
long timeout)
throws SliderException {
assert getAmState().isApplicationLive();
@@ -1651,7 +1690,7 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setComponentName(componentName);
cmd.setRole(componentName);
Map<String, String> hostLevelParams = new TreeMap<String, String>();
- hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME));
+ hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir()));
hostLevelParams.put(PACKAGE_LIST, getPackageList());
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
@@ -1659,9 +1698,20 @@ public class AgentProviderService extends AbstractProviderService implements
Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, componentName);
cmd.setConfigurations(configurations);
- cmd.setCommandParams(setCommandParameters(scriptPath, timeout, false));
+ if(SliderUtils.isSet(scriptPath)) {
+ cmd.setCommandParams(commandParametersSet(scriptPath, timeout, false));
+ } else {
+ // assume it to be default shell command
+ ComponentCommand effectiveCommand = compCmd;
+ if(effectiveCommand == null) {
+ effectiveCommand = ComponentCommand.getDefaultComponentCommand("INSTALL");
+ }
+ cmd.setCommandParams(commandParametersSet(effectiveCommand, timeout, false));
+ configurations.get("global").put("exec_cmd", effectiveCommand.getExec());
+ }
cmd.setHostname(getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME));
+
response.addExecutionCommand(cmd);
}
@@ -1670,12 +1720,19 @@ public class AgentProviderService extends AbstractProviderService implements
String pkgListFormatString = "[%s]";
List<String> packages = new ArrayList();
if (application != null) {
- List<OSSpecific> osSpecifics = application.getOSSpecifics();
- if (osSpecifics != null && osSpecifics.size() > 0) {
- for (OSSpecific osSpecific : osSpecifics) {
- if (osSpecific.getOsType().equals("any")) {
- for (OSPackage osPackage : osSpecific.getPackages()) {
- packages.add(String.format(pkgFormatString, osPackage.getType(), osPackage.getName()));
+ if (application.getPackages().size() > 0) {
+ List<Package> appPackages = application.getPackages();
+ for (Package appPackage : appPackages) {
+ packages.add(String.format(pkgFormatString, appPackage.getType(), appPackage.getName()));
+ }
+ } else {
+ List<OSSpecific> osSpecifics = application.getOSSpecifics();
+ if (osSpecifics != null && osSpecifics.size() > 0) {
+ for (OSSpecific osSpecific : osSpecifics) {
+ if (osSpecific.getOsType().equals("any")) {
+ for (OSPackage osPackage : osSpecific.getPackages()) {
+ packages.add(String.format(pkgFormatString, osPackage.getType(), osPackage.getName()));
+ }
}
}
}
@@ -1690,7 +1747,7 @@ public class AgentProviderService extends AbstractProviderService implements
}
private String getPackageList() {
- return getPackageListFromApplication(getMetainfo().getApplication());
+ return getPackageListFromApplication(getMetaInfo().getApplication());
}
private void prepareExecutionCommand(ExecutionCommand cmd) {
@@ -1698,8 +1755,8 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setCommandId(cmd.getTaskId() + "-1");
}
- private Map<String, String> setCommandParameters(String scriptPath, long timeout, boolean recordConfig) {
- Map<String, String> cmdParams = new TreeMap<String, String>();
+ private Map<String, String> commandParametersSet(String scriptPath, long timeout, boolean recordConfig) {
+ Map<String, String> cmdParams = new TreeMap<>();
cmdParams.put("service_package_folder",
"${AGENT_WORK_ROOT}/work/app/definition/package");
cmdParams.put("script", scriptPath);
@@ -1710,6 +1767,18 @@ public class AgentProviderService extends AbstractProviderService implements
return cmdParams;
}
+ private Map<String, String> commandParametersSet(ComponentCommand compCmd, long timeout, boolean recordConfig) {
+ Map<String, String> cmdParams = new TreeMap<>();
+ cmdParams.put("service_package_folder",
+ "${AGENT_WORK_ROOT}/work/app/definition/package");
+ cmdParams.put("command", compCmd.getExec());
+ cmdParams.put("schema_version", "2.0");
+ cmdParams.put("command_timeout", Long.toString(timeout));
+ cmdParams.put("script_type", compCmd.getType());
+ cmdParams.put("record_config", Boolean.toString(recordConfig));
+ return cmdParams;
+ }
+
@VisibleForTesting
protected void addStatusCommand(String componentName,
String containerId,
@@ -1730,11 +1799,11 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setRoleCommand(StatusCommand.STATUS_COMMAND);
Map<String, String> hostLevelParams = new TreeMap<String, String>();
- hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME));
+ hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir()));
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
- cmd.setCommandParams(setCommandParameters(scriptPath, timeout, false));
+ cmd.setCommandParams(commandParametersSet(scriptPath, timeout, false));
Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, componentName);
@@ -1767,7 +1836,9 @@ public class AgentProviderService extends AbstractProviderService implements
@VisibleForTesting
protected void addStartCommand(String componentName, String containerId, HeartBeatResponse response,
- String scriptPath, long timeout, boolean isMarkedAutoRestart)
+ String scriptPath, ComponentCommand startCommand,
+ ComponentCommand stopCommand,
+ long timeout, boolean isMarkedAutoRestart)
throws
SliderException {
assert getAmState().isApplicationLive();
@@ -1784,20 +1855,28 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setServiceName(clusterName);
cmd.setComponentName(componentName);
cmd.setRole(componentName);
- Map<String, String> hostLevelParams = new TreeMap<String, String>();
- hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME));
+ Map<String, String> hostLevelParams = new TreeMap<>();
+ hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir()));
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
- Map<String, String> roleParams = new TreeMap<String, String>();
+ Map<String, String> roleParams = new TreeMap<>();
cmd.setRoleParams(roleParams);
cmd.getRoleParams().put("auto_restart", Boolean.toString(isMarkedAutoRestart));
- cmd.setCommandParams(setCommandParameters(scriptPath, timeout, true));
-
Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, componentName);
-
cmd.setConfigurations(configurations);
+
+ if(SliderUtils.isSet(scriptPath)) {
+ cmd.setCommandParams(commandParametersSet(scriptPath, timeout, true));
+ } else {
+ if(startCommand == null) {
+ throw new SliderException("Expected START command not found for component " + componentName);
+ }
+ cmd.setCommandParams(commandParametersSet(startCommand, timeout, true));
+ configurations.get("global").put("exec_cmd", startCommand.getExec());
+ }
+
response.addExecutionCommand(cmd);
// With start command, the corresponding command for graceful stop needs to
@@ -1815,16 +1894,25 @@ public class AgentProviderService extends AbstractProviderService implements
cmdStop.setRole(componentName);
Map<String, String> hostLevelParamsStop = new TreeMap<String, String>();
hostLevelParamsStop.put(JAVA_HOME, appConf.getGlobalOptions()
- .getMandatoryOption(JAVA_HOME));
+ .getOption(JAVA_HOME, ""));
hostLevelParamsStop.put(CONTAINER_ID, containerId);
cmdStop.setHostLevelParams(hostLevelParamsStop);
Map<String, String> roleParamsStop = new TreeMap<String, String>();
cmdStop.setRoleParams(roleParamsStop);
cmdStop.getRoleParams().put("auto_restart",
- Boolean.toString(isMarkedAutoRestart));
+ Boolean.toString(isMarkedAutoRestart));
+
+ if(SliderUtils.isSet(scriptPath)) {
+ cmdStop.setCommandParams(commandParametersSet(scriptPath, timeout, true));
+ } else {
+ if(stopCommand == null) {
+ stopCommand = ComponentCommand.getDefaultComponentCommand("STOP");
+ }
+ cmd.setCommandParams(commandParametersSet(stopCommand, timeout, true));
+ configurations.get("global").put("exec_cmd", startCommand.getExec());
+ }
- cmdStop.setCommandParams(setCommandParameters(scriptPath, timeout, true));
Map<String, Map<String, String>> configurationsStop = buildCommandConfigurations(
appConf, containerId, componentName);
@@ -1832,6 +1920,19 @@ public class AgentProviderService extends AbstractProviderService implements
response.addExecutionCommand(cmdStop);
}
+ protected static String getJDKDir() {
+ File javaHome = new File(System.getProperty("java.home")).getParentFile();
+ File jdkDirectory = null;
+ if (javaHome.getName().contains("jdk")) {
+ jdkDirectory = javaHome;
+ }
+ if (jdkDirectory != null) {
+ return jdkDirectory.getAbsolutePath();
+ } else {
+ return "";
+ }
+ }
+
protected Map<String, String> getAllocatedPorts() {
return getAllocatedPorts(SHARED_PORT_TAG);
}
@@ -1938,7 +2039,7 @@ public class AgentProviderService extends AbstractProviderService implements
List<String> configList = new ArrayList<String>();
configList.add(GLOBAL_CONFIG_TAG);
- List<ConfigFile> configFiles = getMetainfo().getApplication().getConfigFiles();
+ List<ConfigFile> configFiles = getMetaInfo().getApplication().getConfigFiles();
for (ConfigFile configFile : configFiles) {
log.info("Expecting config type {}.", configFile.getDictionaryName());
configList.add(configFile.getDictionaryName());
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
index 1d61c15..6fe1161 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
@@ -20,6 +20,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.providers.agent.application.metadata.DefaultConfig;
import org.apache.slider.providers.agent.application.metadata.DefaultConfigParser;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
@@ -28,7 +29,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.FileNotFoundException;
-import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@@ -39,20 +39,30 @@ public class AgentUtils {
private static final Logger log = LoggerFactory.getLogger(AgentUtils.class);
public static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
- String appDef) throws IOException {
+ String appDef) throws IOException, BadConfigException {
log.info("Reading metainfo at {}", appDef);
FileSystem fs = fileSystem.getFileSystem();
Path appPath = new Path(appDef);
- InputStream metainfoStream = SliderUtils.getApplicationResourceInputStream(
- fs, appPath, "metainfo.xml");
- if (metainfoStream == null) {
- log.error("metainfo.xml is unavailable at {}.", appDef);
- throw new FileNotFoundException("metainfo.xml is required in app package. " +
- appPath);
- }
- Metainfo metainfo = new MetainfoParser().parse(metainfoStream);
+ Metainfo metainfo = null;
+ MetainfoParser metainfoParser = new MetainfoParser();
+ InputStream metainfoJsonStream = SliderUtils.getApplicationResourceInputStream(
+ fs, appPath, "metainfo.json");
+ if (metainfoJsonStream == null) {
+ InputStream metainfoXMLStream = SliderUtils.getApplicationResourceInputStream(
+ fs, appPath, "metainfo.xml");
+ if (metainfoXMLStream != null) {
+ metainfo = metainfoParser.fromXmlStream(metainfoXMLStream);
+ }
+ } else {
+ metainfo = metainfoParser.fromJsonStream(metainfoJsonStream);
+ }
+ if (metainfo == null) {
+ log.error("metainfo is unavailable at {}.", appDef);
+ throw new FileNotFoundException("metainfo.xml/json is required in app package. " +
+ appPath);
+ }
return metainfo;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
index f4ace5f..194d6ff 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
@@ -49,13 +49,13 @@ public class ComponentCommandOrder {
if (requiredStates.size() > 0) {
Map<String, List<ComponentState>> compDep = dependencies.get(componentCmd.command);
if (compDep == null) {
- compDep = new HashMap<String, List<ComponentState>>();
+ compDep = new HashMap<>();
dependencies.put(componentCmd.command, compDep);
}
List<ComponentState> requirements = compDep.get(componentCmd.componentName);
if (requirements == null) {
- requirements = new ArrayList<ComponentState>();
+ requirements = new ArrayList<>();
compDep.put(componentCmd.componentName, requirements);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
index bc43d4b..cd5555f 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
@@ -16,29 +16,31 @@
*/
package org.apache.slider.providers.agent.application.metadata;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.codehaus.jackson.annotate.JsonProperty;
+
import java.util.ArrayList;
import java.util.List;
/**
* Application type defined in the metainfo
*/
-public class Application {
+public class Application implements Validate {
String name;
String comment;
String version;
String exportedConfigs;
- List<Component> components;
- List<ExportGroup> exportGroups;
- List<OSSpecific> osSpecifics;
- List<CommandOrder> commandOrders;
- List<ConfigFile> configFiles;
+ List<Component> components = new ArrayList<>();
+ List<ExportGroup> exportGroups = new ArrayList<>();
+ List<OSSpecific> osSpecifics = new ArrayList<>();
+ List<CommandOrder> commandOrders = new ArrayList<>();
+ List<ConfigFile> configFiles = new ArrayList<>();
+ List<Package> packages = new ArrayList<>();
public Application() {
- exportGroups = new ArrayList<ExportGroup>();
- components = new ArrayList<Component>();
- osSpecifics = new ArrayList<OSSpecific>();
- commandOrders = new ArrayList<CommandOrder>();
- configFiles = new ArrayList<ConfigFile>();
}
public String getName() {
@@ -73,18 +75,20 @@ public class Application {
this.exportedConfigs = exportedConfigs;
}
- public List<ConfigFile> getConfigFiles() {
- return configFiles;
- }
-
public void addConfigFile(ConfigFile configFile) {
this.configFiles.add(configFile);
}
+ @JsonProperty("configFiles")
+ public List<ConfigFile> getConfigFiles() {
+ return configFiles;
+ }
+
public void addComponent(Component component) {
components.add(component);
}
+ @JsonProperty("components")
public List<Component> getComponents() {
return components;
}
@@ -93,6 +97,7 @@ public class Application {
exportGroups.add(exportGroup);
}
+ @JsonProperty("exportGroups")
public List<ExportGroup> getExportGroups() {
return exportGroups;
}
@@ -101,6 +106,7 @@ public class Application {
osSpecifics.add(osSpecific);
}
+ @JsonIgnore
public List<OSSpecific> getOSSpecifics() {
return osSpecifics;
}
@@ -109,10 +115,16 @@ public class Application {
commandOrders.add(commandOrder);
}
- public List<CommandOrder> getCommandOrder() {
+ @JsonProperty("commandOrders")
+ public List<CommandOrder> getCommandOrders() {
return commandOrders;
}
+ @JsonProperty("packages")
+ public List<Package> getPackages() {
+ return packages;
+ }
+
@Override
public String toString() {
final StringBuilder sb =
@@ -128,4 +140,54 @@ public class Application {
sb.append('}');
return sb.toString();
}
+
+ public void validate(String version) throws SliderException {
+ if(SliderUtils.isUnset(version)) {
+ throw new BadCommandArgumentsException("schema version cannot be null");
+ }
+
+ Metainfo.checkNonNull(getName(), "name", "application");
+
+ Metainfo.checkNonNull(getVersion(), "version", "application");
+
+ if(getComponents().size() == 0) {
+ throw new SliderException("application must contain at least one component");
+ }
+
+ if(version.equals(Metainfo.VERSION_TWO_ZERO)) {
+ if(getPackages().size() > 0) {
+ throw new SliderException("packages is not supported in version " + version);
+ }
+ }
+
+ if(version.equals(Metainfo.VERSION_TWO_ONE)) {
+ if(getOSSpecifics().size() > 0) {
+ throw new SliderException("osSpecifics is not supported in version " + version);
+ }
+ }
+
+ for(CommandOrder co : getCommandOrders()) {
+ co.validate(version);
+ }
+
+ for(Component comp : getComponents()) {
+ comp.validate(version);
+ }
+
+ for(ConfigFile cf : getConfigFiles()) {
+ cf.validate(version);
+ }
+
+ for(ExportGroup eg : getExportGroups()) {
+ eg.validate(version);
+ }
+
+ for(Package pkg : getPackages()) {
+ pkg.validate(version);
+ }
+
+ for(OSSpecific os : getOSSpecifics()) {
+ os.validate(version);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java
index 825a104..40d8cc6 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java
@@ -16,10 +16,12 @@
*/
package org.apache.slider.providers.agent.application.metadata;
+import org.apache.slider.core.exceptions.SliderException;
+
/**
*
*/
-public class CommandOrder {
+public class CommandOrder implements Validate {
String command;
String requires;
@@ -51,4 +53,9 @@ public class CommandOrder {
sb.append('}');
return sb.toString();
}
+
+ public void validate(String version) throws SliderException {
+ Metainfo.checkNonNull(getCommand(), "command", "package");
+ Metainfo.checkNonNull(getRequires(), "requires", "package");
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java
index 612322f..9915ba1 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java
@@ -16,10 +16,12 @@
*/
package org.apache.slider.providers.agent.application.metadata;
+import org.apache.slider.core.exceptions.SliderException;
+
/**
- *
+ * CommandScript that implements all component commands
*/
-public class CommandScript {
+public class CommandScript implements Validate {
String script;
String scriptType;
long timeout;
@@ -62,4 +64,9 @@ public class CommandScript {
sb.append('}');
return sb.toString();
}
+
+ public void validate(String version) throws SliderException {
+ Metainfo.checkNonNull(getScript(), "script", "commandScript");
+ Metainfo.checkNonNull(getScriptType(), "scriptType", "commandScript");
+ }
}