You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by tt...@apache.org on 2016/12/02 16:15:25 UTC
ambari git commit: AMBARI-19018 - Services should be able to specify
their own resources subdirs for syncing to agents
Repository: ambari
Updated Branches:
refs/heads/trunk 60a6bd457 -> 232a9d87e
AMBARI-19018 - Services should be able to specify their own resources subdirs for syncing to agents
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/232a9d87
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/232a9d87
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/232a9d87
Branch: refs/heads/trunk
Commit: 232a9d87e72dfb87e0d25f6a4cba6738e27beb53
Parents: 60a6bd4
Author: Tim Thorpe <tt...@apache.org>
Authored: Fri Dec 2 08:15:09 2016 -0800
Committer: Tim Thorpe <tt...@apache.org>
Committed: Fri Dec 2 08:15:09 2016 -0800
----------------------------------------------------------------------
.../ambari_agent/CustomServiceOrchestrator.py | 7 +-
.../src/main/python/ambari_agent/FileCache.py | 12 ++-
.../TestCustomServiceOrchestrator.py | 30 +-----
.../test/python/ambari_agent/TestFileCache.py | 10 +-
.../ambari/server/agent/ExecutionCommand.java | 1 +
.../AmbariManagementControllerImpl.java | 9 ++
.../ambari/server/state/ComponentInfo.java | 14 +++
.../python/ambari_server/serverConfiguration.py | 14 +--
.../AMBARI_METRICS/0.1.0/metainfo.xml | 1 +
.../AmbariManagementControllerTest.java | 9 ++
.../server/orm/InMemoryDefaultTestModule.java | 5 +
ambari-server/src/test/python/TestMpacks.py | 102 +++++++++++++------
.../src/test/resources/dashboards/README.txt | 18 ++++
.../stacks/HDP/0.1/services/HDFS/metainfo.xml | 1 +
14 files changed, 150 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
index 7d61611..11c8cbe 100644
--- a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
+++ b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
@@ -154,12 +154,7 @@ class CustomServiceOrchestrator():
self.file_cache.get_host_scripts_base_dir(server_url_prefix)
hook_dir = self.file_cache.get_hook_base_dir(command, server_url_prefix)
base_dir = self.file_cache.get_service_base_dir(command, server_url_prefix)
- from ActionQueue import ActionQueue # To avoid cyclic dependency
- if self.COMMAND_TYPE in command and command[self.COMMAND_TYPE] == ActionQueue.EXECUTION_COMMAND:
- logger.info("Found it - " + str(command[self.COMMAND_TYPE]) + " yeah")
- # limiting to only EXECUTION_COMMANDs for now
- # TODO need a design for limiting to specific role/component such as METRICS_GRAFANA
- self.file_cache.get_dashboard_base_dir(server_url_prefix)
+ self.file_cache.get_custom_resources_subdir(command, server_url_prefix)
script_path = self.resolve_script_path(base_dir, script)
script_tuple = (script_path, base_dir)
http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-agent/src/main/python/ambari_agent/FileCache.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/FileCache.py b/ambari-agent/src/main/python/ambari_agent/FileCache.py
index 83ac373..a9ea8f6 100644
--- a/ambari-agent/src/main/python/ambari_agent/FileCache.py
+++ b/ambari-agent/src/main/python/ambari_agent/FileCache.py
@@ -45,7 +45,6 @@ class FileCache():
STACKS_CACHE_DIRECTORY="stacks"
COMMON_SERVICES_DIRECTORY="common-services"
CUSTOM_ACTIONS_CACHE_DIRECTORY="custom_actions"
- DASHBOARD_DIRECTORY="dashboards"
HOST_SCRIPTS_CACHE_DIRECTORY="host_scripts"
HASH_SUM_FILE=".hash"
ARCHIVE_NAME="archive.zip"
@@ -100,12 +99,17 @@ class FileCache():
server_url_prefix)
- def get_dashboard_base_dir(self, server_url_prefix):
+ def get_custom_resources_subdir(self, command, server_url_prefix):
"""
- Returns a base directory for dashboards
+ Returns a custom directory which must be a subdirectory of the resources dir
"""
+ try:
+ custom_dir = command['commandParams']['custom_folder']
+ except KeyError:
+ return None
+
return self.provide_directory(self.cache_dir,
- self.DASHBOARD_DIRECTORY,
+ custom_dir,
server_url_prefix)
http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
index 0304adc..5323d9a 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
@@ -220,7 +220,7 @@ class TestCustomServiceOrchestrator(TestCase):
except AgentException:
pass # Expected
- @patch.object(FileCache, "get_dashboard_base_dir")
+ @patch.object(FileCache, "get_custom_resources_subdir")
@patch.object(CustomServiceOrchestrator, "resolve_script_path")
@patch.object(CustomServiceOrchestrator, "resolve_hook_script_path")
@patch.object(FileCache, "get_host_scripts_base_dir")
@@ -235,7 +235,7 @@ class TestCustomServiceOrchestrator(TestCase):
get_host_scripts_base_dir_mock,
resolve_hook_script_path_mock,
resolve_script_path_mock,
- get_dashboard_base_dir_mock):
+ get_custom_resources_subdir_mock):
FileCache_mock.return_value = None
command = {
@@ -267,7 +267,6 @@ class TestCustomServiceOrchestrator(TestCase):
unix_process_id = 111
orchestrator.commands_in_progress = {command['taskId']: unix_process_id}
get_hook_base_dir_mock.return_value = "/hooks/"
- get_dashboard_base_dir_mock.return_value = "/dashboards/"
# normal run case
run_file_mock.return_value = {
'stdout' : 'sss',
@@ -278,7 +277,6 @@ class TestCustomServiceOrchestrator(TestCase):
self.assertEqual(ret['exitcode'], 0)
self.assertTrue(run_file_mock.called)
self.assertEqual(run_file_mock.call_count, 3)
- self.assertTrue(get_dashboard_base_dir_mock.called)
run_file_mock.reset_mock()
@@ -301,25 +299,6 @@ class TestCustomServiceOrchestrator(TestCase):
run_file_mock.reset_mock()
- # For role=METRICS_GRAFANA, dashboards should be sync'd
- command['role'] = 'METRICS_GRAFANA'
- get_dashboard_base_dir_mock.reset_mock()
- get_dashboard_base_dir_mock.return_value = "/dashboards/"
-
- run_file_mock.return_value = {
- 'stdout' : 'sss',
- 'stderr' : 'eee',
- 'exitcode': 0,
- }
- ret = orchestrator.runCommand(command, "out.txt", "err.txt")
- self.assertEqual(ret['exitcode'], 0)
- self.assertTrue(run_file_mock.called)
- self.assertEqual(run_file_mock.call_count, 3)
- self.assertTrue(get_dashboard_base_dir_mock.called)
-
- command['role'] = 'REGION_SERVER'
- run_file_mock.reset_mock()
-
# unknown script type case
command['commandParams']['script_type'] = "SOME_TYPE"
ret = orchestrator.runCommand(command, "out.txt", "err.txt")
@@ -332,7 +311,6 @@ class TestCustomServiceOrchestrator(TestCase):
pass
- @patch.object(FileCache, "get_dashboard_base_dir")
@patch("ambari_commons.shell.kill_process_with_children")
@patch.object(CustomServiceOrchestrator, "resolve_script_path")
@patch.object(CustomServiceOrchestrator, "resolve_hook_script_path")
@@ -347,8 +325,7 @@ class TestCustomServiceOrchestrator(TestCase):
get_hook_base_dir_mock, get_service_base_dir_mock,
get_host_scripts_base_dir_mock,
resolve_hook_script_path_mock, resolve_script_path_mock,
- kill_process_with_children_mock,
- get_dashboard_base_dir_mock):
+ kill_process_with_children_mock):
FileCache_mock.return_value = None
command = {
'role' : 'REGION_SERVER',
@@ -378,7 +355,6 @@ class TestCustomServiceOrchestrator(TestCase):
unix_process_id = 111
orchestrator.commands_in_progress = {command['taskId']: unix_process_id}
get_hook_base_dir_mock.return_value = "/hooks/"
- get_dashboard_base_dir_mock.return_value = "/dashboards/"
run_file_mock_return_value = {
'stdout' : 'killed',
'stderr' : 'killed',
http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-agent/src/test/python/ambari_agent/TestFileCache.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestFileCache.py b/ambari-agent/src/test/python/ambari_agent/TestFileCache.py
index fbefc2b..00f6b69 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestFileCache.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestFileCache.py
@@ -119,10 +119,16 @@ class TestFileCache(TestCase):
@patch.object(FileCache, "provide_directory")
- def test_get_dashboard_base_dir(self, provide_directory_mock):
+ def test_get_custom_resources_subdir(self, provide_directory_mock):
provide_directory_mock.return_value = "dummy value"
fileCache = FileCache(self.config)
- res = fileCache.get_dashboard_base_dir("server_url_pref")
+ command = {
+ 'commandParams': {
+ 'custom_folder' : 'dashboards'
+ }
+ }
+
+ res = fileCache.get_custom_resources_subdir(command, "server_url_pref")
self.assertEquals(
pprint.pformat(provide_directory_mock.call_args_list[0][0]),
"('/var/lib/ambari-agent/cache', 'dashboards', 'server_url_pref')")
http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
index 29737ee..ef1ee4f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
@@ -343,6 +343,7 @@ public class ExecutionCommand extends AgentCommand {
String SCRIPT_TYPE = "script_type";
String SERVICE_PACKAGE_FOLDER = "service_package_folder";
String HOOKS_FOLDER = "hooks_folder";
+ String CUSTOM_FOLDER = "custom_folder";
String STACK_NAME = "stack_name";
String SERVICE_TYPE = "service_type";
String STACK_VERSION = "stack_version";
http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index ac9e405..0448b9f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -25,6 +25,7 @@ import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CLIENTS_TO_UPDATE_CONFIGS;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_RETRY_ENABLED;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT;
+import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CUSTOM_FOLDER;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST;
import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER;
@@ -2217,6 +2218,14 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
serviceInfo.getServicePackageFolder());
commandParams.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder());
+ String customCacheDirectory = componentInfo.getCustomFolder();
+ if (customCacheDirectory != null) {
+ File customCache = new File(configs.getResourceDirPath(), customCacheDirectory);
+ if (customCache.exists() && customCache.isDirectory()) {
+ commandParams.put(CUSTOM_FOLDER, customCacheDirectory);
+ }
+ }
+
String clusterName = cluster.getClusterName();
if (customCommandExecutionHelper.isTopologyRefreshRequired(roleCommand.name(), clusterName, serviceName)) {
commandParams.put(ExecutionCommand.KeyNames.REFRESH_TOPOLOGY, "True");
http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-server/src/main/java/org/apache/ambari/server/state/ComponentInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ComponentInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ComponentInfo.java
index 1e494b4..bd9b798 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ComponentInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ComponentInfo.java
@@ -133,6 +133,9 @@ public class ComponentInfo {
private String timelineAppid;
+ @XmlElement(name="customFolder")
+ private String customFolder;
+
public ComponentInfo() {
}
@@ -158,6 +161,7 @@ public class ComponentInfo {
clientConfigFiles = prototype.clientConfigFiles;
timelineAppid = prototype.timelineAppid;
reassignAllowed = prototype.reassignAllowed;
+ customFolder = prototype.customFolder;
}
public String getName() {
@@ -396,6 +400,14 @@ public class ComponentInfo {
this.reassignAllowed = reassignAllowed;
}
+ public String getCustomFolder() {
+ return customFolder;
+ }
+
+ public void setCustomFolder(String customFolder) {
+ this.customFolder = customFolder;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) return true;
@@ -426,6 +438,7 @@ public class ComponentInfo {
if (name != null ? !name.equals(that.name) : that.name != null) return false;
if (clientConfigFiles != null ? !clientConfigFiles.equals(that.clientConfigFiles) :
that.clientConfigFiles != null) return false;
+ if (customFolder != null ? !customFolder.equals(that.customFolder) : that.customFolder != null) return false;
return true;
}
@@ -450,6 +463,7 @@ public class ComponentInfo {
result = 31 * result + (clientConfigFiles != null ? clientConfigFiles.hashCode() : 0);
// NULL = 0, TRUE = 2, FALSE = 1
result = 31 * result + (versionAdvertisedField != null ? (versionAdvertisedField.booleanValue() ? 2 : 1) : 0);
+ result = 31 * result + (customFolder != null ? customFolder.hashCode() : 0);
return result;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-server/src/main/python/ambari_server/serverConfiguration.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/ambari_server/serverConfiguration.py b/ambari-server/src/main/python/ambari_server/serverConfiguration.py
index 9596e0d..6a6c89e 100644
--- a/ambari-server/src/main/python/ambari_server/serverConfiguration.py
+++ b/ambari-server/src/main/python/ambari_server/serverConfiguration.py
@@ -186,7 +186,6 @@ SETUP_OR_UPGRADE_MSG = "- If this is a new setup, then run the \"ambari-server s
DEFAULT_DB_NAME = "ambari"
SECURITY_KEYS_DIR = "security.server.keys_dir"
-DASHBOARD_PATH_PROPERTY = 'dashboards.path'
EXTENSION_PATH_PROPERTY = 'extensions.path'
COMMON_SERVICES_PATH_PROPERTY = 'common.services.path'
MPACKS_STAGING_PATH_PROPERTY = 'mpacks.staging.path'
@@ -398,8 +397,8 @@ class ServerConfigDefaults(object):
self.EXTENSION_LOCATION_DEFAULT = ""
self.COMMON_SERVICES_LOCATION_DEFAULT = ""
self.MPACKS_STAGING_LOCATION_DEFAULT = ""
- self.DASHBOARD_LOCATION_DEFAULT = ""
self.SERVER_TMP_DIR_DEFAULT = ""
+ self.DASHBOARD_DIRNAME = "dashboards"
self.DEFAULT_VIEWS_DIR = ""
@@ -470,7 +469,6 @@ class ServerConfigDefaultsWindows(ServerConfigDefaults):
self.EXTENSION_LOCATION_DEFAULT = "resources\\extensions"
self.COMMON_SERVICES_LOCATION_DEFAULT = "resources\\common-services"
self.MPACKS_STAGING_LOCATION_DEFAULT = "resources\\mpacks"
- self.DASHBOARD_LOCATION_DEFAULT = "resources\\dashboards"
self.SERVER_TMP_DIR_DEFAULT = "data\\tmp"
self.DEFAULT_VIEWS_DIR = "resources\\views"
@@ -557,7 +555,6 @@ class ServerConfigDefaultsLinux(ServerConfigDefaults):
self.EXTENSION_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/extensions")
self.COMMON_SERVICES_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/common-services")
self.MPACKS_STAGING_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/mpacks")
- self.DASHBOARD_LOCATION_DEFAULT = AmbariPath.get("/var/lib/ambari-server/resources/dashboards")
self.SERVER_TMP_DIR_DEFAULT = AmbariPath.get("/var/lib/ambari-server/data/tmp")
self.DEFAULT_VIEWS_DIR = AmbariPath.get("/var/lib/ambari-server/resources/views")
@@ -1448,13 +1445,8 @@ def get_mpacks_staging_location(properties):
# Dashboard location
#
def get_dashboard_location(properties):
- try:
- dashboard_location = properties[DASHBOARD_PATH_PROPERTY]
- except KeyError:
- dashboard_location = configDefaults.DASHBOARD_LOCATION_DEFAULT
-
- if not dashboard_location:
- dashboard_location = configDefaults.DASHBOARD_LOCATION_DEFAULT
+ resources_dir = get_resources_location(properties)
+ dashboard_location = os.path.join(resources_dir, configDefaults.DASHBOARD_DIRNAME)
return dashboard_location
#
http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
index 43a4320..740a91a 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml
@@ -101,6 +101,7 @@
<category>MASTER</category>
<cardinality>0-1</cardinality>
<versionAdvertised>false</versionAdvertised>
+ <customFolder>dashboards</customFolder>
<commandScript>
<script>scripts/metrics_grafana.py</script>
<scriptType>PYTHON</scriptType>
http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index 0fdaa46..db64f17 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -1283,6 +1283,15 @@ public class AmbariManagementControllerTest {
assertEquals("false", ec.getCommandParams().get("command_retry_enabled"));
Map<String, Set<String>> chInfo = ec.getClusterHostInfo();
assertTrue(chInfo.containsKey("namenode_host"));
+ assertFalse(ec.getCommandParams().containsKey("custom_folder"));
+
+ ec = controller.getExecutionCommand(cluster,
+ s1.getServiceComponent("DATANODE").getServiceComponentHost(host1),
+ RoleCommand.START);
+ assertEquals(cluster1, ec.getClusterName());
+ assertNotNull(ec.getCommandParams());
+ assertTrue(ec.getCommandParams().containsKey("custom_folder"));
+ assertEquals("dashboards", ec.getCommandParams().get("custom_folder"));
}
@Test
http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-server/src/test/java/org/apache/ambari/server/orm/InMemoryDefaultTestModule.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/orm/InMemoryDefaultTestModule.java b/ambari-server/src/test/java/org/apache/ambari/server/orm/InMemoryDefaultTestModule.java
index 6773176..a4a9e26 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/orm/InMemoryDefaultTestModule.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/orm/InMemoryDefaultTestModule.java
@@ -90,6 +90,7 @@ public class InMemoryDefaultTestModule extends AbstractModule {
String stacks = "src/test/resources/stacks";
String version = "src/test/resources/version";
String sharedResourcesDir = "src/test/resources/";
+ String resourcesDir = "src/test/resources/";
if (System.getProperty("os.name").contains("Windows")) {
stacks = ClassLoader.getSystemClassLoader().getResource("stacks").getPath();
version = new File(new File(ClassLoader.getSystemClassLoader().getResource("").getPath()).getParent(), "version").getPath();
@@ -116,6 +117,10 @@ public class InMemoryDefaultTestModule extends AbstractModule {
properties.setProperty(Configuration.SHARED_RESOURCES_DIR.getKey(), sharedResourcesDir);
}
+ if (!properties.containsKey(Configuration.RESOURCES_DIR.getKey())) {
+ properties.setProperty(Configuration.RESOURCES_DIR.getKey(), resourcesDir);
+ }
+
try {
install(Modules.override(new BeanDefinitionsCachingTestControllerModule(properties)).with(new AbstractModule() {
@Override
http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-server/src/test/python/TestMpacks.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestMpacks.py b/ambari-server/src/test/python/TestMpacks.py
index ba37856..32db005 100644
--- a/ambari-server/src/test/python/TestMpacks.py
+++ b/ambari-server/src/test/python/TestMpacks.py
@@ -64,7 +64,7 @@ def get_configs():
serverConfiguration.STACK_LOCATION_KEY : "/var/lib/ambari-server/resources/stacks",
serverConfiguration.COMMON_SERVICES_PATH_PROPERTY : "/var/lib/ambari-server/resources/common-services",
serverConfiguration.EXTENSION_PATH_PROPERTY : "/var/lib/ambari-server/resources/extensions",
- serverConfiguration.DASHBOARD_PATH_PROPERTY : "/var/lib/ambari-server/resources/dashboards",
+ serverConfiguration.RESOURCES_DIR_PROPERTY : "/var/lib/ambari-server/resources",
serverConfiguration.MPACKS_STAGING_PATH_PROPERTY : mpacks_directory,
serverConfiguration.SERVER_TMP_DIR_PROPERTY : "/tmp",
serverConfiguration.JDBC_DATABASE_PROPERTY: "postgres"
@@ -100,9 +100,11 @@ class TestMpacks(TestCase):
fail = True
self.assertTrue(fail)
+ @patch("os.path.exists")
@patch("ambari_server.setupMpacks.get_YN_input")
@patch("ambari_server.setupMpacks.run_mpack_install_checker")
- def test_validate_purge(self, run_mpack_install_checker_mock, get_YN_input_mock):
+ @patch("ambari_server.setupMpacks.get_ambari_properties")
+ def test_validate_purge(self, get_ambari_properties_mock, run_mpack_install_checker_mock, get_YN_input_mock, os_path_exists_mock):
options = self._create_empty_options_mock()
options.purge = True
purge_list = options.purge_list.split(',')
@@ -112,6 +114,7 @@ class TestMpacks(TestCase):
replay_mode = False
run_mpack_install_checker_mock.return_value = (0, "No errors found", "")
get_YN_input_mock.return_value = True
+ os_path_exists_mock.return_value = True
fail = False
try:
@@ -160,22 +163,28 @@ class TestMpacks(TestCase):
extensions_directory = configs[serverConfiguration.EXTENSION_PATH_PROPERTY]
common_services_directory = configs[serverConfiguration.COMMON_SERVICES_PATH_PROPERTY]
mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
- os_path_exists_mock.return_value = False
+ os_path_exists_mock.side_effect = [True]
purge_stacks_and_mpacks(None)
- os_path_exists_calls = []
+ os_path_exists_calls = [
+ call('/var/lib/ambari-server/resources'),
+ ]
os_path_exists_mock.assert_has_calls(os_path_exists_calls)
+ os_path_exists_mock.side_effect = [True, False, False]
purge_stacks_and_mpacks(options.purge_list.split(","))
os_path_exists_calls = [
+ call('/var/lib/ambari-server/resources'),
call(stacks_directory),
call(mpacks_directory)
]
os_path_exists_mock.assert_has_calls(os_path_exists_calls)
options.purge_list = ",".join([STACK_DEFINITIONS_RESOURCE_NAME, SERVICE_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME])
+ os_path_exists_mock.side_effect = [True, False, False, False]
purge_stacks_and_mpacks(options.purge_list.split(","))
os_path_exists_calls = [
+ call('/var/lib/ambari-server/resources'),
call(stacks_directory),
call(common_services_directory),
call(mpacks_directory)
@@ -183,8 +192,10 @@ class TestMpacks(TestCase):
os_path_exists_mock.assert_has_calls(os_path_exists_calls)
options.purge_list = ",".join([STACK_DEFINITIONS_RESOURCE_NAME, EXTENSION_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME])
+ os_path_exists_mock.side_effect = [True, False, False, False]
purge_stacks_and_mpacks(options.purge_list.split(","))
os_path_exists_calls = [
+ call('/var/lib/ambari-server/resources'),
call(stacks_directory),
call(extensions_directory),
call(mpacks_directory)
@@ -193,10 +204,13 @@ class TestMpacks(TestCase):
options.purge_list = ",".join([STACK_DEFINITIONS_RESOURCE_NAME, SERVICE_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME])
options.replay_mode = True
+ os_path_exists_mock.side_effect = [True, False, False, False]
purge_stacks_and_mpacks(options.purge_list.split(","))
os_path_exists_calls = [
+ call('/var/lib/ambari-server/resources'),
call(stacks_directory),
- call(common_services_directory)
+ call(common_services_directory),
+ call(mpacks_directory)
]
os_path_exists_mock.assert_has_calls(os_path_exists_calls)
@@ -266,10 +280,11 @@ class TestMpacks(TestCase):
get_ambari_version_mock.return_value = "2.4.0.0"
run_os_command_mock.return_value = (0, "", "")
mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
- """
+
os_path_exists_calls = [call('/tmp/mystack.tar.gz'),
call('mpacks/mystack-ambari-mpack-1.0.0.0/mpack.json'),
call('mpacks/mystack-ambari-mpack-1.0.0.0/hooks/before_install.py'),
+ call('/var/lib/ambari-server/resources'),
call('/var/lib/ambari-server/resources/stacks'),
call('/var/lib/ambari-server/resources/extensions'),
call('/var/lib/ambari-server/resources/common-services'),
@@ -277,12 +292,14 @@ class TestMpacks(TestCase):
call(mpacks_directory + '/cache'),
call('/var/lib/ambari-server/resources/dashboards'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0'),
+ call('/var/lib/ambari-server/resources'),
call('/var/lib/ambari-server/resources/common-services/SERVICEA'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/common-services/SERVICEA/1.0/dashboards'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/common-services/SERVICEA/2.0/dashboards'),
call('/var/lib/ambari-server/resources/common-services/SERVICEB'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/common-services/SERVICEB/1.0.0/dashboards'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/common-services/SERVICEB/2.0.0/dashboards'),
+ call('/var/lib/ambari-server/resources'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0/services'),
@@ -295,22 +312,26 @@ class TestMpacks(TestCase):
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/stacks/MYSTACK/2.0/services/SERVICEA/dashboards'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/stacks/MYSTACK/2.0/services/SERVICEB/dashboards'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/hooks/after_install.py')]
- """
- os_path_exists_mock.side_effect = [True, True, True, False, True, False, False, False, False,
- False, False, False, False, False, False,
- False, False, False, False, False, False, False, False,
- False, False, False, False, True]
+
+ os_path_exists_mock.side_effect = [True, True, True, True, False, True, False, False, False, False,
+ False, True, False, False, False, False, False, False, True, False,
+ False, False, False, False, False, False, False, False, False, False,
+ True]
get_ambari_properties_mock.return_value = configs
shutil_move_mock.return_value = True
- install_mpack(options)
+ try:
+ install_mpack(options)
+ except Exception as e:
+ print e
stacks_directory = configs[serverConfiguration.STACK_LOCATION_KEY]
common_services_directory = configs[serverConfiguration.COMMON_SERVICES_PATH_PROPERTY]
extensions_directory = configs[serverConfiguration.EXTENSION_PATH_PROPERTY]
mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
- dashboards_directory = serverConfiguration.get_dashboard_location(configs)
mpacks_staging_directory = os.path.join(mpacks_directory, "mystack-ambari-mpack-1.0.0.0")
+ resources_directory = configs[serverConfiguration.RESOURCES_DIR_PROPERTY]
+ dashboards_directory = os.path.join(resources_directory, "dashboards")
run_os_command_calls = [
call([
@@ -377,6 +398,7 @@ class TestMpacks(TestCase):
"SERVICEB", None)
]
+ os_path_exists_mock.assert_has_calls(os_path_exists_calls)
self.assertTrue(purge_stacks_and_mpacks_mock.called)
run_os_command_mock.assert_has_calls(run_os_command_calls)
os_mkdir_mock.assert_has_calls(os_mkdir_calls)
@@ -403,8 +425,8 @@ class TestMpacks(TestCase):
expand_mpack_mock.return_value = "mpacks/myextension-ambari-mpack-1.0.0.0"
get_ambari_version_mock.return_value = "2.4.0.0"
- os_path_exists_mock.side_effect = [True, True, True, False, True, False, False, False,
- False, True, False, False, False]
+ os_path_exists_mock.side_effect = [True, True, True, True, False, True, False, False, False,
+ False, True, True, False, False, False]
get_ambari_properties_mock.return_value = configs
shutil_move_mock.return_value = True
@@ -413,10 +435,12 @@ class TestMpacks(TestCase):
extensions_directory = configs[serverConfiguration.EXTENSION_PATH_PROPERTY]
mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
mpacks_staging_directory = os.path.join(mpacks_directory, "myextension-ambari-mpack-1.0.0.0")
- dashboards_directory = serverConfiguration.get_dashboard_location(configs)
+ resources_directory = configs[serverConfiguration.RESOURCES_DIR_PROPERTY]
+ dashboards_directory = os.path.join(resources_directory, "dashboards")
os_path_exists_calls = [call('/tmp/myextension.tar.gz'),
call('mpacks/myextension-ambari-mpack-1.0.0.0/mpack.json'),
+ call('/var/lib/ambari-server/resources'),
call('/var/lib/ambari-server/resources/stacks'),
call('/var/lib/ambari-server/resources/extensions'),
call('/var/lib/ambari-server/resources/common-services'),
@@ -424,6 +448,7 @@ class TestMpacks(TestCase):
call(mpacks_directory + '/cache'),
call('/var/lib/ambari-server/resources/dashboards'),
call(mpacks_directory + '/myextension-ambari-mpack-1.0.0.0'),
+ call('/var/lib/ambari-server/resources'),
call('/var/lib/ambari-server/resources/extensions'),
call('/var/lib/ambari-server/resources/extensions/MYEXTENSION'),
call(mpacks_directory + '/myextension-ambari-mpack-1.0.0.0/extensions/MYEXTENSION/1.0/services'),
@@ -476,9 +501,9 @@ class TestMpacks(TestCase):
expand_mpack_mock.return_value = "mpacks/myservice-ambari-mpack-1.0.0.0"
get_ambari_version_mock.return_value = "2.4.0.0"
- os_path_exists_mock.side_effect = [True, True, True, True, True, True,
- True, True, False, False, False, False,
- True, True, True, False, True, True,
+ os_path_exists_mock.side_effect = [True, True, True, True, True, True, True,
+ True, True, False, False, True, False, False,
+ True, True, True, True, False, True, True,
True, False]
get_ambari_properties_mock.return_value = configs
@@ -491,11 +516,13 @@ class TestMpacks(TestCase):
common_services_directory = configs[serverConfiguration.COMMON_SERVICES_PATH_PROPERTY]
mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
mpacks_staging_directory = os.path.join(mpacks_directory, "myservice-ambari-mpack-1.0.0.0")
- dashboards_directory = serverConfiguration.get_dashboard_location(configs)
+ resources_directory = configs[serverConfiguration.RESOURCES_DIR_PROPERTY]
+ dashboards_directory = os.path.join(resources_directory, "dashboards")
os_path_exists_calls = [call('/tmp/myservice.tar.gz'),
call('mpacks/myservice-ambari-mpack-1.0.0.0/mpack.json'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0'),
+ call('/var/lib/ambari-server/resources'),
call('/var/lib/ambari-server/resources/stacks'),
call('/var/lib/ambari-server/resources/extensions'),
call('/var/lib/ambari-server/resources/common-services'),
@@ -503,8 +530,10 @@ class TestMpacks(TestCase):
call(mpacks_directory + '/cache'),
call('/var/lib/ambari-server/resources/dashboards'),
call(mpacks_directory + '/myservice-ambari-mpack-1.0.0.0'),
+ call('/var/lib/ambari-server/resources'),
call('/var/lib/ambari-server/resources/common-services/MYSERVICE'),
call(mpacks_directory + '/myservice-ambari-mpack-1.0.0.0/common-services/MYSERVICE/1.0.0/dashboards'),
+ call('/var/lib/ambari-server/resources'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0/services'),
@@ -562,14 +591,14 @@ class TestMpacks(TestCase):
get_ambari_version_mock.return_value = "2.4.0.0"
run_os_command_mock.return_value = (0, "", "")
mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
- os_path_exists_mock.side_effect = [True, True, True, True, True, True, True, True,
- True, True, True, True, True, True, True, False,
- False, True, False, False, True, False, False,
- False, False, False, True, True, True, False,
- True, True, False, True, True, False, False,
- False, False, False, True, True, True, True,
- True, True, True, False, True, False, True, True,
- True, True, True, True]
+ os_path_exists_mock.side_effect = [True, True, True, True, True, True, True, True, True, True,
+ True, True, True, True, True, True, True, False, False, True,
+ True, False, False, True, False, False, False, False, False, True,
+ True, True, True, False, True, True, False, True, True, False,
+ False, False, False, False, True, True, True, True, True, True,
+ True, False, True, False, True, True, True, True, True, True,
+ True]
+
get_ambari_properties_mock.return_value = configs
shutil_move_mock.return_value = True
@@ -579,10 +608,12 @@ class TestMpacks(TestCase):
common_services_directory = configs[serverConfiguration.COMMON_SERVICES_PATH_PROPERTY]
mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
mpacks_staging_directory = os.path.join(mpacks_directory, "mystack-ambari-mpack-1.0.0.1")
- dashboards_directory = serverConfiguration.get_dashboard_location(configs)
+ resources_directory = configs[serverConfiguration.RESOURCES_DIR_PROPERTY]
+ dashboards_directory = os.path.join(resources_directory, "dashboards")
os_path_exists_calls = [call('/tmp/mystack-1.0.0.1.tar.gz'),
call('mpacks/mystack-ambari-mpack-1.0.0.1/mpack.json'),
+ call('/var/lib/ambari-server/resources'),
call(mpacks_directory),
call(mpacks_directory + '/myextension-ambari-mpack-1.0.0.0/mpack.json'),
call(mpacks_directory + '/myservice-ambari-mpack-1.0.0.0/mpack.json'),
@@ -591,6 +622,7 @@ class TestMpacks(TestCase):
call('/tmp/mystack-1.0.0.1.tar.gz'),
call('mpacks/mystack-ambari-mpack-1.0.0.1/mpack.json'),
call('mpacks/mystack-ambari-mpack-1.0.0.1/hooks/before_upgrade.py'),
+ call('/var/lib/ambari-server/resources'),
call('/var/lib/ambari-server/resources/stacks'),
call('/var/lib/ambari-server/resources/extensions'),
call('/var/lib/ambari-server/resources/common-services'),
@@ -598,6 +630,7 @@ class TestMpacks(TestCase):
call(mpacks_directory + '/cache'),
call('/var/lib/ambari-server/resources/dashboards'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1'),
+ call('/var/lib/ambari-server/resources'),
call('/var/lib/ambari-server/resources/common-services/SERVICEA'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/common-services/SERVICEA/1.0/dashboards'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/common-services/SERVICEA/2.0/dashboards'),
@@ -607,6 +640,7 @@ class TestMpacks(TestCase):
call('/var/lib/ambari-server/resources/common-services/SERVICEC'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/common-services/SERVICEC/1.0.0/dashboards'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/common-services/SERVICEC/2.0.0/dashboards'),
+ call('/var/lib/ambari-server/resources'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0/services'),
@@ -631,6 +665,7 @@ class TestMpacks(TestCase):
call('/var/lib/ambari-server/resources/dashboards/grafana-dashboards/SERVICEC'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/stacks/MYSTACK/3.0/services/SERVICEC/dashboards/service-metrics/SERVICEC.txt'),
call('/var/lib/ambari-server/resources/dashboards/service-metrics/SERVICEC.txt'),
+ call('/var/lib/ambari-server/resources'),
call(mpacks_directory),
call(mpacks_directory + '/myextension-ambari-mpack-1.0.0.0/mpack.json'),
call(mpacks_directory + '/myservice-ambari-mpack-1.0.0.0/mpack.json'),
@@ -775,7 +810,7 @@ class TestMpacks(TestCase):
serverConfiguration.COMMON_SERVICES_PATH_PROPERTY : os.path.join(uninstall_directory, "common-services"),
serverConfiguration.EXTENSION_PATH_PROPERTY : os.path.join(uninstall_directory, "extensions"),
serverConfiguration.MPACKS_STAGING_PATH_PROPERTY : mpacks_directory,
- serverConfiguration.DASHBOARD_PATH_PROPERTY : os.path.join(uninstall_directory, "dashboards"),
+ serverConfiguration.RESOURCES_DIR_PROPERTY : uninstall_directory,
serverConfiguration.SERVER_TMP_DIR_PROPERTY : "/tmp"
}
@@ -784,7 +819,8 @@ class TestMpacks(TestCase):
stacks_directory = fake_configs[serverConfiguration.STACK_LOCATION_KEY]
extension_directory = fake_configs[serverConfiguration.EXTENSION_PATH_PROPERTY]
common_services_directory = fake_configs[serverConfiguration.COMMON_SERVICES_PATH_PROPERTY]
- dashboard_directory = fake_configs[serverConfiguration.DASHBOARD_PATH_PROPERTY]
+ resources_directory = fake_configs[serverConfiguration.RESOURCES_DIR_PROPERTY]
+ dashboards_directory = os.path.join(resources_directory, "dashboards")
_uninstall_mpack("mystack-ambari-mpack", "1.0.0.1")
@@ -794,8 +830,8 @@ class TestMpacks(TestCase):
call(os.path.join(stacks_directory, "2.0/files/metainfo2.xml")),
call(os.path.join(extension_directory, "SERVICEB")),
call(os.path.join(common_services_directory, "SERVICEB")),
- call(os.path.join(dashboard_directory, "SERVICEB")),
- call(os.path.join(dashboard_directory, "files/STORM.txt"))]
+ call(os.path.join(dashboards_directory, "SERVICEB")),
+ call(os.path.join(dashboards_directory, "files/STORM.txt"))]
def _create_empty_options_mock(self):
options = MagicMock()
http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-server/src/test/resources/dashboards/README.txt
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/dashboards/README.txt b/ambari-server/src/test/resources/dashboards/README.txt
new file mode 100644
index 0000000..919d510
--- /dev/null
+++ b/ambari-server/src/test/resources/dashboards/README.txt
@@ -0,0 +1,18 @@
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+Test directory for sync'ing a components custom folder - AMBARI-19018
http://git-wip-us.apache.org/repos/asf/ambari/blob/232a9d87/ambari-server/src/test/resources/stacks/HDP/0.1/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/HDP/0.1/services/HDFS/metainfo.xml b/ambari-server/src/test/resources/stacks/HDP/0.1/services/HDFS/metainfo.xml
index f4195a8..7629552 100644
--- a/ambari-server/src/test/resources/stacks/HDP/0.1/services/HDFS/metainfo.xml
+++ b/ambari-server/src/test/resources/stacks/HDP/0.1/services/HDFS/metainfo.xml
@@ -57,6 +57,7 @@
<name>DATANODE</name>
<category>SLAVE</category>
<cardinality>1+</cardinality>
+ <customFolder>dashboards</customFolder>
<commandScript>
<script>scripts/datanode.py</script>
<scriptType>PYTHON</scriptType>