You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/06/11 19:02:38 UTC
[2/2] git commit: SLIDER-115. Move default data folder location for
HBase and Accumulo as sub-folder of the cluster directory
SLIDER-115. Move default data folder location for HBase and Accumulo as sub-folder of the cluster directory
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3dfb3197
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3dfb3197
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3dfb3197
Branch: refs/heads/feature/slider_119_agent_cleanup
Commit: 3dfb31978a2a7f85f6b8e98317ab641a5eda060b
Parents: 2079fb8
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Jun 11 10:02:06 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Jun 11 10:02:06 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo-v1_5/appConfig.json | 2 +-
app-packages/hbase-v0_96/appConfig.json | 4 ++--
.../main/python/agent/CustomServiceOrchestrator.py | 2 ++
.../src/test/python/agent/TestActionQueue.py | 1 +
.../python/agent/TestCustomServiceOrchestrator.py | 2 +-
.../providers/agent/AgentProviderService.java | 17 +++++++++++------
.../providers/agent/TestAgentProviderService.java | 1 +
7 files changed, 19 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dfb3197/app-packages/accumulo-v1_5/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo-v1_5/appConfig.json b/app-packages/accumulo-v1_5/appConfig.json
index 52f8624..f453b91 100644
--- a/app-packages/accumulo-v1_5/appConfig.json
+++ b/app-packages/accumulo-v1_5/appConfig.json
@@ -25,7 +25,7 @@
"site.global.accumulo_root_password": "secret",
"site.global.user_group": "hadoop",
"site.global.security_enabled": "false",
- "site.accumulo-site.instance.dfs.dir": "/apps/accumulo/data",
+ "site.accumulo-site.instance.dfs.dir": "${DEFAULT_DATA_DIR}/data",
"site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
"site.accumulo-site.instance.secret": "DEFAULT",
"site.accumulo-site.tserver.memory.maps.max": "80M",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dfb3197/app-packages/hbase-v0_96/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/appConfig.json b/app-packages/hbase-v0_96/appConfig.json
index fd884cb..1d828e2 100644
--- a/app-packages/hbase-v0_96/appConfig.json
+++ b/app-packages/hbase-v0_96/appConfig.json
@@ -23,8 +23,8 @@
"site.hbase-site.hbase.hstore.flush.retries.number": "120",
"site.hbase-site.hbase.client.keyvalue.maxsize": "10485760",
"site.hbase-site.hbase.hstore.compactionThreshold": "3",
- "site.hbase-site.hbase.rootdir": "${NN_URI}/apps/hbase/data",
- "site.hbase-site.hbase.stagingdir": "${NN_URI}/apps/hbase/staging",
+ "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}/data",
+ "site.hbase-site.hbase.stagingdir": "${DEFAULT_DATA_DIR}/staging",
"site.hbase-site.hbase.regionserver.handler.count": "60",
"site.hbase-site.hbase.regionserver.global.memstore.lowerLimit": "0.38",
"site.hbase-site.hbase.hregion.memstore.block.multiplier": "2",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dfb3197/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 328084d..6296033 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -88,6 +88,7 @@ class CustomServiceOrchestrator():
py_file_list = [script_tuple]
# filter None values
filtered_py_file_list = [i for i in py_file_list if i]
+ logger_level = logging.getLevelName(logger.level)
# Executing hooks and script
ret = None
@@ -101,6 +102,7 @@ class CustomServiceOrchestrator():
ret = self.python_executor.run_file(py_file, script_params,
tmpoutfile, tmperrfile, timeout,
tmpstrucoutfile,
+ logger_level,
override_output_files,
environment_vars)
# Next run_file() invocations should always append to current output
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dfb3197/slider-agent/src/test/python/agent/TestActionQueue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestActionQueue.py b/slider-agent/src/test/python/agent/TestActionQueue.py
index 2e1e4cf..b3a840c 100644
--- a/slider-agent/src/test/python/agent/TestActionQueue.py
+++ b/slider-agent/src/test/python/agent/TestActionQueue.py
@@ -304,6 +304,7 @@ class TestActionQueue(TestCase):
def side_effect(py_file, script_params,
tmpoutfile, tmperrfile, timeout,
tmpstrucoutfile,
+ loglevel,
override_output_files,
environment_vars):
unfreeze_flag.wait()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dfb3197/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index 6f20db9..d2439b1 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -152,7 +152,7 @@ class TestCustomServiceOrchestrator(TestCase):
}
ret = orchestrator.runCommand(command, "out.txt", "err.txt")
## Check that override_output_files was true only during first call
- self.assertEquals(run_file_mock.call_args_list[0][0][6], True)
+ self.assertEquals(run_file_mock.call_args_list[0][0][7], True)
run_file_mock.reset_mock()
# Case when we force another command
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dfb3197/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index f62198c..5ac142b 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -462,7 +462,7 @@ public class AgentProviderService extends AbstractProviderService implements
}
private void processFolderPaths(Map<String, String> folders, String containerId, String hostFqdn) {
- for(String key : folders.keySet()) {
+ for (String key : folders.keySet()) {
workFolders.put(String.format("%s-%s-%s", hostFqdn, containerId, key), folders.get(key));
}
@@ -657,7 +657,7 @@ public class AgentProviderService extends AbstractProviderService implements
return cmdParams;
}
- private void setInstallCommandConfigurations(ExecutionCommand cmd) {
+ private void setInstallCommandConfigurations(ExecutionCommand cmd) throws SliderException {
ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf);
cmd.setConfigurations(configurations);
@@ -751,7 +751,8 @@ public class AgentProviderService extends AbstractProviderService implements
return this.allocatedPorts;
}
- private Map<String, Map<String, String>> buildCommandConfigurations(ConfTreeOperations appConf) {
+ private Map<String, Map<String, String>> buildCommandConfigurations(ConfTreeOperations appConf)
+ throws SliderException {
Map<String, Map<String, String>> configurations = new TreeMap<>();
Map<String, String> tokens = getStandardTokenMap(appConf);
@@ -767,12 +768,16 @@ public class AgentProviderService extends AbstractProviderService implements
return configurations;
}
- private Map<String, String> getStandardTokenMap(ConfTreeOperations appConf) {
+ private Map<String, String> getStandardTokenMap(ConfTreeOperations appConf) throws SliderException {
Map<String, String> tokens = new HashMap<>();
String nnuri = appConf.get("site.fs.defaultFS");
tokens.put("${NN_URI}", nnuri);
tokens.put("${NN_HOST}", URI.create(nnuri).getHost());
tokens.put("${ZK_HOST}", appConf.get(OptionKeys.ZOOKEEPER_HOSTS));
+ tokens.put("${DEFAULT_DATA_DIR}", getAmState()
+ .getInternalsSnapshot()
+ .getGlobalOptions()
+ .getMandatoryOption(OptionKeys.INTERNAL_DATA_DIR_PATH));
return tokens;
}
@@ -855,7 +860,7 @@ public class AgentProviderService extends AbstractProviderService implements
@Override
public void applyInitialRegistryDefinitions(URL amWebAPI,
- ServiceInstanceData instanceData) throws IOException {
+ ServiceInstanceData instanceData) throws IOException {
super.applyInitialRegistryDefinitions(amWebAPI, instanceData);
try {
@@ -863,7 +868,7 @@ public class AgentProviderService extends AbstractProviderService implements
CustomRegistryConstants.AGENT_REST_API,
new RegisteredEndpoint(
new URL(amWebAPI, SLIDER_PATH_AGENTS),
- "Agent REST API") );
+ "Agent REST API"));
} catch (URISyntaxException e) {
throw new IOException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dfb3197/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index b0ac967..918fda3 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -761,6 +761,7 @@ public class TestAgentProviderService {
treeOps.getGlobalOptions().put(AgentKeys.JAVA_HOME, "java_home");
treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE");
treeOps.set("site.fs.defaultFS", "hdfs://HOST1:8020/");
+ treeOps.set("internal.data.dir.path", "hdfs://HOST1:8020/database");
treeOps.set(OptionKeys.ZOOKEEPER_HOSTS, "HOST1");
treeOps.set("config_types", "hbase-site");
treeOps.getGlobalOptions().put("site.hbase-site.a.port", "${HBASE_MASTER.ALLOCATED_PORT}");