You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by bi...@apache.org on 2016/05/13 01:55:02 UTC
[1/7] incubator-slider git commit: SLIDER-1107 make am config
generation optional
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-1107_AM_config_generation b1cc78a63 -> 511891c15
SLIDER-1107 make am config generation optional
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7899ebd0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7899ebd0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7899ebd0
Branch: refs/heads/feature/SLIDER-1107_AM_config_generation
Commit: 7899ebd0a7d530445a548ed7efba1e7ac51daa60
Parents: b1cc78a
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed May 4 09:25:35 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Wed May 4 09:25:35 2016 -0700
----------------------------------------------------------------------
app-packages/hbase-nopkg/appConfig-default.json | 1 +
.../hbase-nopkg/appConfig-secured-default.json | 1 +
.../org/apache/slider/providers/agent/AgentKeys.java | 1 +
.../slider/providers/agent/AgentProviderService.java | 14 ++++++++------
4 files changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7899ebd0/app-packages/hbase-nopkg/appConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-nopkg/appConfig-default.json b/app-packages/hbase-nopkg/appConfig-default.json
index b482db9..fffad49 100644
--- a/app-packages/hbase-nopkg/appConfig-default.json
+++ b/app-packages/hbase-nopkg/appConfig-default.json
@@ -3,6 +3,7 @@
"metadata": {
},
"global": {
+ "am.config.generation": "true",
"create.default.zookeeper.node": "true",
"java_home": "/usr/jdk64/jdk1.8.0_60",
"system_configs": "core-site",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7899ebd0/app-packages/hbase-nopkg/appConfig-secured-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-nopkg/appConfig-secured-default.json b/app-packages/hbase-nopkg/appConfig-secured-default.json
index 40040e3..7c40072 100644
--- a/app-packages/hbase-nopkg/appConfig-secured-default.json
+++ b/app-packages/hbase-nopkg/appConfig-secured-default.json
@@ -3,6 +3,7 @@
"metadata": {
},
"global": {
+ "am.config.generation": "true",
"create.default.zookeeper.node": "true",
"java_home": "/usr/jdk64/jdk1.8.0_60",
"system_configs": "core-site,hdfs-site",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7899ebd0/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index 063ee77..24f161f 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -100,6 +100,7 @@ public interface AgentKeys {
String CERT_FILE_LOCALIZATION_PATH = INFRA_RUN_SECURITY_DIR + "ca.crt";
String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec";
String TEST_RELAX_VERIFICATION = "test.relax.validation";
+ String AM_CONFIG_GENERATION = "am.config.generation";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7899ebd0/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index d2a7e2c..e368129 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -497,12 +497,14 @@ public class AgentProviderService extends AbstractProviderService implements
generatedConfPath,
SliderKeys.PROPAGATED_CONF_DIR_NAME));
- // build and localize configuration files
- Map<String, Map<String, String>> configurations =
- buildCommandConfigurations(instanceDefinition.getAppConfOperations(),
- container.getId().toString(), roleName, roleGroup);
- localizeConfigFiles(launcher, roleName, roleGroup, getMetaInfo(),
- configurations, launcher.getEnv(), fileSystem);
+ if (appComponent.getOptionBool(AgentKeys.AM_CONFIG_GENERATION, false)) {
+ // build and localize configuration files
+ Map<String, Map<String, String>> configurations =
+ buildCommandConfigurations(instanceDefinition.getAppConfOperations(),
+ container.getId().toString(), roleName, roleGroup);
+ localizeConfigFiles(launcher, roleName, roleGroup, getMetaInfo(),
+ configurations, launcher.getEnv(), fileSystem);
+ }
String label = getContainerLabel(container, roleName, roleGroup);
CommandLineBuilder operation = new CommandLineBuilder();
[5/7] incubator-slider git commit: SLIDER-1107 update configs for
hbase-nopkg
Posted by bi...@apache.org.
SLIDER-1107 update configs for hbase-nopkg
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c7a7c0d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c7a7c0d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c7a7c0d9
Branch: refs/heads/feature/SLIDER-1107_AM_config_generation
Commit: c7a7c0d92b5c85fe95828d1e5e2904954bc733be
Parents: bafc5e7
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Tue May 10 10:10:39 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Thu May 12 18:51:38 2016 -0700
----------------------------------------------------------------------
app-packages/hbase-nopkg/appConfig-default.json | 9 +--
.../hbase-nopkg/appConfig-secured-default.json | 19 +++--
.../clientInstallConfig-default.json | 6 ++
app-packages/hbase-nopkg/metainfo-secured.xml | 78 ++++++++++++--------
app-packages/hbase-nopkg/metainfo.xml | 22 +++---
5 files changed, 79 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c7a7c0d9/app-packages/hbase-nopkg/appConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-nopkg/appConfig-default.json b/app-packages/hbase-nopkg/appConfig-default.json
index fffad49..980538a 100644
--- a/app-packages/hbase-nopkg/appConfig-default.json
+++ b/app-packages/hbase-nopkg/appConfig-default.json
@@ -11,7 +11,6 @@
"site.global.app_user": "yarn",
"site.global.app_version": "${hbase.version}",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/packages/hbase-${hbase.version}",
- "site.global.app_conf_dir": "${AGENT_WORK_ROOT}/app/conf",
"site.global.pid_file": "${@//site/global/app_pid_dir}/hbase-${@//site/global/app_user}-${@//site/global/hbase_component}.pid",
"site.global.daemon_cmd": "env HBASE_IDENT_STRING=${@//site/global/app_user} ${@//site/global/app_root}/bin/hbase-daemon.sh --config ${@//site/global/app_conf_dir}",
@@ -100,14 +99,10 @@
"site.global.hbase_component": "rest"
},
"HBASE_THRIFT": {
- "site.global.hbase_component": "thrift",
- "site.global.start_command": "${@//site/global/daemon_cmd} start thrift -p ${@//site/global/thrift_port}",
- "site.global.stop_command": "${@//site/global/daemon_cmd} stop thrift"
+ "site.global.hbase_component": "thrift"
},
"HBASE_THRIFT2": {
- "site.global.hbase_component": "thrift2",
- "site.global.start_command": "${@//site/global/daemon_cmd} start thrift2 -p ${@//site/global/thrift2_port}",
- "site.global.stop_command": "${@//site/global/daemon_cmd} stop thrift2"
+ "site.global.hbase_component": "thrift2"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c7a7c0d9/app-packages/hbase-nopkg/appConfig-secured-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-nopkg/appConfig-secured-default.json b/app-packages/hbase-nopkg/appConfig-secured-default.json
index 7c40072..ac6e45b 100644
--- a/app-packages/hbase-nopkg/appConfig-secured-default.json
+++ b/app-packages/hbase-nopkg/appConfig-secured-default.json
@@ -10,16 +10,18 @@
"site.global.app_user": "${USER_NAME}",
"site.global.app_version": "${hbase.version}",
- "site.global.app_root": "app/packages/hbase-${hbase.version}",
- "site.global.app_conf_dir": "app/conf",
+ "site.global.app_root": "${AGENT_WORK_ROOT}/app/packages/hbase-${hbase.version}",
+
+ "site.global.pid_file": "${@//site/global/app_pid_dir}/hbase-${@//site/global/app_user}-${@//site/global/hbase_component}.pid",
+ "site.global.daemon_cmd": "env HBASE_IDENT_STRING=${@//site/global/app_user} ${@//site/global/app_root}/bin/hbase-daemon.sh --config ${@//site/global/app_conf_dir}",
"site.global.hbase_instance_name": "instancename",
"site.global.hbase_root_password": "secret",
"site.global.user_group": "hadoop",
- "site.global.monitor_protocol": "http",
- "site.global.hbase_thrift_port": "0",
- "site.global.hbase_thrift2_port": "0",
- "site.global.hbase_rest_port": "0",
+
+ "site.global.thrift_port": "0",
+ "site.global.thrift2_port": "0",
+ "site.global.rest_port": "0",
"site.hbase-env.template.file": "hbase-env-secured.sh.j2",
"site.hbase-env.java64_home": "${JAVA_HOME}",
@@ -99,19 +101,24 @@
"slider.keytab.principal.name": "${USER_NAME}"
},
"HBASE_MASTER": {
+ "site.global.hbase_component": "master",
"site.hbase-metrics2.template.file": "hadoop-metrics2-hbase.properties-MASTER.j2",
"site.hbase-jaas.template.file": "hbase_master_jaas.conf.j2"
},
"HBASE_REGIONSERVER": {
+ "site.global.hbase_component": "regionserver",
"site.hbase-jaas.template.file": "hbase_regionserver_jaas.conf.j2"
},
"HBASE_REST": {
+ "site.global.hbase_component": "rest",
"site.hbase-jaas.template.file": "hbase_regionserver_jaas.conf.j2"
},
"HBASE_THRIFT": {
+ "site.global.hbase_component": "thrift",
"site.hbase-jaas.template.file": "hbase_regionserver_jaas.conf.j2"
},
"HBASE_THRIFT2": {
+ "site.global.hbase_component": "thrift2",
"site.hbase-jaas.template.file": "hbase_regionserver_jaas.conf.j2"
},
"HBASE_CLIENT": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c7a7c0d9/app-packages/hbase-nopkg/clientInstallConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-nopkg/clientInstallConfig-default.json b/app-packages/hbase-nopkg/clientInstallConfig-default.json
new file mode 100644
index 0000000..f195e7f
--- /dev/null
+++ b/app-packages/hbase-nopkg/clientInstallConfig-default.json
@@ -0,0 +1,6 @@
+{
+ "schema":"http://example.org/specification/v2.0.0",
+ "global":{
+ "client_root": "hbase-${hbase.version}"
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c7a7c0d9/app-packages/hbase-nopkg/metainfo-secured.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase-nopkg/metainfo-secured.xml b/app-packages/hbase-nopkg/metainfo-secured.xml
index 11c7d0a..bc5759e 100644
--- a/app-packages/hbase-nopkg/metainfo-secured.xml
+++ b/app-packages/hbase-nopkg/metainfo-secured.xml
@@ -43,19 +43,19 @@
</export>
<export>
<name>org.apache.slider.hbase.rest</name>
- <value>http://${HBASE_REST_HOST}:${site.global.hbase_rest_port}</value>
+ <value>http://${HBASE_REST_HOST}:${site.global.rest_port}</value>
</export>
<export>
<name>org.apache.slider.hbase.thrift2</name>
- <value>http://${HBASE_THRIFT2_HOST}:${site.global.hbase_thrift2_port}</value>
+ <value>http://${HBASE_THRIFT2_HOST}:${site.global.thrift2_port}</value>
</export>
<export>
<name>org.apache.slider.hbase.thrift</name>
- <value>http://${HBASE_THRIFT_HOST}:${site.global.hbase_thrift_port}</value>
+ <value>http://${HBASE_THRIFT_HOST}:${site.global.thrift_port}</value>
</export>
<export>
<name>org.apache.slider.metrics</name>
- <value>http://${site.global.metric_collector_host}:${site.global.metric_collector_port}/ws/v1/timeline/metrics</value>
+ <value>http://${site.hbase-metrics2.metric_collector_host}:${site.hbase-metrics2.metric_collector_port}/ws/v1/timeline/metrics</value>
</export>
<export>
<name>org.apache.slider.hbase.zk-path</name>
@@ -96,11 +96,15 @@
<value>${site.hbase-site.hbase.zookeeper.quorum}:${site.hbase-site.zookeeper.znode.parent}</value>
</componentExport>
</componentExports>
- <commandScript>
- <script>scripts/hbase_master.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
+ <commands>
+ <command>
+ <exec>{$conf:@//site/global/daemon_cmd} start master</exec>
+ </command>
+ <command>
+ <exec>{$conf:@//site/global/daemon_cmd} stop master</exec>
+ <name>STOP</name>
+ </command>
+ </commands>
<configFile>
<type>template</type>
<fileName>app/conf/hbase_master_jaas.conf</fileName>
@@ -112,10 +116,15 @@
<name>HBASE_REGIONSERVER</name>
<category>SLAVE</category>
<minInstanceCount>1</minInstanceCount>
- <commandScript>
- <script>scripts/hbase_regionserver.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
+ <commands>
+ <command>
+ <exec>{$conf:@//site/global/daemon_cmd} start regionserver</exec>
+ </command>
+ <command>
+ <exec>{$conf:@//site/global/daemon_cmd} stop regionserver</exec>
+ <name>STOP</name>
+ </command>
+ </commands>
<configFile>
<type>template</type>
<fileName>app/conf/hbase_regionserver_jaas.conf</fileName>
@@ -127,10 +136,15 @@
<name>HBASE_REST</name>
<category>MASTER</category>
<appExports>QuickLinks-org.apache.slider.hbase.rest</appExports>
- <commandScript>
- <script>scripts/hbase_rest.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
+ <commands>
+ <command>
+ <exec>{$conf:@//site/global/daemon_cmd} start rest -p {$conf:@//site/global/rest_port}</exec>
+ </command>
+ <command>
+ <exec>{$conf:@//site/global/daemon_cmd} stop rest</exec>
+ <name>STOP</name>
+ </command>
+ </commands>
<configFile>
<type>template</type>
<fileName>app/conf/hbase_rest_jaas.conf</fileName>
@@ -142,10 +156,15 @@
<name>HBASE_THRIFT</name>
<category>MASTER</category>
<appExports>QuickLinks-org.apache.slider.hbase.thrift</appExports>
- <commandScript>
- <script>scripts/hbase_thrift.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
+ <commands>
+ <command>
+ <exec>{$conf:@//site/global/daemon_cmd} start thrift -p {$conf:@//site/global/thrift_port}</exec>
+ </command>
+ <command>
+ <exec>{$conf:@//site/global/daemon_cmd} stop thrift</exec>
+ <name>STOP</name>
+ </command>
+ </commands>
<configFile>
<type>template</type>
<fileName>app/conf/hbase_thrift2_jaas.conf</fileName>
@@ -158,10 +177,15 @@
<category>MASTER</category>
<minInstanceCount>0</minInstanceCount>
<appExports>QuickLinks-org.apache.slider.hbase.thrift2</appExports>
- <commandScript>
- <script>scripts/hbase_thrift2.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
+ <commands>
+ <command>
+ <exec>{$conf:@//site/global/daemon_cmd} start thrift2 -p {$conf:@//site/global/thrift_port}</exec>
+ </command>
+ <command>
+ <exec>{$conf:@//site/global/daemon_cmd} stop thrift2</exec>
+ <name>STOP</name>
+ </command>
+ </commands>
<configFile>
<type>template</type>
<fileName>app/conf/hbase_thrift_jaas.conf</fileName>
@@ -172,10 +196,6 @@
<component>
<name>HBASE_CLIENT</name>
<category>CLIENT</category>
- <commandScript>
- <script>scripts/hbase_client.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
<configFile>
<type>template</type>
<fileName>app/conf/hbase_client_jaas.conf</fileName>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c7a7c0d9/app-packages/hbase-nopkg/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase-nopkg/metainfo.xml b/app-packages/hbase-nopkg/metainfo.xml
index 9abc9dc..60ef10b 100644
--- a/app-packages/hbase-nopkg/metainfo.xml
+++ b/app-packages/hbase-nopkg/metainfo.xml
@@ -43,19 +43,19 @@
</export>
<export>
<name>org.apache.slider.hbase.rest</name>
- <value>http://${HBASE_REST_HOST}:${site.global.hbase_rest_port}</value>
+ <value>http://${HBASE_REST_HOST}:${site.global.rest_port}</value>
</export>
<export>
<name>org.apache.slider.hbase.thrift2</name>
- <value>http://${HBASE_THRIFT2_HOST}:${site.global.hbase_thrift2_port}</value>
+ <value>http://${HBASE_THRIFT2_HOST}:${site.global.thrift2_port}</value>
</export>
<export>
<name>org.apache.slider.hbase.thrift</name>
- <value>http://${HBASE_THRIFT_HOST}:${site.global.hbase_thrift_port}</value>
+ <value>http://${HBASE_THRIFT_HOST}:${site.global.thrift_port}</value>
</export>
<export>
<name>org.apache.slider.metrics</name>
- <value>http://${site.global.metric_collector_host}:${site.global.metric_collector_port}/ws/v1/timeline/metrics</value>
+ <value>http://${site.hbase-metrics2.metric_collector_host}:${site.hbase-metrics2.metric_collector_port}/ws/v1/timeline/metrics</value>
</export>
<export>
<name>org.apache.slider.hbase.zk-path</name>
@@ -171,10 +171,6 @@
<component>
<name>HBASE_CLIENT</name>
<category>CLIENT</category>
- <commandScript>
- <script>scripts/hbase_client.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
</component>
</components>
@@ -200,27 +196,27 @@
<configFiles>
<configFile>
<type>xml</type>
- <fileName>app/conf/hbase-site.xml</fileName>
+ <fileName>hbase-site.xml</fileName>
<dictionaryName>hbase-site</dictionaryName>
</configFile>
<configFile>
<type>template</type>
- <fileName>app/conf/hbase-env.sh</fileName>
+ <fileName>hbase-env.sh</fileName>
<dictionaryName>hbase-env</dictionaryName>
</configFile>
<configFile>
<type>template</type>
- <fileName>app/conf/log4j.properties</fileName>
+ <fileName>log4j.properties</fileName>
<dictionaryName>hbase-log4j</dictionaryName>
</configFile>
<configFile>
<type>xml</type>
- <fileName>app/conf/hbase-policy.xml</fileName>
+ <fileName>hbase-policy.xml</fileName>
<dictionaryName>hbase-policy</dictionaryName>
</configFile>
<configFile>
<type>template</type>
- <fileName>app/conf/hadoop-metrics2-hbase.properties</fileName>
+ <fileName>hadoop-metrics2-hbase.properties</fileName>
<dictionaryName>hbase-metrics2</dictionaryName>
</configFile>
</configFiles>
[4/7] incubator-slider git commit: SLIDER-1107 add client install
capability in the case of AM generated configs
Posted by bi...@apache.org.
SLIDER-1107 add client install capability in the case of AM generated configs
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bafc5e72
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bafc5e72
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bafc5e72
Branch: refs/heads/feature/SLIDER-1107_AM_config_generation
Commit: bafc5e726e0bfdb5adb08e1755bc8ad5cd9f3ec2
Parents: 3c47ec2
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Tue May 10 10:09:57 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Thu May 12 18:51:37 2016 -0700
----------------------------------------------------------------------
.../org/apache/slider/client/ClientUtils.java | 111 ++++++++++++++++
.../org/apache/slider/client/SliderClient.java | 91 ++++---------
.../slider/common/tools/CoreFileSystem.java | 13 ++
.../slider/core/conf/ConfTreeOperations.java | 16 ++-
.../providers/AbstractClientProvider.java | 9 +-
.../providers/agent/AgentClientProvider.java | 127 ++++++++++++++++---
.../slider/providers/agent/AgentKeys.java | 3 +
.../providers/agent/AgentProviderService.java | 4 +-
.../agent/TestAgentClientProvider2.java | 8 +-
9 files changed, 294 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bafc5e72/slider-core/src/main/java/org/apache/slider/client/ClientUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/ClientUtils.java b/slider-core/src/main/java/org/apache/slider/client/ClientUtils.java
new file mode 100644
index 0000000..c3ccb1d
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/client/ClientUtils.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.client;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.exceptions.NoRecordException;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.NotFoundException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.registry.docstore.ConfigFormat;
+import org.apache.slider.core.registry.docstore.PublishedConfigSet;
+import org.apache.slider.core.registry.docstore.PublishedConfiguration;
+import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
+import org.apache.slider.core.registry.retrieve.RegistryRetriever;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.currentUser;
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath;
+
+public class ClientUtils {
+ public static ServiceRecord lookupServiceRecord(RegistryOperations rops,
+ String user, String name) throws IOException, SliderException {
+ return lookupServiceRecord(rops, user, null, name);
+ }
+
+ public static ServiceRecord lookupServiceRecord(RegistryOperations rops,
+ String user, String type, String name) throws IOException,
+ SliderException {
+ if (StringUtils.isEmpty(user)) {
+ user = currentUser();
+ } else {
+ user = RegistryPathUtils.encodeForRegistry(user);
+ }
+ if (StringUtils.isEmpty(type)) {
+ type = SliderKeys.APP_TYPE;
+ }
+
+ String path = servicePath(user, type, name);
+ return resolve(rops, path);
+ }
+
+ public static ServiceRecord resolve(RegistryOperations rops, String path)
+ throws IOException, SliderException {
+ try {
+ return rops.resolve(path);
+ } catch (PathNotFoundException | NoRecordException e) {
+ throw new NotFoundException(e.getPath().toString(), e);
+ }
+ }
+
+ public static PublishedConfiguration getConfigFromRegistry(
+ RegistryOperations rops, Configuration configuration,
+ String configName, String appName, String user, boolean external)
+ throws IOException, SliderException {
+ ServiceRecord instance = lookupServiceRecord(rops, user, appName);
+
+ RegistryRetriever retriever = new RegistryRetriever(configuration, instance);
+ PublishedConfigSet configurations = retriever.getConfigurations(external);
+
+ PublishedConfiguration published = retriever.retrieveConfiguration(
+ configurations, configName, external);
+ return published;
+ }
+
+ public static String saveOrReturnConfig(PublishedConfiguration published,
+ String format, File destPath, String fileName)
+ throws BadCommandArgumentsException, IOException {
+ ConfigFormat configFormat = ConfigFormat.resolve(format);
+ if (configFormat == null) {
+ throw new BadCommandArgumentsException(
+ "Unknown/Unsupported format %s ", format);
+ }
+ PublishedConfigurationOutputter outputter =
+ PublishedConfigurationOutputter.createOutputter(configFormat,
+ published);
+ boolean print = destPath == null;
+ if (!print) {
+ if (destPath.isDirectory()) {
+ // creating it under a directory
+ destPath = new File(destPath, fileName);
+ }
+ outputter.save(destPath);
+ return null;
+ } else {
+ return outputter.asString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bafc5e72/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index f332a66..eb0630d 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1301,8 +1301,21 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
E_INVALID_INSTALL_PATH + ": " + clientInfo.installLocation.getAbsolutePath());
File pkgFile;
- require(isSet(clientInfo.packageURI), E_INVALID_APPLICATION_PACKAGE_LOCATION);
- pkgFile = new File(clientInfo.packageURI);
+ File tmpDir = null;
+
+ require(isSet(clientInfo.packageURI) || isSet(clientInfo.name),
+ E_INVALID_APPLICATION_PACKAGE_LOCATION);
+ if (isSet(clientInfo.packageURI)) {
+ pkgFile = new File(clientInfo.packageURI);
+ } else {
+ Path appDirPath = sliderFileSystem.buildAppDefDirPath(clientInfo.name);
+ Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG);
+ require(sliderFileSystem.isFile(appDefPath),
+ E_INVALID_APPLICATION_PACKAGE_LOCATION);
+ tmpDir = Files.createTempDir();
+ pkgFile = new File(tmpDir, SliderKeys.DEFAULT_APP_PKG);
+ sliderFileSystem.copyHdfsFileToLocal(appDefPath, pkgFile);
+ }
require(pkgFile.isFile(),
E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE + " at %s", pkgFile.getAbsolutePath());
@@ -1324,6 +1337,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
AbstractClientProvider
provider = createClientProvider(SliderProviderFactory.DEFAULT_CLUSTER_TYPE);
provider.processClientOperation(sliderFileSystem,
+ getRegistryOperations(),
+ getConfig(),
"INSTALL",
clientInfo.installLocation,
pkgFile,
@@ -4164,17 +4179,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
@VisibleForTesting
public PublishedConfiguration actionRegistryGetConfig(ActionRegistryArgs registryArgs)
throws YarnException, IOException {
- ServiceRecord instance = lookupServiceRecord(registryArgs);
-
- RegistryRetriever retriever = new RegistryRetriever(getConfig(), instance);
- boolean external = !registryArgs.internal;
- PublishedConfigSet configurations =
- retriever.getConfigurations(external);
-
- PublishedConfiguration published = retriever.retrieveConfiguration(configurations,
- registryArgs.getConf,
- external);
- return published;
+ return ClientUtils.getConfigFromRegistry(getRegistryOperations(),
+ getConfig(), registryArgs.getConf, registryArgs.name, registryArgs.user,
+ !registryArgs.internal);
}
/**
@@ -4217,27 +4224,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// decide whether or not to print
String entry = registryArgs.getConf;
String format = registryArgs.format;
- ConfigFormat configFormat = ConfigFormat.resolve(format);
- if (configFormat == null) {
- throw new BadCommandArgumentsException(
- "Unknown/Unsupported format %s ", format);
+ String output = ClientUtils.saveOrReturnConfig(published,
+ registryArgs.format, registryArgs.out, entry + "." + format);
+ if (output != null) {
+ print(output);
}
- PublishedConfigurationOutputter outputter =
- PublishedConfigurationOutputter.createOutputter(configFormat,
- published);
- boolean print = registryArgs.out == null;
- if (!print) {
- File outputPath = registryArgs.out;
- if (outputPath.isDirectory()) {
- // creating it under a directory
- outputPath = new File(outputPath, entry + "." + format);
- }
- log.debug("Destination path: {}", outputPath);
- outputter.save(outputPath);
- } else {
- print(outputter.asString());
- }
-
}
/**
@@ -4287,32 +4278,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private ServiceRecord lookupServiceRecord(ActionRegistryArgs registryArgs) throws
SliderException,
IOException {
- String user;
- if (StringUtils.isNotEmpty(registryArgs.user)) {
- user = RegistryPathUtils.encodeForRegistry(registryArgs.user);
- } else {
- user = currentUser();
- }
-
- String path = servicePath(user, registryArgs.serviceType,
- registryArgs.name);
- return resolve(path);
- }
-
- /**
- * Look up a service record of the current user
- * @param serviceType service type
- * @param id instance ID
- * @return instance data
- * @throws UnknownApplicationInstanceException no path or service record
- * at the end of the path
- * @throws SliderException other failures
- * @throws IOException IO problems or wrapped exceptions
- */
- public ServiceRecord lookupServiceRecord(String serviceType, String id)
- throws IOException, SliderException {
- String path = servicePath(currentUser(), serviceType, id);
- return resolve(path);
+ return ClientUtils.lookupServiceRecord(getRegistryOperations(),
+ registryArgs.user, registryArgs.serviceType, registryArgs.name);
}
/**
@@ -4327,11 +4294,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
*/
public ServiceRecord resolve(String path)
throws IOException, SliderException {
- try {
- return getRegistryOperations().resolve(path);
- } catch (PathNotFoundException | NoRecordException e) {
- throw new NotFoundException(e.getPath().toString(), e);
- }
+ return ClientUtils.resolve(getRegistryOperations(), path);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bafc5e72/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index 0d3436b..df086d1 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -765,6 +765,19 @@ public class CoreFileSystem {
fileSystem.setPermission(destPath, fp);
}
+ public void copyHdfsFileToLocal(Path hdfsPath, File destFile)
+ throws IOException {
+ if (hdfsPath == null || destFile == null) {
+ throw new IOException("Either hdfsPath or destPath is null");
+ }
+ log.info("Copying file {} to {}", hdfsPath.toUri(), destFile.toURI());
+
+ Path destPath = new Path(destFile.getPath());
+ fileSystem.copyToLocalFile(hdfsPath, destPath);
+ // set file permissions of the destPath
+ fileSystem.setPermission(destPath, new FsPermission("755"));
+ }
+
/**
* list entries in a filesystem directory
*
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bafc5e72/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
index 9013edb..7e88abd 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
@@ -345,7 +345,21 @@ public class ConfTreeOperations {
confTreeSerDeser.fromFile(resource) );
return ops;
}
-
+
+ /**
+ * Load from a json String. The inner conf tree is the loaded data -unresolved
+ * @param json json string
+ * @return loaded value
+ * @throws IOException load failure
+ */
+ public static ConfTreeOperations fromString(String json) throws
+ IOException {
+ ConfTreeSerDeser confTreeSerDeser = new ConfTreeSerDeser();
+ ConfTreeOperations ops = new ConfTreeOperations(
+ confTreeSerDeser.fromJson(json) );
+ return ops;
+ }
+
/**
* Build from an existing instance -which is cloned via JSON ser/deser
* @param instance the source instance
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bafc5e72/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
index fcab65e..510de5d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
@@ -21,6 +21,7 @@ package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTreeOperations;
@@ -223,18 +224,22 @@ public abstract class AbstractClientProvider extends Configured {
/**
* Process client operations for applications such as install, configure
* @param fileSystem
+ * @param registryOperations
+ * @param configuration
* @param operation
* @param clientInstallPath
* @param clientPackage
- * @param config
+ * @param clientConfig
* @param name
* @throws SliderException
*/
public void processClientOperation(SliderFileSystem fileSystem,
+ RegistryOperations registryOperations,
+ Configuration configuration,
String operation,
File clientInstallPath,
File clientPackage,
- JSONObject config,
+ JSONObject clientConfig,
String name)
throws SliderException {
throw new SliderException("Provider does not support client operations.");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bafc5e72/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 12d581c..e5c3bec 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -26,9 +26,11 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.registry.client.binding.RegistryUtils;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.client.ClientUtils;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
@@ -38,13 +40,18 @@ import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.AbstractLauncher;
+import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.providers.AbstractClientProvider;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderUtils;
import org.apache.slider.providers.agent.application.metadata.Application;
import org.apache.slider.providers.agent.application.metadata.Component;
+import org.apache.slider.providers.agent.application.metadata.ConfigFile;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.providers.agent.application.metadata.MetainfoParser;
+import org.apache.slider.providers.agent.application.metadata.OSPackage;
+import org.apache.slider.providers.agent.application.metadata.OSSpecific;
+import org.apache.slider.providers.agent.application.metadata.Package;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
@@ -291,6 +298,8 @@ public class AgentClientProvider extends AbstractClientProvider
@Override
public void processClientOperation(SliderFileSystem fileSystem,
+ RegistryOperations rops,
+ Configuration configuration,
String operation,
File clientInstallPath,
File appPackage,
@@ -379,38 +388,105 @@ public class AgentClientProvider extends AbstractClientProvider
throw new BadConfigException(E_COULD_NOT_READ_METAINFO);
}
- expandAgentTar(agentPkgDir);
-
- JSONObject commandJson = getCommandJson(defaultConfig, config, metaInfo, clientInstallPath, name);
- FileWriter file = new FileWriter(new File(cmdDir, "command.json"));
- try {
- file.write(commandJson.toString());
-
- } catch (IOException e) {
- e.printStackTrace();
- } finally {
- file.flush();
- file.close();
- }
-
String client_script = null;
+ Component clientComponent = null;
for (Component component : metaInfo.getApplication().getComponents()) {
if (component.getCategory().equals("CLIENT")) {
+ clientComponent = component;
client_script = component.getCommandScript().getScript();
- log.info("Installing CLIENT {} using script {}", component.getName(), client_script);
break;
}
}
if (SliderUtils.isUnset(client_script)) {
- throw new SliderException("No valid CLIENT component found. Aborting install.");
- }
+ log.info("Installing CLIENT without script");
+ List<Package> packages = metaInfo.getApplication().getPackages();
+ if (packages != null && packages.size() > 0) {
+ // retrieve package resources from HDFS and extract
+ for (Package pkg : packages) {
+ Path pkgPath = fileSystem.buildResourcePath(pkg.getName());
+ if (!fileSystem.isFile(pkgPath) && name != null) {
+ pkgPath = fileSystem.buildResourcePath(name, pkg.getName());
+ }
+ if (!fileSystem.isFile(pkgPath)) {
+ throw new IOException("Package doesn't exist as a resource: " +
+ pkg.getName());
+ }
+ if ("archive".equals(pkg.getType())) {
+ File pkgFile = new File(tmpDir, pkg.getName());
+ fileSystem.copyHdfsFileToLocal(pkgPath, pkgFile);
+ expandTar(pkgFile, clientInstallPath);
+ } else {
+ File pkgFile = new File(clientInstallPath, pkg.getName());
+ fileSystem.copyHdfsFileToLocal(pkgPath, pkgFile);
+ }
+ }
+ } else {
+ // extract tarball from app def
+ for (OSSpecific osSpecific : metaInfo.getApplication()
+ .getOSSpecifics()) {
+ for (OSPackage pkg : osSpecific.getPackages()) {
+ if ("tarball".equals(pkg.getType())) {
+ File pkgFile = new File(appPkgDir, pkg.getName());
+ expandTar(pkgFile, clientInstallPath);
+ }
+ }
+ }
+ }
+ if (name == null) {
+ log.warn("Conf files not being generated because no app name was " +
+ "providied");
+ return;
+ }
+ File confInstallDir;
+ String clientRoot = null;
+ try {
+ clientRoot = defaultConfig.getJSONObject("global")
+ .getString(AgentKeys.APP_CLIENT_ROOT);
+ } catch (JSONException e) {
+ e.printStackTrace();
+ }
+ if (clientRoot == null) {
+ confInstallDir = clientInstallPath;
+ } else {
+ confInstallDir = new File(new File(clientInstallPath, clientRoot), "conf");
+ if (!confInstallDir.exists()) {
+ confInstallDir.mkdirs();
+ }
+ }
+ String user = RegistryUtils.currentUser();
+ for (ConfigFile configFile : metaInfo.getApplication().getConfigFiles()) {
+ retrieveConfigFile(rops, configuration, configFile, name, user,
+ confInstallDir);
+ }
+ for (ConfigFile configFile : clientComponent.getConfigFiles()) {
+ retrieveConfigFile(rops, configuration, configFile, name, user,
+ confInstallDir);
+ }
+ } else {
+ log.info("Installing CLIENT using script {}", client_script);
+ expandAgentTar(agentPkgDir);
+
+ JSONObject commandJson = getCommandJson(defaultConfig, config, metaInfo, clientInstallPath, name);
+ FileWriter file = new FileWriter(new File(cmdDir, "command.json"));
+ try {
+ file.write(commandJson.toString());
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ file.flush();
+ file.close();
+ }
- runCommand(appPkgDir, agentPkgDir, cmdDir, client_script);
+ runCommand(appPkgDir, agentPkgDir, cmdDir, client_script);
+ }
} catch (IOException ioex) {
log.warn("Error while executing INSTALL command {}", ioex.getMessage());
throw new SliderException("INSTALL client failed.");
+ } finally {
+ tmpDir.delete();
}
}
@@ -481,6 +557,10 @@ public class AgentClientProvider extends AbstractClientProvider
String libDirProp =
System.getProperty(SliderKeys.PROPERTY_LIB_DIR);
File tarFile = new File(libDirProp, SliderKeys.AGENT_TAR);
+ expandTar(tarFile, agentPkgDir);
+ }
+
+ private void expandTar(File tarFile, File destDir) throws IOException {
TarArchiveInputStream tarIn = new TarArchiveInputStream(
new GzipCompressorInputStream(
new BufferedInputStream(
@@ -491,7 +571,7 @@ public class AgentClientProvider extends AbstractClientProvider
try {
TarArchiveEntry tarEntry = tarIn.getNextTarEntry();
while (tarEntry != null) {
- File destPath = new File(agentPkgDir, tarEntry.getName());
+ File destPath = new File(destDir, tarEntry.getName());
if (tarEntry.isDirectory()) {
destPath.mkdirs();
} else {
@@ -515,6 +595,15 @@ public class AgentClientProvider extends AbstractClientProvider
}
}
+ private void retrieveConfigFile(RegistryOperations rops,
+ Configuration configuration, ConfigFile configFile, String name,
+ String user, File destDir) throws IOException, SliderException {
+ PublishedConfiguration published = ClientUtils.getConfigFromRegistry(rops,
+ configuration, configFile.getDictionaryName(), name, user, true);
+ ClientUtils.saveOrReturnConfig(published, configFile.getType(),
+ destDir, configFile.getFileName());
+ }
+
protected JSONObject getCommandJson(JSONObject defaultConfig,
JSONObject inputConfig,
Metainfo metainfo,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bafc5e72/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index 24f161f..01a3f1a 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -48,6 +48,7 @@ public interface AgentKeys {
*/
String APP_HOME = "app.home";
String APP_ROOT = "site.global.app_root";
+ String APP_CLIENT_ROOT = "client_root";
/**
* Runas user of the application
*/
@@ -77,6 +78,8 @@ public interface AgentKeys {
String APP_RESOURCES = "application.resources";
String APP_RESOURCES_DIR = "app/resources";
+ String APP_CONF_DIR = "app/conf";
+
String AGENT_INSTALL_DIR = "infra/agent";
String APP_DEFINITION_DIR = "app/definition";
String ADDON_DEFINITION_DIR = "addon/definition";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bafc5e72/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index e368129..ab4da47 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -839,7 +839,8 @@ public class AgentProviderService extends AbstractProviderService implements
SliderKeys.RESOURCE_DIR + "/" + destFile.getName(),
configResource, fileName);
} else {
- launcher.addLocalResource(fileName, configResource);
+ launcher.addLocalResource(AgentKeys.APP_CONF_DIR + "/" + fileName,
+ configResource);
}
}
}
@@ -3127,6 +3128,7 @@ public class AgentProviderService extends AbstractProviderService implements
config.put("app_log_dir", "${AGENT_LOG_ROOT}");
config.put("app_pid_dir", "${AGENT_WORK_ROOT}/app/run");
config.put("app_install_dir", "${AGENT_WORK_ROOT}/app/install");
+ config.put("app_conf_dir", "${AGENT_WORK_ROOT}/" + AgentKeys.APP_CONF_DIR);
config.put("app_input_conf_dir", "${AGENT_WORK_ROOT}/" + SliderKeys.PROPAGATED_CONF_DIR_NAME);
config.put("app_container_id", containerId);
config.put("app_container_tag", tags.getTag(roleName, containerId));
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bafc5e72/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java
index b919fcf..b959e2f 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java
@@ -239,7 +239,11 @@ public class TestAgentClientProvider2 extends SliderTestUtils {
public void testSliderClientForInstallFailures() throws Exception {
describe(" IGNORE ANY STACK TRACES BELOW ");
- SliderClient client = new SliderClient();
+ SliderClient client = PowerMock.createPartialMock(SliderClient.class,
+ "getRegistryOperations");
+ expect(client.getRegistryOperations()).andReturn(null).anyTimes();
+ PowerMock.replay(SliderClient.class);
+
client.bindArgs(new Configuration(), "client", "--dest", "a_random_path/none", "--package", "a_random_pkg.zip");
ActionClientArgs args = new ActionClientArgs();
@@ -287,5 +291,7 @@ public class TestAgentClientProvider2 extends SliderTestUtils {
assertExceptionDetails(e, SliderExitCodes.EXIT_BAD_CONFIGURATION,
AgentClientProvider.E_COULD_NOT_READ_METAINFO);
}
+
+ PowerMock.verify(SliderClient.class);
}
}
[3/7] incubator-slider git commit: SLIDER-1107 add unit tests for
PublishedConfigurationOutputter
Posted by bi...@apache.org.
SLIDER-1107 add unit tests for PublishedConfigurationOutputter
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3c47ec2d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3c47ec2d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3c47ec2d
Branch: refs/heads/feature/SLIDER-1107_AM_config_generation
Commit: 3c47ec2ddf0df8a6c49dfc2278f3e5383f65c2f7
Parents: 6a9a2b7
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Tue May 10 09:54:43 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Tue May 10 09:54:43 2016 -0700
----------------------------------------------------------------------
.../PublishedConfigurationOutputter.java | 6 +-
.../TestPublishedConfigurationOutputter.java | 220 +++++++++++++++++++
2 files changed, 225 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c47ec2d/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
index 8e6acbd..23cfc8f 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
@@ -26,6 +26,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.slider.common.tools.ConfigHelper;
import org.apache.slider.common.tools.SliderFileSystem;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.DumperOptions.FlowStyle;
import org.yaml.snakeyaml.Yaml;
import java.io.File;
@@ -246,7 +248,9 @@ public abstract class PublishedConfigurationOutputter {
public YamlOutputter(PublishedConfiguration owner) {
super(owner);
- yaml = new Yaml();
+ DumperOptions options = new DumperOptions();
+ options.setDefaultFlowStyle(FlowStyle.BLOCK);
+ yaml = new Yaml(options);
}
public String asString() throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3c47ec2d/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java b/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
new file mode 100644
index 0000000..eec5c38
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.core.registry.docstore;
+
+import com.google.common.base.Charsets;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter.TemplateOutputter;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.powermock.api.easymock.PowerMock;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.expect;
+import static org.mockito.Matchers.anyString;
+import static org.powermock.api.easymock.PowerMock.createNiceMock;
+
+public class TestPublishedConfigurationOutputter {
+ private static HashMap<String, String> config = new HashMap<>();
+
+ @Rule
+ public TemporaryFolder tmpDir = new TemporaryFolder();
+
+ @Before
+ public void setup() {
+ config.put("key1", "val1");
+ }
+
+ @Test
+ public void testJson() throws IOException {
+ PublishedConfigurationOutputter configurationOutputter =
+ PublishedConfigurationOutputter.createOutputter(ConfigFormat.JSON,
+ new PublishedConfiguration("description",
+ config.entrySet(), null, null));
+
+ String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
+ "");
+ assert "{\"key1\":\"val1\"}".equals(output);
+
+ File file = tmpDir.newFile();
+ configurationOutputter.save(file);
+
+ ObjectMapper mapper = new ObjectMapper();
+ Map<String, String> read = mapper.readValue(file, Map.class);
+ assert 1 == read.size();
+ assert "val1".equals(read.get("key1"));
+ }
+
+ @Test
+ public void testXml() throws IOException {
+ PublishedConfigurationOutputter configurationOutputter =
+ PublishedConfigurationOutputter.createOutputter(ConfigFormat.XML,
+ new PublishedConfiguration("description",
+ config.entrySet(), null, null));
+
+ String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
+ "");
+ assert output.contains(
+ "<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>");
+
+ File file = tmpDir.newFile();
+ configurationOutputter.save(file);
+
+ assert FileUtils.readFileToString(file, Charsets.UTF_8)
+ .replaceAll("( |\\r|\\n)", "")
+ .contains(
+ "<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>");
+ }
+
+ @Test
+ public void testHadoopXml() throws IOException {
+ PublishedConfigurationOutputter configurationOutputter =
+ PublishedConfigurationOutputter.createOutputter(ConfigFormat.HADOOP_XML,
+ new PublishedConfiguration("description",
+ config.entrySet(), null, null));
+
+ String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
+ "");
+ assert output.contains("<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>");
+
+ File file = tmpDir.newFile();
+ configurationOutputter.save(file);
+
+ assert FileUtils.readFileToString(file, Charsets.UTF_8)
+ .replaceAll("( |\\r|\\n)", "")
+ .contains( "<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>");
+ }
+
+ @Test
+ public void testProperties() throws IOException {
+ PublishedConfigurationOutputter configurationOutputter =
+ PublishedConfigurationOutputter.createOutputter(ConfigFormat.PROPERTIES,
+ new PublishedConfiguration("description",
+ config.entrySet(), null, null));
+
+ String output = configurationOutputter.asString();
+ assert output.contains("key1=val1");
+
+ File file = tmpDir.newFile();
+ configurationOutputter.save(file);
+
+ Properties properties = new Properties();
+ FileInputStream fis = null;
+ try {
+ fis = new FileInputStream(file);
+ properties.load(fis);
+ } finally {
+ if (fis != null) {
+ fis.close();
+ }
+ }
+ assert 1 == properties.size();
+ assert "val1".equals(properties.getProperty("key1"));
+ }
+
+ @Test
+ public void testYaml() throws IOException {
+ PublishedConfigurationOutputter configurationOutputter =
+ PublishedConfigurationOutputter.createOutputter(ConfigFormat.YAML,
+ new PublishedConfiguration("description",
+ config.entrySet(), null, null));
+
+ String output = configurationOutputter.asString().replaceAll("(\\r|\\n)",
+ "");
+ assert "key1: val1".equals(output);
+
+ File file = tmpDir.newFile();
+ configurationOutputter.save(file);
+
+ Yaml yaml = new Yaml();
+ FileInputStream fis = null;
+ Map<String, String> read;
+ try {
+ fis = new FileInputStream(file);
+ read = (Map<String, String>) yaml.load(fis);
+ } finally {
+ if (fis != null) {
+ fis.close();
+ }
+ }
+ assert 1 == read.size();
+ assert "val1".equals(read.get("key1"));
+ }
+
+ @Test
+ public void testEnv() throws IOException {
+ HashMap<String, String> envConfig = new HashMap<>(config);
+ envConfig.put("content", "content {{key1}} ");
+
+ PublishedConfigurationOutputter configurationOutputter =
+ PublishedConfigurationOutputter.createOutputter(ConfigFormat.ENV,
+ new PublishedConfiguration("description",
+ envConfig.entrySet(), null, null));
+
+ String output = configurationOutputter.asString();
+ assert "content val1 ".equals(output);
+
+ File file = tmpDir.newFile();
+ configurationOutputter.save(file);
+
+ assert "content val1 ".equals(FileUtils.readFileToString(file,
+ Charsets.UTF_8));
+ }
+
+ @Test
+ public void testTemplate1() throws IOException {
+ HashMap<String, String> templateConfig = new HashMap<>(config);
+ templateConfig.put(TemplateOutputter.TEMPLATE_FILE, "templateFileName");
+
+ SliderFileSystem fileSystem = createNiceMock(SliderFileSystem.class);
+ expect(fileSystem.buildResourcePath(anyString())).andReturn(new Path("path")).anyTimes();
+ expect(fileSystem.isFile(anyObject(Path.class))).andReturn(true).anyTimes();
+ expect(fileSystem.cat(anyObject(Path.class))).andReturn("content {{key1}}\n more ${key1} content").anyTimes();
+
+ PowerMock.replay(fileSystem);
+
+ PublishedConfigurationOutputter configurationOutputter =
+ PublishedConfigurationOutputter.createOutputter(ConfigFormat.TEMPLATE,
+ new PublishedConfiguration("description",
+ templateConfig.entrySet(), fileSystem, "clusterName"));
+
+ String output = configurationOutputter.asString();
+ assert "content val1\n more val1 content".equals(output);
+
+ File file = tmpDir.newFile();
+ configurationOutputter.save(file);
+
+ PowerMock.verify(fileSystem);
+
+ assert "content val1\n more val1 content".equals(
+ FileUtils.readFileToString(file, Charsets.UTF_8));
+ }
+}
[2/7] incubator-slider git commit: SLIDER-1107 add yaml config type
Posted by bi...@apache.org.
SLIDER-1107 add yaml config type
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/6a9a2b7b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/6a9a2b7b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/6a9a2b7b
Branch: refs/heads/feature/SLIDER-1107_AM_config_generation
Commit: 6a9a2b7bf0fd8f4fed6609bc3e465d19db21b1d0
Parents: 7899ebd
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed May 4 09:26:14 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Wed May 4 09:26:14 2016 -0700
----------------------------------------------------------------------
pom.xml | 7 ++++
slider-core/pom.xml | 6 ++++
.../core/registry/docstore/ConfigFormat.java | 2 +-
.../PublishedConfigurationOutputter.java | 38 ++++++++++----------
4 files changed, 32 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6a9a2b7b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b616e15..64a78e4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -176,6 +176,7 @@
<protobuf.version>2.5.0</protobuf.version>
<slf4j.version>1.7.5</slf4j.version>
+ <snakeyaml.version>1.16</snakeyaml.version>
<storm.version>0.9.3</storm.version>
<stringtemplate.version>2.4.1</stringtemplate.version>
<zookeeper.version>3.4.6</zookeeper.version>
@@ -1221,6 +1222,12 @@
<version>${slf4j.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ <version>${snakeyaml.version}</version>
+ </dependency>
+
<!-- Used for testing -->
<dependency>
<groupId>junit</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6a9a2b7b/slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index 17728c3..8b19e7f 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -254,6 +254,12 @@
</dependency>
<dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<scope>compile</scope>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6a9a2b7b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
index 9e0c095..ddab606 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
@@ -26,7 +26,7 @@ public enum ConfigFormat {
HADOOP_XML("hadoop-xml"),
ENV("env"),
TEMPLATE("template"),
-// YAML("yaml");
+ YAML("yaml"),
;
ConfigFormat(String suffix) {
this.suffix = suffix;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/6a9a2b7b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
index bae9674..8e6acbd 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
@@ -26,13 +26,12 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.slider.common.tools.ConfigHelper;
import org.apache.slider.common.tools.SliderFileSystem;
+import org.yaml.snakeyaml.Yaml;
import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.StringWriter;
-import java.util.Collections;
import java.util.Map;
import java.util.Properties;
@@ -61,14 +60,7 @@ public abstract class PublishedConfigurationOutputter {
}
*/
public void save(File dest) throws IOException {
- FileOutputStream out = null;
- try {
- out = new FileOutputStream(dest);
- save(out);
- out.close();
- } finally {
- org.apache.hadoop.io.IOUtils.closeStream(out);
- }
+ FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8);
}
/**
@@ -109,6 +101,8 @@ public abstract class PublishedConfigurationOutputter {
return new EnvOutputter(owner);
case TEMPLATE:
return new TemplateOutputter(owner);
+ case YAML:
+ return new YamlOutputter(owner);
default:
throw new RuntimeException("Unsupported format :" + format);
}
@@ -169,11 +163,6 @@ public abstract class PublishedConfigurationOutputter {
}
@Override
- public void save(File dest) throws IOException {
- FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8);
- }
-
- @Override
public String asString() throws IOException {
return owner.asJson();
}
@@ -187,11 +176,6 @@ public abstract class PublishedConfigurationOutputter {
}
@Override
- public void save(File dest) throws IOException {
- FileUtils.writeStringToFile(dest, asString(), Charsets.UTF_8);
- }
-
- @Override
public String asString() throws IOException {
if (!owner.entries.containsKey("content")) {
throw new IOException("Configuration has no content field and cannot " +
@@ -256,4 +240,18 @@ public abstract class PublishedConfigurationOutputter {
}
}
+ public static class YamlOutputter extends PublishedConfigurationOutputter {
+
+ private final Yaml yaml;
+
+ public YamlOutputter(PublishedConfiguration owner) {
+ super(owner);
+ yaml = new Yaml();
+ }
+
+ public String asString() throws IOException {
+ return yaml.dump(owner.entries);
+ }
+ }
+
}
[6/7] incubator-slider git commit: SLIDER-1107 fix AM-generated
config publishing and race condition in config generation
Posted by bi...@apache.org.
SLIDER-1107 fix AM-generated config publishing and race condition in config generation
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1699a714
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1699a714
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1699a714
Branch: refs/heads/feature/SLIDER-1107_AM_config_generation
Commit: 1699a714dfcea6473d9256a7e0a4536976485acc
Parents: c7a7c0d
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Thu May 12 18:22:08 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Thu May 12 18:51:38 2016 -0700
----------------------------------------------------------------------
.../core/registry/docstore/ConfigUtils.java | 37 ++++++
.../docstore/PublishedConfiguration.java | 22 ----
.../PublishedConfigurationOutputter.java | 53 +--------
.../providers/agent/AgentProviderService.java | 113 ++++++++++++++-----
4 files changed, 120 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1699a714/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java
index 7e6ee5a..2e1615b 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigUtils.java
@@ -17,6 +17,10 @@
*/
package org.apache.slider.core.registry.docstore;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.tools.SliderFileSystem;
+
+import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
@@ -24,6 +28,8 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class ConfigUtils {
+ public static final String TEMPLATE_FILE = "template.file";
+
public static String replaceProps(Map<String, String> config, String content) {
Map<String, String> tokens = new HashMap<>();
for (Entry<String, String> entry : config.entrySet()) {
@@ -56,4 +62,35 @@ public class ConfigUtils {
return newConfig;
}
+ public static void prepConfigForTemplateOutputter(ConfigFormat configFormat,
+ Map<String, String> config, SliderFileSystem fileSystem,
+ String clusterName, String fileName) throws IOException {
+ if (!configFormat.equals(ConfigFormat.TEMPLATE)) {
+ return;
+ }
+ Path templateFile = null;
+ if (config.containsKey(TEMPLATE_FILE)) {
+ templateFile = fileSystem.buildResourcePath(config.get(TEMPLATE_FILE));
+ if (!fileSystem.isFile(templateFile)) {
+ templateFile = fileSystem.buildResourcePath(clusterName,
+ config.get(TEMPLATE_FILE));
+ }
+ if (!fileSystem.isFile(templateFile)) {
+ throw new IOException("config specified template file " + config
+ .get(TEMPLATE_FILE) + " but " + templateFile + " doesn't exist");
+ }
+ }
+ if (templateFile == null && fileName != null) {
+ templateFile = fileSystem.buildResourcePath(fileName);
+ if (!fileSystem.isFile(templateFile)) {
+ templateFile = fileSystem.buildResourcePath(clusterName,
+ fileName);
+ }
+ }
+ if (fileSystem.isFile(templateFile)) {
+ config.put("content", fileSystem.cat(templateFile));
+ } else {
+ config.put("content", "");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1699a714/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
index cabdce6..477f7d3 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
@@ -20,7 +20,6 @@ package org.apache.slider.core.registry.docstore;
import org.apache.hadoop.conf.Configuration;
import org.apache.slider.common.tools.ConfigHelper;
-import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.exceptions.BadConfigException;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.ObjectMapper;
@@ -50,9 +49,6 @@ public class PublishedConfiguration {
public Map<String, String> entries = new HashMap<>();
- public SliderFileSystem fileSystem;
- public String clusterName;
-
public PublishedConfiguration() {
}
@@ -90,24 +86,6 @@ public class PublishedConfiguration {
}
/**
- * Build a configuration from the entries
- * @param description configuration description
- * @param entries entries to put
- * @param fileSystem Slider file system (source of configuration templates)
- * @param clusterName cluster name
- */
- public PublishedConfiguration(String description,
- Iterable<Map.Entry<String, String>> entries,
- SliderFileSystem fileSystem,
- String clusterName) {
- this.description = description;
- putValues(entries);
- this.fileSystem = fileSystem;
- this.clusterName = clusterName;
- }
-
-
- /**
* Is the configuration empty. This means either that it has not
* been given any values, or it is stripped down copy set down over the
* wire.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1699a714/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
index 23cfc8f..9bdcfcb 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigurationOutputter.java
@@ -23,9 +23,7 @@ import com.google.common.base.Preconditions;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
import org.apache.slider.common.tools.ConfigHelper;
-import org.apache.slider.common.tools.SliderFileSystem;
import org.yaml.snakeyaml.DumperOptions;
import org.yaml.snakeyaml.DumperOptions.FlowStyle;
import org.yaml.snakeyaml.Yaml;
@@ -34,7 +32,6 @@ import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.io.StringWriter;
-import java.util.Map;
import java.util.Properties;
/**
@@ -188,58 +185,10 @@ public abstract class PublishedConfigurationOutputter {
}
}
- public static class TemplateOutputter extends PublishedConfigurationOutputter {
-
- public static final String TEMPLATE_FILE = "template.file";
-
+ public static class TemplateOutputter extends EnvOutputter {
public TemplateOutputter(PublishedConfiguration owner) {
super(owner);
}
-
- @Override
- public void save(File dest) throws IOException {
- FileUtils.writeStringToFile(dest, asString(dest.getName()),
- Charsets.UTF_8);
- }
-
- public String asString(String fileName) throws IOException {
- if (owner.fileSystem == null) {
- throw new IOException("File system not specified for template " +
- "configuration");
- }
- Map<String,String> config = owner.entries;
- SliderFileSystem fileSystem = owner.fileSystem;
- Path templateFile = null;
- if (config.containsKey(TEMPLATE_FILE)) {
- templateFile = fileSystem.buildResourcePath(config.get(TEMPLATE_FILE));
- if (!fileSystem.isFile(templateFile)) {
- templateFile = fileSystem.buildResourcePath(owner.clusterName,
- config.get(TEMPLATE_FILE));
- }
- if (!fileSystem.isFile(templateFile)) {
- throw new IOException("config specified template file " + config
- .get(TEMPLATE_FILE) + " for config " + owner.description +
- " but " + templateFile + " doesn't exist");
- }
- }
- if (templateFile == null && fileName != null) {
- templateFile = fileSystem.buildResourcePath(fileName);
- if (!fileSystem.isFile(templateFile)) {
- templateFile = fileSystem.buildResourcePath(owner.clusterName,
- fileName);
- }
- }
- if (fileSystem.isFile(templateFile)) {
- return ConfigUtils.replaceProps(config, fileSystem.cat(templateFile));
- } else {
- return "";
- }
- }
-
- @Override
- public String asString() throws IOException {
- return asString(null);
- }
}
public static class YamlOutputter extends PublishedConfigurationOutputter {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1699a714/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index ab4da47..155ff09 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -170,6 +170,8 @@ public class AgentProviderService extends AbstractProviderService implements
private AgentClientProvider clientProvider;
private AtomicInteger taskId = new AtomicInteger(0);
private volatile Metainfo metaInfo = null;
+ private AggregateConf instanceDefinition = null;
+ private SliderFileSystem fileSystem = null;
private Map<String, DefaultConfig> defaultConfigs = null;
private ComponentCommandOrder commandOrder = null;
private HeartbeatMonitor monitor;
@@ -282,6 +284,8 @@ public class AgentProviderService extends AbstractProviderService implements
if (metaInfo == null) {
synchronized (syncLock) {
if (metaInfo == null) {
+ this.instanceDefinition = instanceDefinition;
+ this.fileSystem = fileSystem;
readAndSetHeartbeatMonitoringInterval(instanceDefinition);
initializeAgentDebugCommands(instanceDefinition);
@@ -679,21 +683,6 @@ public class AgentProviderService extends AbstractProviderService implements
return uploadResource(resource, fileSystem, certsDir);
}
- private Path checkResourceExists(File resource, SliderFileSystem
- fileSystem, String roleName) throws IOException {
- Path dir;
- if (roleName == null) {
- dir = fileSystem.buildClusterResourcePath(getClusterName());
- } else {
- dir = fileSystem.buildClusterResourcePath(getClusterName(), roleName);
- }
- Path destPath = new Path(dir, resource.getName());
- if (fileSystem.getFileSystem().exists(destPath)) {
- return destPath;
- }
- return null;
- }
-
private Path uploadResource(File resource, SliderFileSystem fileSystem,
String roleName) throws IOException {
Path dir;
@@ -705,8 +694,8 @@ public class AgentProviderService extends AbstractProviderService implements
return uploadResource(resource, fileSystem, dir);
}
- private static Path uploadResource(File resource, SliderFileSystem fileSystem,
- Path parentDir) throws IOException {
+ private static synchronized Path uploadResource(File resource,
+ SliderFileSystem fileSystem, Path parentDir) throws IOException {
if (!fileSystem.getFileSystem().exists(parentDir)) {
fileSystem.getFileSystem().mkdirs(parentDir,
new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
@@ -783,14 +772,19 @@ public class AgentProviderService extends AbstractProviderService implements
}
private void createConfigFile(SliderFileSystem fileSystem, File file,
- ConfigFile configFile, Map<String, String> config) throws IOException {
+ ConfigFile configFile, Map<String, String> config)
+ throws IOException {
ConfigFormat configFormat = ConfigFormat.resolve(configFile.getType());
log.info("Writing {} file {}", configFormat, file);
+ ConfigUtils.prepConfigForTemplateOutputter(configFormat, config,
+ fileSystem, getClusterName(), file.getName());
+ PublishedConfiguration publishedConfiguration =
+ new PublishedConfiguration(configFile.getDictionaryName(),
+ config.entrySet());
PublishedConfigurationOutputter configurationOutputter =
PublishedConfigurationOutputter.createOutputter(configFormat,
- new PublishedConfiguration(configFile.getDictionaryName(),
- config.entrySet(), fileSystem, getClusterName()));
+ publishedConfiguration);
configurationOutputter.save(file);
}
@@ -820,16 +814,11 @@ public class AgentProviderService extends AbstractProviderService implements
folder = roleGroup;
}
- Path destPath = checkResourceExists(localFile, fileSystem, folder);
- if (destPath == null) {
- log.info("Localizing {} configs to config file {} (destination {}) " +
- "based on {} configs",
- config.size(), localFile, fileName, configFile.getDictionaryName());
- createConfigFile(fileSystem, localFile, configFile, config);
- destPath = uploadResource(localFile, fileSystem, folder);
- } else {
- log.info("Config already exists at {}, not recreating it", destPath);
- }
+ log.info("Localizing {} configs to config file {} (destination {}) " +
+ "based on {} configs", config.size(), localFile, fileName,
+ configFile.getDictionaryName());
+ createConfigFile(fileSystem, localFile, configFile, config);
+ Path destPath = uploadResource(localFile, fileSystem, folder);
LocalResource configResource = fileSystem.createAmResource(destPath,
LocalResourceType.FILE);
@@ -1033,7 +1022,7 @@ public class AgentProviderService extends AbstractProviderService implements
role = amState.getOwnedContainer(containerId);
role.ip = status.getIp();
}
- if(status.getHostname() != null & !status.getHostname().isEmpty()){
+ if(status.getHostname() != null && !status.getHostname().isEmpty()){
role = amState.getOwnedContainer(containerId);
role.hostname = status.getHostname();
}
@@ -1369,6 +1358,68 @@ public class AgentProviderService extends AbstractProviderService implements
} catch (URISyntaxException e) {
throw new IOException(e);
}
+
+ // identify client component
+ Component client = null;
+ for (Component component : getMetaInfo().getApplication().getComponents()) {
+ if (component != null && component.getCategory().equals("CLIENT")) {
+ client = component;
+ break;
+ }
+ }
+ if (client == null) {
+ log.info("No client component specified, not publishing client configs");
+ return;
+ }
+
+ // register AM-generated client configs
+ ConfTreeOperations appConf = instanceDefinition.getAppConfOperations();
+ MapOperations clientOperations = instanceDefinition.getAppConfOperations()
+ .getOrAddComponent(client.getName());
+ if (!clientOperations.getOptionBool(AgentKeys.AM_CONFIG_GENERATION,
+ false)) {
+ log.info("AM config generation is false, not publishing client configs");
+ return;
+ }
+
+ // build and localize configuration files
+ Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>();
+ Map<String, String> tokens = null;
+ try {
+ tokens = getStandardTokenMap(appConf, client.getName(), client.getName());
+ } catch (SliderException e) {
+ throw new IOException(e);
+ }
+
+ for (ConfigFile configFile : getMetaInfo()
+ .getComponentConfigFiles(client.getName())) {
+ addNamedConfiguration(configFile.getDictionaryName(),
+ appConf.getGlobalOptions().options, configurations, tokens, null,
+ client.getName());
+ if (appConf.getComponent(client.getName()) != null) {
+ addNamedConfiguration(configFile.getDictionaryName(),
+ appConf.getComponent(client.getName()).options, configurations,
+ tokens, null, client.getName());
+ }
+ }
+
+ //do a final replacement of re-used configs
+ dereferenceAllConfigs(configurations);
+
+ for (ConfigFile configFile : getMetaInfo()
+ .getComponentConfigFiles(client.getName())) {
+ ConfigFormat configFormat = ConfigFormat.resolve(configFile.getType());
+
+ Map<String, String> config = configurations.get(configFile.getDictionaryName());
+ ConfigUtils.prepConfigForTemplateOutputter(configFormat, config,
+ fileSystem, getClusterName(),
+ new File(configFile.getFileName()).getName());
+ PublishedConfiguration publishedConfiguration =
+ new PublishedConfiguration(configFile.getDictionaryName(),
+ config.entrySet());
+ getAmState().getPublishedSliderConfigurations().put(
+ configFile.getDictionaryName(), publishedConfiguration);
+ }
}
@Override
[7/7] incubator-slider git commit: SLIDER-1107 fix client installation
Posted by bi...@apache.org.
SLIDER-1107 fix client installation
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/511891c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/511891c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/511891c1
Branch: refs/heads/feature/SLIDER-1107_AM_config_generation
Commit: 511891c151746b2bd9e5de9cd781b8dd58ec512d
Parents: 1699a71
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Thu May 12 18:30:07 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Thu May 12 18:51:38 2016 -0700
----------------------------------------------------------------------
app-packages/hbase-nopkg/appConfig-default.json | 4 +
.../resources/hbase-env-client.sh.j2 | 49 +++++++++++
app-packages/hbase-nopkg/src/assembly/hbase.xml | 7 ++
.../slider/common/tools/CoreFileSystem.java | 2 -
.../providers/agent/AgentClientProvider.java | 93 +++++++++-----------
.../TestPublishedConfigurationOutputter.java | 19 ++--
6 files changed, 113 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/511891c1/app-packages/hbase-nopkg/appConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-nopkg/appConfig-default.json b/app-packages/hbase-nopkg/appConfig-default.json
index 980538a..1095efd 100644
--- a/app-packages/hbase-nopkg/appConfig-default.json
+++ b/app-packages/hbase-nopkg/appConfig-default.json
@@ -103,6 +103,10 @@
},
"HBASE_THRIFT2": {
"site.global.hbase_component": "thrift2"
+ },
+ "HBASE_CLIENT": {
+ "site.hbase-env.template.file": "hbase-env-client.sh.j2",
+ "site.hbase-site.instance": "${CLUSTER_NAME}"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/511891c1/app-packages/hbase-nopkg/resources/hbase-env-client.sh.j2
----------------------------------------------------------------------
diff --git a/app-packages/hbase-nopkg/resources/hbase-env-client.sh.j2 b/app-packages/hbase-nopkg/resources/hbase-env-client.sh.j2
new file mode 100644
index 0000000..2511a29
--- /dev/null
+++ b/app-packages/hbase-nopkg/resources/hbase-env-client.sh.j2
@@ -0,0 +1,49 @@
+# Set environment variables here.
+
+# The java implementation to use. Java 1.6 required.
+export JAVA_HOME={{java64_home}}
+
+# Extra Java CLASSPATH elements. Optional.
+export HBASE_CLASSPATH=${HBASE_CLASSPATH}
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+# export HBASE_HEAPSIZE=1000
+
+# Extra Java runtime options.
+# Below are what we set by default. May only work with SUN JVM.
+# For more on why as well as other possible settings,
+# see http://wiki.apache.org/hadoop/PerformanceTuning
+export HBASE_OPTS="-XX:+UseConcMarkSweepGC"
+# Uncomment below to enable java garbage collection logging.
+# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log"
+
+# Uncomment and adjust to enable JMX exporting
+# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.
+# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
+#
+# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
+export HBASE_MASTER_OPTS="-Xmx{{master_heapsize}}"
+export HBASE_REGIONSERVER_OPTS="-Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}"
+# export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103"
+# export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104"
+
+# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.
+export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers
+
+# Extra ssh options. Empty by default.
+# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR"
+
+# A string representing this instance of hbase. $USER by default.
+# export HBASE_IDENT_STRING=$USER
+
+# The scheduling priority for daemon processes. See 'man nice'.
+# export HBASE_NICENESS=10
+
+# Seconds to sleep between slave commands. Unset by default. This
+# can be useful in large clusters, where, e.g., slave rsyncs can
+# otherwise arrive faster than the master can service them.
+# export HBASE_SLAVE_SLEEP=0.1
+
+# Tell HBase whether it should manage it's own instance of Zookeeper or not.
+export HBASE_MANAGES_ZK=false
+export HBASE_OPTS="$HBASE_OPTS {{hbase_opts}}"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/511891c1/app-packages/hbase-nopkg/src/assembly/hbase.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase-nopkg/src/assembly/hbase.xml b/app-packages/hbase-nopkg/src/assembly/hbase.xml
index da10fb6..0558319 100644
--- a/app-packages/hbase-nopkg/src/assembly/hbase.xml
+++ b/app-packages/hbase-nopkg/src/assembly/hbase.xml
@@ -42,6 +42,12 @@
<fileMode>0755</fileMode>
</file>
<file>
+ <source>clientInstallConfig-default.json</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
<source>metainfo.xml</source>
<outputDirectory>/</outputDirectory>
<filtered>true</filtered>
@@ -71,6 +77,7 @@
<exclude>target/**</exclude>
<exclude>appConfig-default.json</exclude>
<exclude>appConfig-secured-default.json</exclude>
+ <exclude>clientInstallConfig-default.json</exclude>
<exclude>metainfo.xml</exclude>
<exclude>metainfo-secured.xml</exclude>
</excludes>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/511891c1/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index df086d1..0b0f1bc 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -774,8 +774,6 @@ public class CoreFileSystem {
Path destPath = new Path(destFile.getPath());
fileSystem.copyToLocalFile(hdfsPath, destPath);
- // set file permissions of the destPath
- fileSystem.setPermission(destPath, new FsPermission("755"));
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/511891c1/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index e5c3bec..18e4702 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -22,6 +22,7 @@ import com.google.common.io.Files;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
@@ -60,7 +61,6 @@ import org.slf4j.LoggerFactory;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@@ -328,51 +328,25 @@ public class AgentClientProvider extends AbstractClientProvider
{
ZipEntry zipEntry = zipInputStream.getNextEntry();
while (zipEntry != null) {
- if ("metainfo.xml".equals(zipEntry.getName())) {
- int size = (int) zipEntry.getSize();
- if (size != -1) {
- log.info("Reading {} of size {}", zipEntry.getName(),
- zipEntry.getSize());
- byte[] content = new byte[size];
- int offset = 0;
- while (offset < size) {
- offset += zipInputStream.read(content, offset, size - offset);
- }
- metaInfo = new MetainfoParser().fromXmlStream(new ByteArrayInputStream(content));
- }
- } else if ("metainfo.json".equals(zipEntry.getName())) {
- int size = (int) zipEntry.getSize();
- if (size != -1) {
- log.info("Reading {} of size {}", zipEntry.getName(),
- zipEntry.getSize());
- byte[] content = new byte[size];
- int offset = 0;
- while (offset < size) {
- offset += zipInputStream.read(content, offset, size - offset);
- }
- metaInfo = new MetainfoParser().fromJsonStream(new ByteArrayInputStream(content));
- }
- } else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) {
- int size = (int) zipEntry.getSize();
- if (size != -1) {
- log.info("Reading {} of size {}", zipEntry.getName(),
- zipEntry.getSize());
- byte[] content = new byte[size];
- int offset = 0;
- while (offset < size) {
- offset += zipInputStream.read(content, offset, size - offset);
- }
+ log.info("Processing {}", zipEntry.getName());
+ String filePath = appPkgDir + File.separator + zipEntry.getName();
+ if (!zipEntry.isDirectory()) {
+ log.info("Extracting file {}", filePath);
+ extractFile(zipInputStream, filePath);
+
+ if ("metainfo.xml".equals(zipEntry.getName())) {
+ metaInfo = new MetainfoParser().fromXmlStream(new FileInputStream(filePath));
+ } else if ("metainfo.json".equals(zipEntry.getName())) {
+ metaInfo = new MetainfoParser().fromJsonStream(new FileInputStream(filePath));
+ } else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) {
try {
- defaultConfig = new JSONObject(new String(content, Charset.defaultCharset()));
+ defaultConfig = new JSONObject(FileUtils.readFileToString(new File(filePath), Charset.defaultCharset()));
} catch (JSONException jex) {
throw new SliderException("Unable to read default client config.", jex);
}
}
- }
- String filePath = appPkgDir + File.separator + zipEntry.getName();
- if (!zipEntry.isDirectory()) {
- extractFile(zipInputStream, filePath);
} else {
+ log.info("Creating dir {}", filePath);
File dir = new File(filePath);
dir.mkdir();
}
@@ -393,7 +367,9 @@ public class AgentClientProvider extends AbstractClientProvider
for (Component component : metaInfo.getApplication().getComponents()) {
if (component.getCategory().equals("CLIENT")) {
clientComponent = component;
- client_script = component.getCommandScript().getScript();
+ if (component.getCommandScript() != null) {
+ client_script = component.getCommandScript().getScript();
+ }
break;
}
}
@@ -435,16 +411,26 @@ public class AgentClientProvider extends AbstractClientProvider
}
if (name == null) {
log.warn("Conf files not being generated because no app name was " +
- "providied");
+ "provided");
return;
}
File confInstallDir;
String clientRoot = null;
- try {
- clientRoot = defaultConfig.getJSONObject("global")
- .getString(AgentKeys.APP_CLIENT_ROOT);
- } catch (JSONException e) {
- e.printStackTrace();
+ if (defaultConfig != null) {
+ try {
+ clientRoot = defaultConfig.getJSONObject("global")
+ .getString(AgentKeys.APP_CLIENT_ROOT);
+ } catch (JSONException e) {
+ e.printStackTrace();
+ }
+ }
+ if (config != null) {
+ try {
+ clientRoot = config.getJSONObject("global")
+ .getString(AgentKeys.APP_CLIENT_ROOT);
+ } catch (JSONException e) {
+ e.printStackTrace();
+ }
}
if (clientRoot == null) {
confInstallDir = clientInstallPath;
@@ -485,8 +471,6 @@ public class AgentClientProvider extends AbstractClientProvider
} catch (IOException ioex) {
log.warn("Error while executing INSTALL command {}", ioex.getMessage());
throw new SliderException("INSTALL client failed.");
- } finally {
- tmpDir.delete();
}
}
@@ -561,6 +545,7 @@ public class AgentClientProvider extends AbstractClientProvider
}
private void expandTar(File tarFile, File destDir) throws IOException {
+ log.info("Expanding tar {} to {}", tarFile, destDir);
TarArchiveInputStream tarIn = new TarArchiveInputStream(
new GzipCompressorInputStream(
new BufferedInputStream(
@@ -572,10 +557,13 @@ public class AgentClientProvider extends AbstractClientProvider
TarArchiveEntry tarEntry = tarIn.getNextTarEntry();
while (tarEntry != null) {
File destPath = new File(destDir, tarEntry.getName());
+ File parent = destPath.getParentFile();
+ if (!parent.exists()) {
+ parent.mkdirs();
+ }
if (tarEntry.isDirectory()) {
destPath.mkdirs();
} else {
- destPath.createNewFile();
byte[] byteToRead = new byte[1024];
BufferedOutputStream buffOut =
new BufferedOutputStream(new FileOutputStream(destPath));
@@ -588,6 +576,9 @@ public class AgentClientProvider extends AbstractClientProvider
buffOut.close();
}
}
+ if ((tarEntry.getMode() & 0100) != 0) {
+ destPath.setExecutable(true);
+ }
tarEntry = tarIn.getNextTarEntry();
}
} finally {
@@ -598,6 +589,8 @@ public class AgentClientProvider extends AbstractClientProvider
private void retrieveConfigFile(RegistryOperations rops,
Configuration configuration, ConfigFile configFile, String name,
String user, File destDir) throws IOException, SliderException {
+ log.info("Retrieving config {} to {}", configFile.getDictionaryName(),
+ destDir);
PublishedConfiguration published = ClientUtils.getConfigFromRegistry(rops,
configuration, configFile.getDictionaryName(), name, user, true);
ClientUtils.saveOrReturnConfig(published, configFile.getType(),
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/511891c1/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java b/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
index eec5c38..63d5961 100644
--- a/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
+++ b/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
@@ -21,7 +21,6 @@ import com.google.common.base.Charsets;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.Path;
import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter.TemplateOutputter;
import org.codehaus.jackson.map.ObjectMapper;
import org.junit.Before;
import org.junit.Rule;
@@ -58,7 +57,7 @@ public class TestPublishedConfigurationOutputter {
PublishedConfigurationOutputter configurationOutputter =
PublishedConfigurationOutputter.createOutputter(ConfigFormat.JSON,
new PublishedConfiguration("description",
- config.entrySet(), null, null));
+ config.entrySet()));
String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
"");
@@ -78,7 +77,7 @@ public class TestPublishedConfigurationOutputter {
PublishedConfigurationOutputter configurationOutputter =
PublishedConfigurationOutputter.createOutputter(ConfigFormat.XML,
new PublishedConfiguration("description",
- config.entrySet(), null, null));
+ config.entrySet()));
String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
"");
@@ -99,7 +98,7 @@ public class TestPublishedConfigurationOutputter {
PublishedConfigurationOutputter configurationOutputter =
PublishedConfigurationOutputter.createOutputter(ConfigFormat.HADOOP_XML,
new PublishedConfiguration("description",
- config.entrySet(), null, null));
+ config.entrySet()));
String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
"");
@@ -118,7 +117,7 @@ public class TestPublishedConfigurationOutputter {
PublishedConfigurationOutputter configurationOutputter =
PublishedConfigurationOutputter.createOutputter(ConfigFormat.PROPERTIES,
new PublishedConfiguration("description",
- config.entrySet(), null, null));
+ config.entrySet()));
String output = configurationOutputter.asString();
assert output.contains("key1=val1");
@@ -145,7 +144,7 @@ public class TestPublishedConfigurationOutputter {
PublishedConfigurationOutputter configurationOutputter =
PublishedConfigurationOutputter.createOutputter(ConfigFormat.YAML,
new PublishedConfiguration("description",
- config.entrySet(), null, null));
+ config.entrySet()));
String output = configurationOutputter.asString().replaceAll("(\\r|\\n)",
"");
@@ -177,7 +176,7 @@ public class TestPublishedConfigurationOutputter {
PublishedConfigurationOutputter configurationOutputter =
PublishedConfigurationOutputter.createOutputter(ConfigFormat.ENV,
new PublishedConfiguration("description",
- envConfig.entrySet(), null, null));
+ envConfig.entrySet()));
String output = configurationOutputter.asString();
assert "content val1 ".equals(output);
@@ -192,7 +191,7 @@ public class TestPublishedConfigurationOutputter {
@Test
public void testTemplate1() throws IOException {
HashMap<String, String> templateConfig = new HashMap<>(config);
- templateConfig.put(TemplateOutputter.TEMPLATE_FILE, "templateFileName");
+ templateConfig.put(ConfigUtils.TEMPLATE_FILE, "templateFileName");
SliderFileSystem fileSystem = createNiceMock(SliderFileSystem.class);
expect(fileSystem.buildResourcePath(anyString())).andReturn(new Path("path")).anyTimes();
@@ -201,10 +200,12 @@ public class TestPublishedConfigurationOutputter {
PowerMock.replay(fileSystem);
+ ConfigUtils.prepConfigForTemplateOutputter(ConfigFormat.TEMPLATE,
+ templateConfig, fileSystem, "clusterName", null);
PublishedConfigurationOutputter configurationOutputter =
PublishedConfigurationOutputter.createOutputter(ConfigFormat.TEMPLATE,
new PublishedConfiguration("description",
- templateConfig.entrySet(), fileSystem, "clusterName"));
+ templateConfig.entrySet()));
String output = configurationOutputter.asString();
assert "content val1\n more val1 content".equals(output);