You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by bi...@apache.org on 2016/05/13 01:55:08 UTC
[7/7] incubator-slider git commit: SLIDER-1107 fix client installation
SLIDER-1107 fix client installation
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/511891c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/511891c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/511891c1
Branch: refs/heads/feature/SLIDER-1107_AM_config_generation
Commit: 511891c151746b2bd9e5de9cd781b8dd58ec512d
Parents: 1699a71
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Thu May 12 18:30:07 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Thu May 12 18:51:38 2016 -0700
----------------------------------------------------------------------
app-packages/hbase-nopkg/appConfig-default.json | 4 +
.../resources/hbase-env-client.sh.j2 | 49 +++++++++++
app-packages/hbase-nopkg/src/assembly/hbase.xml | 7 ++
.../slider/common/tools/CoreFileSystem.java | 2 -
.../providers/agent/AgentClientProvider.java | 93 +++++++++-----------
.../TestPublishedConfigurationOutputter.java | 19 ++--
6 files changed, 113 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/511891c1/app-packages/hbase-nopkg/appConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-nopkg/appConfig-default.json b/app-packages/hbase-nopkg/appConfig-default.json
index 980538a..1095efd 100644
--- a/app-packages/hbase-nopkg/appConfig-default.json
+++ b/app-packages/hbase-nopkg/appConfig-default.json
@@ -103,6 +103,10 @@
},
"HBASE_THRIFT2": {
"site.global.hbase_component": "thrift2"
+ },
+ "HBASE_CLIENT": {
+ "site.hbase-env.template.file": "hbase-env-client.sh.j2",
+ "site.hbase-site.instance": "${CLUSTER_NAME}"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/511891c1/app-packages/hbase-nopkg/resources/hbase-env-client.sh.j2
----------------------------------------------------------------------
diff --git a/app-packages/hbase-nopkg/resources/hbase-env-client.sh.j2 b/app-packages/hbase-nopkg/resources/hbase-env-client.sh.j2
new file mode 100644
index 0000000..2511a29
--- /dev/null
+++ b/app-packages/hbase-nopkg/resources/hbase-env-client.sh.j2
@@ -0,0 +1,49 @@
+# Set environment variables here.
+
+# The java implementation to use. Java 1.6 required.
+export JAVA_HOME={{java64_home}}
+
+# Extra Java CLASSPATH elements. Optional.
+export HBASE_CLASSPATH=${HBASE_CLASSPATH}
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+# export HBASE_HEAPSIZE=1000
+
+# Extra Java runtime options.
+# Below are what we set by default. May only work with SUN JVM.
+# For more on why as well as other possible settings,
+# see http://wiki.apache.org/hadoop/PerformanceTuning
+export HBASE_OPTS="-XX:+UseConcMarkSweepGC"
+# Uncomment below to enable java garbage collection logging.
+# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log"
+
+# Uncomment and adjust to enable JMX exporting
+# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.
+# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
+#
+# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
+export HBASE_MASTER_OPTS="-Xmx{{master_heapsize}}"
+export HBASE_REGIONSERVER_OPTS="-Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}"
+# export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103"
+# export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104"
+
+# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.
+export HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers
+
+# Extra ssh options. Empty by default.
+# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR"
+
+# A string representing this instance of hbase. $USER by default.
+# export HBASE_IDENT_STRING=$USER
+
+# The scheduling priority for daemon processes. See 'man nice'.
+# export HBASE_NICENESS=10
+
+# Seconds to sleep between slave commands. Unset by default. This
+# can be useful in large clusters, where, e.g., slave rsyncs can
+# otherwise arrive faster than the master can service them.
+# export HBASE_SLAVE_SLEEP=0.1
+
+# Tell HBase whether it should manage it's own instance of Zookeeper or not.
+export HBASE_MANAGES_ZK=false
+export HBASE_OPTS="$HBASE_OPTS {{hbase_opts}}"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/511891c1/app-packages/hbase-nopkg/src/assembly/hbase.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase-nopkg/src/assembly/hbase.xml b/app-packages/hbase-nopkg/src/assembly/hbase.xml
index da10fb6..0558319 100644
--- a/app-packages/hbase-nopkg/src/assembly/hbase.xml
+++ b/app-packages/hbase-nopkg/src/assembly/hbase.xml
@@ -42,6 +42,12 @@
<fileMode>0755</fileMode>
</file>
<file>
+ <source>clientInstallConfig-default.json</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
<source>metainfo.xml</source>
<outputDirectory>/</outputDirectory>
<filtered>true</filtered>
@@ -71,6 +77,7 @@
<exclude>target/**</exclude>
<exclude>appConfig-default.json</exclude>
<exclude>appConfig-secured-default.json</exclude>
+ <exclude>clientInstallConfig-default.json</exclude>
<exclude>metainfo.xml</exclude>
<exclude>metainfo-secured.xml</exclude>
</excludes>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/511891c1/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index df086d1..0b0f1bc 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -774,8 +774,6 @@ public class CoreFileSystem {
Path destPath = new Path(destFile.getPath());
fileSystem.copyToLocalFile(hdfsPath, destPath);
- // set file permissions of the destPath
- fileSystem.setPermission(destPath, new FsPermission("755"));
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/511891c1/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index e5c3bec..18e4702 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -22,6 +22,7 @@ import com.google.common.io.Files;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
@@ -60,7 +61,6 @@ import org.slf4j.LoggerFactory;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@@ -328,51 +328,25 @@ public class AgentClientProvider extends AbstractClientProvider
{
ZipEntry zipEntry = zipInputStream.getNextEntry();
while (zipEntry != null) {
- if ("metainfo.xml".equals(zipEntry.getName())) {
- int size = (int) zipEntry.getSize();
- if (size != -1) {
- log.info("Reading {} of size {}", zipEntry.getName(),
- zipEntry.getSize());
- byte[] content = new byte[size];
- int offset = 0;
- while (offset < size) {
- offset += zipInputStream.read(content, offset, size - offset);
- }
- metaInfo = new MetainfoParser().fromXmlStream(new ByteArrayInputStream(content));
- }
- } else if ("metainfo.json".equals(zipEntry.getName())) {
- int size = (int) zipEntry.getSize();
- if (size != -1) {
- log.info("Reading {} of size {}", zipEntry.getName(),
- zipEntry.getSize());
- byte[] content = new byte[size];
- int offset = 0;
- while (offset < size) {
- offset += zipInputStream.read(content, offset, size - offset);
- }
- metaInfo = new MetainfoParser().fromJsonStream(new ByteArrayInputStream(content));
- }
- } else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) {
- int size = (int) zipEntry.getSize();
- if (size != -1) {
- log.info("Reading {} of size {}", zipEntry.getName(),
- zipEntry.getSize());
- byte[] content = new byte[size];
- int offset = 0;
- while (offset < size) {
- offset += zipInputStream.read(content, offset, size - offset);
- }
+ log.info("Processing {}", zipEntry.getName());
+ String filePath = appPkgDir + File.separator + zipEntry.getName();
+ if (!zipEntry.isDirectory()) {
+ log.info("Extracting file {}", filePath);
+ extractFile(zipInputStream, filePath);
+
+ if ("metainfo.xml".equals(zipEntry.getName())) {
+ metaInfo = new MetainfoParser().fromXmlStream(new FileInputStream(filePath));
+ } else if ("metainfo.json".equals(zipEntry.getName())) {
+ metaInfo = new MetainfoParser().fromJsonStream(new FileInputStream(filePath));
+ } else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) {
try {
- defaultConfig = new JSONObject(new String(content, Charset.defaultCharset()));
+ defaultConfig = new JSONObject(FileUtils.readFileToString(new File(filePath), Charset.defaultCharset()));
} catch (JSONException jex) {
throw new SliderException("Unable to read default client config.", jex);
}
}
- }
- String filePath = appPkgDir + File.separator + zipEntry.getName();
- if (!zipEntry.isDirectory()) {
- extractFile(zipInputStream, filePath);
} else {
+ log.info("Creating dir {}", filePath);
File dir = new File(filePath);
dir.mkdir();
}
@@ -393,7 +367,9 @@ public class AgentClientProvider extends AbstractClientProvider
for (Component component : metaInfo.getApplication().getComponents()) {
if (component.getCategory().equals("CLIENT")) {
clientComponent = component;
- client_script = component.getCommandScript().getScript();
+ if (component.getCommandScript() != null) {
+ client_script = component.getCommandScript().getScript();
+ }
break;
}
}
@@ -435,16 +411,26 @@ public class AgentClientProvider extends AbstractClientProvider
}
if (name == null) {
log.warn("Conf files not being generated because no app name was " +
- "providied");
+ "provided");
return;
}
File confInstallDir;
String clientRoot = null;
- try {
- clientRoot = defaultConfig.getJSONObject("global")
- .getString(AgentKeys.APP_CLIENT_ROOT);
- } catch (JSONException e) {
- e.printStackTrace();
+ if (defaultConfig != null) {
+ try {
+ clientRoot = defaultConfig.getJSONObject("global")
+ .getString(AgentKeys.APP_CLIENT_ROOT);
+ } catch (JSONException e) {
+ e.printStackTrace();
+ }
+ }
+ if (config != null) {
+ try {
+ clientRoot = config.getJSONObject("global")
+ .getString(AgentKeys.APP_CLIENT_ROOT);
+ } catch (JSONException e) {
+ e.printStackTrace();
+ }
}
if (clientRoot == null) {
confInstallDir = clientInstallPath;
@@ -485,8 +471,6 @@ public class AgentClientProvider extends AbstractClientProvider
} catch (IOException ioex) {
log.warn("Error while executing INSTALL command {}", ioex.getMessage());
throw new SliderException("INSTALL client failed.");
- } finally {
- tmpDir.delete();
}
}
@@ -561,6 +545,7 @@ public class AgentClientProvider extends AbstractClientProvider
}
private void expandTar(File tarFile, File destDir) throws IOException {
+ log.info("Expanding tar {} to {}", tarFile, destDir);
TarArchiveInputStream tarIn = new TarArchiveInputStream(
new GzipCompressorInputStream(
new BufferedInputStream(
@@ -572,10 +557,13 @@ public class AgentClientProvider extends AbstractClientProvider
TarArchiveEntry tarEntry = tarIn.getNextTarEntry();
while (tarEntry != null) {
File destPath = new File(destDir, tarEntry.getName());
+ File parent = destPath.getParentFile();
+ if (!parent.exists()) {
+ parent.mkdirs();
+ }
if (tarEntry.isDirectory()) {
destPath.mkdirs();
} else {
- destPath.createNewFile();
byte[] byteToRead = new byte[1024];
BufferedOutputStream buffOut =
new BufferedOutputStream(new FileOutputStream(destPath));
@@ -588,6 +576,9 @@ public class AgentClientProvider extends AbstractClientProvider
buffOut.close();
}
}
+ if ((tarEntry.getMode() & 0100) != 0) {
+ destPath.setExecutable(true);
+ }
tarEntry = tarIn.getNextTarEntry();
}
} finally {
@@ -598,6 +589,8 @@ public class AgentClientProvider extends AbstractClientProvider
private void retrieveConfigFile(RegistryOperations rops,
Configuration configuration, ConfigFile configFile, String name,
String user, File destDir) throws IOException, SliderException {
+ log.info("Retrieving config {} to {}", configFile.getDictionaryName(),
+ destDir);
PublishedConfiguration published = ClientUtils.getConfigFromRegistry(rops,
configuration, configFile.getDictionaryName(), name, user, true);
ClientUtils.saveOrReturnConfig(published, configFile.getType(),
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/511891c1/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java b/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
index eec5c38..63d5961 100644
--- a/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
+++ b/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
@@ -21,7 +21,6 @@ import com.google.common.base.Charsets;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.Path;
import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter.TemplateOutputter;
import org.codehaus.jackson.map.ObjectMapper;
import org.junit.Before;
import org.junit.Rule;
@@ -58,7 +57,7 @@ public class TestPublishedConfigurationOutputter {
PublishedConfigurationOutputter configurationOutputter =
PublishedConfigurationOutputter.createOutputter(ConfigFormat.JSON,
new PublishedConfiguration("description",
- config.entrySet(), null, null));
+ config.entrySet()));
String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
"");
@@ -78,7 +77,7 @@ public class TestPublishedConfigurationOutputter {
PublishedConfigurationOutputter configurationOutputter =
PublishedConfigurationOutputter.createOutputter(ConfigFormat.XML,
new PublishedConfiguration("description",
- config.entrySet(), null, null));
+ config.entrySet()));
String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
"");
@@ -99,7 +98,7 @@ public class TestPublishedConfigurationOutputter {
PublishedConfigurationOutputter configurationOutputter =
PublishedConfigurationOutputter.createOutputter(ConfigFormat.HADOOP_XML,
new PublishedConfiguration("description",
- config.entrySet(), null, null));
+ config.entrySet()));
String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
"");
@@ -118,7 +117,7 @@ public class TestPublishedConfigurationOutputter {
PublishedConfigurationOutputter configurationOutputter =
PublishedConfigurationOutputter.createOutputter(ConfigFormat.PROPERTIES,
new PublishedConfiguration("description",
- config.entrySet(), null, null));
+ config.entrySet()));
String output = configurationOutputter.asString();
assert output.contains("key1=val1");
@@ -145,7 +144,7 @@ public class TestPublishedConfigurationOutputter {
PublishedConfigurationOutputter configurationOutputter =
PublishedConfigurationOutputter.createOutputter(ConfigFormat.YAML,
new PublishedConfiguration("description",
- config.entrySet(), null, null));
+ config.entrySet()));
String output = configurationOutputter.asString().replaceAll("(\\r|\\n)",
"");
@@ -177,7 +176,7 @@ public class TestPublishedConfigurationOutputter {
PublishedConfigurationOutputter configurationOutputter =
PublishedConfigurationOutputter.createOutputter(ConfigFormat.ENV,
new PublishedConfiguration("description",
- envConfig.entrySet(), null, null));
+ envConfig.entrySet()));
String output = configurationOutputter.asString();
assert "content val1 ".equals(output);
@@ -192,7 +191,7 @@ public class TestPublishedConfigurationOutputter {
@Test
public void testTemplate1() throws IOException {
HashMap<String, String> templateConfig = new HashMap<>(config);
- templateConfig.put(TemplateOutputter.TEMPLATE_FILE, "templateFileName");
+ templateConfig.put(ConfigUtils.TEMPLATE_FILE, "templateFileName");
SliderFileSystem fileSystem = createNiceMock(SliderFileSystem.class);
expect(fileSystem.buildResourcePath(anyString())).andReturn(new Path("path")).anyTimes();
@@ -201,10 +200,12 @@ public class TestPublishedConfigurationOutputter {
PowerMock.replay(fileSystem);
+ ConfigUtils.prepConfigForTemplateOutputter(ConfigFormat.TEMPLATE,
+ templateConfig, fileSystem, "clusterName", null);
PublishedConfigurationOutputter configurationOutputter =
PublishedConfigurationOutputter.createOutputter(ConfigFormat.TEMPLATE,
new PublishedConfiguration("description",
- templateConfig.entrySet(), fileSystem, "clusterName"));
+ templateConfig.entrySet()));
String output = configurationOutputter.asString();
assert "content val1\n more val1 content".equals(output);