You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by go...@apache.org on 2016/03/24 01:15:32 UTC
incubator-slider git commit: SLIDER-1090 Speed up SliderClient and AM
by reading metainfo from summary file (shanyu zhao via gourksaha)
Repository: incubator-slider
Updated Branches:
refs/heads/develop 4bebbf711 -> 2954a6f19
SLIDER-1090 Speed up SliderClient and AM by reading metainfo from summary file (shanyu zhao via gourksaha)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2954a6f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2954a6f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2954a6f1
Branch: refs/heads/develop
Commit: 2954a6f19fa0de65ac3b5f452199df343f24b416
Parents: 4bebbf7
Author: Gour Saha <go...@apache.org>
Authored: Wed Mar 23 17:15:09 2016 -0700
Committer: Gour Saha <go...@apache.org>
Committed: Wed Mar 23 17:15:09 2016 -0700
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 33 +++++++++
.../slider/providers/agent/AgentUtils.java | 60 +++++++++++++---
.../slider/providers/agent/TestAgentUtils.java | 76 ++++++++++++++++++++
3 files changed, 160 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2954a6f1/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index cf15dfc..2e5c45a 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -25,6 +25,7 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
@@ -179,6 +180,7 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.InterruptedIOException;
import java.io.PrintStream;
@@ -1355,6 +1357,36 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
+ private void createSummaryMetainfoFile(Path srcFile, Path destFile,
+ boolean overwrite) throws IOException {
+ FileSystem srcFs = srcFile.getFileSystem(getConfig());
+ try (InputStream inputStreamJson = SliderUtils
+ .getApplicationResourceInputStream(srcFs, srcFile, "metainfo.json");
+ InputStream inputStreamXml = SliderUtils
+ .getApplicationResourceInputStream(srcFs, srcFile, "metainfo.xml");) {
+ InputStream inputStream = null;
+ Path summaryFileInFs = null;
+ if (inputStreamJson != null) {
+ inputStream = inputStreamJson;
+ summaryFileInFs = new Path(destFile.getParent(), destFile.getName()
+ + ".metainfo.json");
+ log.info("Found JSON metainfo file in package");
+ } else if (inputStreamXml != null) {
+ inputStream = inputStreamXml;
+ summaryFileInFs = new Path(destFile.getParent(), destFile.getName()
+ + ".metainfo.xml");
+ log.info("Found XML metainfo file in package");
+ }
+ if (inputStream != null) {
+ try (FSDataOutputStream dataOutputStream = sliderFileSystem
+ .getFileSystem().create(summaryFileInFs, overwrite)) {
+ log.info("Creating summary metainfo file");
+ IOUtils.copy(inputStream, dataOutputStream);
+ }
+ }
+ }
+ }
+
private int actionPackageInstall(ActionPackageArgs actionPackageArgs)
throws YarnException, IOException {
requireArgumentSet(Arguments.ARG_NAME, actionPackageArgs.name);
@@ -1375,6 +1407,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
log.info("Installing package {} to {} (overwrite set to {})", srcFile,
fileInFs, actionPackageArgs.replacePkg);
fs.copyFromLocalFile(false, actionPackageArgs.replacePkg, srcFile, fileInFs);
+ createSummaryMetainfoFile(srcFile, fileInFs, actionPackageArgs.replacePkg);
String destPathWithHomeDir = Path
.getPathWithoutSchemeAndAuthority(fileInFs).toString();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2954a6f1/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
index b38dc6e..cfcfc5d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
@@ -16,6 +16,7 @@
*/
package org.apache.slider.providers.agent;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.slider.common.tools.SliderFileSystem;
@@ -40,30 +41,55 @@ import java.io.InputStream;
public class AgentUtils {
private static final Logger log = LoggerFactory.getLogger(AgentUtils.class);
+ public static Metainfo getApplicationMetainfoFromSummaryFile(
+ SliderFileSystem fileSystem, String metainfoPath, boolean metainfoForAddon) {
+ FileSystem fs = fileSystem.getFileSystem();
+ Path appPathXML = new Path(metainfoPath + ".metainfo.xml");
+ Path appPathJson = new Path(metainfoPath + ".metainfo.json");
+ Path appPathUsed = null;
+ try {
+ FSDataInputStream appStream = null;
+ if (fs.exists(appPathXML)) {
+ appPathUsed = appPathXML;
+ appStream = fs.open(appPathXML);
+ return parseMetainfo(appStream, metainfoForAddon, "xml");
+ } else if (fs.exists(appPathJson)) {
+ appPathUsed = appPathJson;
+ appStream = fs.open(appPathJson);
+ return parseMetainfo(appStream, metainfoForAddon, "json");
+ }
+ } catch (IOException e) {
+ log.info("Failed to get metainfo from summary file {} - {}", appPathUsed,
+ e.getMessage());
+ log.debug("Failed to get metainfo", e);
+ }
+ return null;
+ }
+
public static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
String metainfoPath, boolean metainfoForAddon) throws IOException,
BadConfigException {
log.info("Reading metainfo at {}", metainfoPath);
+ Metainfo metainfo = getApplicationMetainfoFromSummaryFile(fileSystem,
+ metainfoPath, metainfoForAddon);
+ if (metainfo != null) {
+ log.info("Got metainfo from summary file");
+ return metainfo;
+ }
+
FileSystem fs = fileSystem.getFileSystem();
Path appPath = new Path(metainfoPath);
- Metainfo metainfo = null;
- AbstractMetainfoParser metainfoParser = null;
- if (metainfoForAddon) {
- metainfoParser = new AddonPackageMetainfoParser();
- } else {
- metainfoParser = new MetainfoParser();
- }
InputStream metainfoJsonStream = SliderUtils.getApplicationResourceInputStream(
fs, appPath, "metainfo.json");
if (metainfoJsonStream == null) {
InputStream metainfoXMLStream = SliderUtils.getApplicationResourceInputStream(
fs, appPath, "metainfo.xml");
if (metainfoXMLStream != null) {
- metainfo = metainfoParser.fromXmlStream(metainfoXMLStream);
+ metainfo = parseMetainfo(metainfoXMLStream, metainfoForAddon, "xml");
}
} else {
- metainfo = metainfoParser.fromJsonStream(metainfoJsonStream);
+ metainfo = parseMetainfo(metainfoJsonStream, metainfoForAddon, "json");
}
if (metainfo == null) {
@@ -74,6 +100,22 @@ public class AgentUtils {
return metainfo;
}
+ private static Metainfo parseMetainfo(InputStream stream,
+ boolean metainfoForAddon, String type) throws IOException {
+ AbstractMetainfoParser metainfoParser = null;
+ if (metainfoForAddon) {
+ metainfoParser = new AddonPackageMetainfoParser();
+ } else {
+ metainfoParser = new MetainfoParser();
+ }
+ if (type.equals("xml")) {
+ return metainfoParser.fromXmlStream(stream);
+ } else if (type.equals("json")) {
+ return metainfoParser.fromJsonStream(stream);
+ }
+ return null;
+ }
+
static DefaultConfig getDefaultConfig(SliderFileSystem fileSystem,
String appDef, String configFileName)
throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2954a6f1/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java
new file mode 100644
index 0000000..db8c138
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java
@@ -0,0 +1,76 @@
+package org.apache.slider.providers.agent;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.providers.agent.application.metadata.Metainfo;
+import org.apache.slider.tools.TestUtility;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestAgentUtils {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestAgentUtils.class);
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+ private static final String metainfo_str = "<metainfo>\n"
+ + " <schemaVersion>2.0</schemaVersion>\n"
+ + " <application>\n"
+ + " <name>MYTESTAPPLICATION</name>\n"
+ + " <comment>\n"
+ + " My Test Application\n"
+ + " </comment>\n"
+ + " <version>1.0</version>\n"
+ + " <type>YARN-APP</type>\n"
+ + " <components>\n"
+ + " <component>\n"
+ + " <name>REST</name>\n"
+ + " <category>MASTER</category>\n"
+ + " <commandScript>\n"
+ + " <script>scripts/rest.py</script>\n"
+ + " <scriptType>PYTHON</scriptType>\n"
+ + " <timeout>600</timeout>\n"
+ + " </commandScript>\n"
+ + " </component>\n"
+ + " </components>\n"
+ + " </application>\n"
+ + "</metainfo>";
+
+ @Test
+ public void testGetApplicationMetainfo() throws Exception {
+ String zipFileName = TestUtility.createAppPackage(
+ folder,
+ "testpkg",
+ "test.zip",
+ "target/test-classes/org/apache/slider/common/tools/test");
+ Configuration configuration = new Configuration();
+ FileSystem fs = FileSystem.getLocal(configuration);
+ log.info("fs working dir is {}", fs.getWorkingDirectory().toString());
+ SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration);
+
+ // Without accompany metainfo file, read metainfo from the zip file
+ Metainfo metainfo = AgentUtils.getApplicationMetainfo(
+ sliderFileSystem, zipFileName, false);
+ Assert.assertNotNull(metainfo.getApplication());
+ Assert.assertEquals("STORM", metainfo.getApplication().getName());
+
+ // With accompany metainfo file, read metainfo from the accompany file
+ String acompanyFileName = zipFileName + ".metainfo.xml";
+ File f = new File(acompanyFileName);
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(f))) {
+ writer.write(metainfo_str);
+ }
+ metainfo = AgentUtils.getApplicationMetainfo(
+ sliderFileSystem, zipFileName, false);
+ Assert.assertNotNull(metainfo.getApplication());
+ Assert.assertEquals("MYTESTAPPLICATION", metainfo.getApplication().getName());
+ }
+}