You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2015/03/19 04:40:10 UTC
[4/5] incubator-slider git commit: SLIDER-663. Make it easy to
develop and deploy application packages that are essentially shell commands
(part-II)
SLIDER-663. Make it easy to develop and deploy application packages that are essentially shell commands (part-II)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/210a1c94
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/210a1c94
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/210a1c94
Branch: refs/heads/develop
Commit: 210a1c94ca347380f7b22fba807c12dc1e30deff
Parents: df35225
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Mar 18 14:44:07 2015 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Mar 18 14:44:07 2015 -0700
----------------------------------------------------------------------
.../core/persist/AppDefinitionPersister.java | 64 +++++-
.../agent/TestAgentClientProvider2.java | 10 +-
.../agent/TestAppDefinitionPersister.java | 227 +++++++++++++++++++
3 files changed, 288 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/210a1c94/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
index 8f0f7b0..f394542 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
@@ -18,6 +18,7 @@
package org.apache.slider.core.persist;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.io.Files;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.Path;
@@ -26,7 +27,6 @@ import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.exceptions.BadClusterStateException;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.providers.agent.AgentKeys;
@@ -107,11 +107,23 @@ public class AppDefinitionPersister {
}
public void processSuppliedDefinitions(String clustername,
- AbstractClusterBuildingActionArgs buildInfo,
- ConfTreeOperations appConf)
+ AbstractClusterBuildingActionArgs buildInfo,
+ ConfTreeOperations appConf)
throws BadConfigException, IOException, BadCommandArgumentsException {
// if metainfo is provided add to the app instance
if (buildInfo.appMetaInfo != null) {
+
+ if (!buildInfo.appMetaInfo.canRead() || !buildInfo.appMetaInfo.isFile()) {
+ throw new BadConfigException("--metainfo file cannot be read.");
+ }
+
+ if (buildInfo.appDef != null) {
+ throw new BadConfigException("both --metainfo and --appdef may not be specified.");
+ }
+ if (SliderUtils.isSet(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) {
+ throw new BadConfigException("application.def must not be set if --metainfo is provided.");
+ }
+
File tempDir = Files.createTempDir();
File pkgSrcDir = new File(tempDir, "default");
pkgSrcDir.mkdirs();
@@ -127,6 +139,14 @@ public class AppDefinitionPersister {
}
if (buildInfo.appDef != null) {
+ if (SliderUtils.isSet(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) {
+ throw new BadConfigException("application.def must not be set if --appdef is provided.");
+ }
+
+ if (!buildInfo.appDef.exists()) {
+ throw new BadConfigException("--appdef is not a valid path.");
+ }
+
Path appDirPath = sliderFileSystem.buildAppDefDirPath(clustername);
appDefinitions.add(new AppDefinition(appDirPath, buildInfo.appDef, SliderKeys.DEFAULT_APP_PKG));
Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG);
@@ -135,16 +155,30 @@ public class AppDefinitionPersister {
}
if (buildInfo.addonDelegate.getAddonMap().size() > 0) {
+ if (SliderUtils.isUnset(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) {
+ throw new BadConfigException("addon package can only be specified if main app package is specified.");
+ }
+
List<String> addons = new ArrayList<String>();
Map<String, String> addonMap = buildInfo.addonDelegate.getAddonMap();
for (String key : addonMap.keySet()) {
+ File defPath = new File(addonMap.get(key));
+ if (SliderUtils.isUnset(addonMap.get(key))) {
+ throw new BadConfigException("Invalid path for addon package " + key);
+ }
+
+ if (!defPath.exists()) {
+ throw new BadConfigException("addon folder or package path is not valid.");
+ }
+
Path addonPath = sliderFileSystem.buildAddonDirPath(clustername, key);
String addonPkgName = "addon_" + key + ".zip";
- appDefinitions.add(new AppDefinition(addonPath, buildInfo.appDef, addonPkgName));
+ appDefinitions.add(new AppDefinition(addonPath, defPath, addonPkgName));
String addOnKey = AgentKeys.ADDON_PREFIX + key;
Path addonPkgPath = new Path(addonPath, addonPkgName);
log.info("Setting addon package {} to {}.", addOnKey, addonPkgPath);
appConf.getGlobalOptions().set(addOnKey, addonPkgPath);
+ addons.add(addOnKey);
}
String existingList = appConf.getGlobalOptions().get(AgentKeys.ADDONS);
@@ -156,19 +190,33 @@ public class AppDefinitionPersister {
}
+ @VisibleForTesting
+ public List<AppDefinitionPersister.AppDefinition> getAppDefinitions() {
+ return appDefinitions;
+ }
+
// Helper class to hold details for the app and addon packages
- class AppDefinition {
+ public class AppDefinition {
// The target folder where the package will be stored
- Path targetFolderInFs;
+ public Path targetFolderInFs;
// The on disk location of the app def package or folder
- File appDefPkgOrFolder;
+ public File appDefPkgOrFolder;
// Package name
- String pkgName;
+ public String pkgName;
public AppDefinition(Path targetFolderInFs, File appDefPkgOrFolder, String pkgName) {
this.targetFolderInFs = targetFolderInFs;
this.appDefPkgOrFolder = appDefPkgOrFolder;
this.pkgName = pkgName;
}
+
+ @Override
+ public String toString() {
+ return new StringBuilder().append("targetFolderInFs").append(" : ").append(targetFolderInFs.toString())
+ .append(", ")
+ .append("appDefPkgOrFolder").append(" : ").append(appDefPkgOrFolder.toString())
+ .append(", ")
+ .append("pkgName").append(" : ").append(pkgName).toString();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/210a1c94/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java
index 4bd9842..65e0fbd 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java
@@ -225,7 +225,7 @@ public class TestAgentClientProvider2 {
args.install = true;
try {
client.actionClient(args);
- }catch(BadCommandArgumentsException e) {
+ } catch (BadCommandArgumentsException e) {
log.info(e.getMessage());
Assert.assertTrue(e.getMessage().contains("A valid install location must be provided for the client"));
}
@@ -235,7 +235,7 @@ public class TestAgentClientProvider2 {
args.installLocation = dest;
try {
client.actionClient(args);
- }catch(BadCommandArgumentsException e) {
+ } catch (BadCommandArgumentsException e) {
log.info(e.getMessage());
Assert.assertTrue(e.getMessage().contains("Install path does not exist at"));
}
@@ -243,7 +243,7 @@ public class TestAgentClientProvider2 {
dest.mkdir();
try {
client.actionClient(args);
- }catch(BadCommandArgumentsException e) {
+ } catch (BadCommandArgumentsException e) {
log.info(e.getMessage());
Assert.assertTrue(e.getMessage().contains("A valid application package location required"));
}
@@ -253,7 +253,7 @@ public class TestAgentClientProvider2 {
args.clientConfig = tmpFile;
try {
client.actionClient(args);
- }catch(SliderException e) {
+ } catch (SliderException e) {
log.info(e.getMessage());
Assert.assertTrue(e.getMessage().contains("Invalid configuration. Must be a valid json file"));
}
@@ -261,7 +261,7 @@ public class TestAgentClientProvider2 {
args.clientConfig = null;
try {
client.actionClient(args);
- }catch(SliderException e) {
+ } catch (SliderException e) {
log.info(e.getMessage());
Assert.assertTrue(e.getMessage().contains("Not a valid app package. Could not read metainfo"));
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/210a1c94/slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java
new file mode 100644
index 0000000..eaf496c
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent;
+
+import com.google.common.io.Files;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.slider.common.params.ActionCreateArgs;
+import org.apache.slider.common.params.AddonArgsDelegate;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.persist.AppDefinitionPersister;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ */
+public class TestAppDefinitionPersister {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestAppDefinitionPersister.class);
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+
+ /**
+ * @BeforeClass public static void initialize() { BasicConfigurator.resetConfiguration();
+ * BasicConfigurator.configure(); }*
+ */
+
+
+ @Test
+ public void testAppDefinitionPersister() throws Exception {
+ Configuration configuration = new Configuration();
+ FileSystem fs = FileSystem.getLocal(configuration);
+ log.info("fs working dir is {}", fs.getWorkingDirectory().toString());
+ SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration);
+
+ AppDefinitionPersister adp = new AppDefinitionPersister(sliderFileSystem);
+ String clustername = "c1";
+ ActionCreateArgs buildInfo = new ActionCreateArgs();
+ buildInfo.appMetaInfo = null;
+ buildInfo.appDef = null;
+ buildInfo.addonDelegate = new AddonArgsDelegate();
+
+ // nothing to do
+ adp.processSuppliedDefinitions(clustername, buildInfo, null);
+ adp.persistPackages();
+ List<AppDefinitionPersister.AppDefinition> appDefinitions = adp.getAppDefinitions();
+ Assert.assertTrue(appDefinitions.size() == 0);
+
+ ConfTree ct = new ConfTree();
+ ConfTreeOperations appConf = new ConfTreeOperations(ct);
+ final File tempDir = Files.createTempDir();
+ final File metainfo = new File(tempDir, "metainfo.json");
+
+ // unreadable metainfo
+ buildInfo.appMetaInfo = metainfo;
+
+ try {
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ } catch (BadConfigException bce) {
+ log.info(bce.getMessage());
+ Assert.assertTrue(bce.getMessage().contains("--metainfo file cannot be read"));
+ }
+
+ try (PrintWriter writer = new PrintWriter(metainfo.getAbsolutePath(), "UTF-8")) {
+ writer.println("{");
+ writer.println("}");
+ }
+ buildInfo.appDef = metainfo;
+
+ try {
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ } catch (BadConfigException bce) {
+ log.info(bce.getMessage());
+ Assert.assertTrue(bce.getMessage().contains("both --metainfo and --appdef may not be specified"));
+ }
+
+ buildInfo.appDef = null;
+
+ appConf.getGlobalOptions().set(AgentKeys.APP_DEF, metainfo.getAbsolutePath());
+
+ try {
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ } catch (BadConfigException bce) {
+ log.info(bce.getMessage());
+ Assert.assertTrue(bce.getMessage().contains("application.def must not be set if --metainfo is provided"));
+ }
+
+ appConf.getGlobalOptions().remove(AgentKeys.APP_DEF);
+
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ appDefinitions = adp.getAppDefinitions();
+ Assert.assertTrue(appDefinitions.size() == 1);
+ Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip"));
+ log.info(appDefinitions.get(0).toString());
+ Assert.assertTrue(appDefinitions.get(0).appDefPkgOrFolder.toString().endsWith("default"));
+ Assert.assertTrue(appDefinitions.get(0).targetFolderInFs.toString().contains("cluster/c1/appdef"));
+ Assert.assertEquals("appPkg.zip", appDefinitions.get(0).pkgName);
+
+ buildInfo.appDef = tempDir;
+ buildInfo.appMetaInfo = null;
+
+ appConf.getGlobalOptions().set(AgentKeys.APP_DEF, metainfo.getAbsolutePath());
+
+ try {
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ } catch (BadConfigException bce) {
+ log.info(bce.getMessage());
+ Assert.assertTrue(bce.getMessage().contains("application.def must not be set if --appdef is provided"));
+ }
+
+ adp.getAppDefinitions().clear();
+ appConf.getGlobalOptions().remove(AgentKeys.APP_DEF);
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ appDefinitions = adp.getAppDefinitions();
+ Assert.assertTrue(appDefinitions.size() == 1);
+ Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip"));
+ log.info(appDefinitions.get(0).toString());
+ Assert.assertTrue(appDefinitions.get(0).appDefPkgOrFolder.toString().endsWith(tempDir.toString()));
+ Assert.assertTrue(appDefinitions.get(0).targetFolderInFs.toString().contains("cluster/c1/appdef"));
+ Assert.assertEquals("appPkg.zip", appDefinitions.get(0).pkgName);
+
+ adp.getAppDefinitions().clear();
+ buildInfo.appDef = null;
+ buildInfo.appMetaInfo = null;
+ appConf.getGlobalOptions().remove(AgentKeys.APP_DEF);
+
+ ArrayList<String> list = new ArrayList<String>() {{
+ add("addon1");
+ add("");
+ add("addon2");
+ add(metainfo.getAbsolutePath());
+ }};
+
+ buildInfo.addonDelegate.addonTuples = list;
+ try {
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ } catch (BadConfigException bce) {
+ log.info(bce.getMessage());
+ Assert.assertTrue(bce.getMessage().contains("addon package can only be specified if main app package is specified"));
+ }
+
+ buildInfo.appMetaInfo = metainfo;
+
+ try {
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ } catch (BadConfigException bce) {
+ log.info(bce.getMessage());
+ Assert.assertTrue(bce.getMessage().contains("Invalid path for addon package addon1"));
+ }
+
+ appConf.getGlobalOptions().remove(AgentKeys.APP_DEF);
+
+ list = new ArrayList<String>() {{
+ add("addon1");
+ add(tempDir.getAbsolutePath());
+ add("addon2");
+ add(metainfo.getAbsolutePath());
+ }};
+
+ buildInfo.addonDelegate.addonTuples = list;
+ adp.getAppDefinitions().clear();
+
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ appDefinitions = adp.getAppDefinitions();
+
+ Assert.assertTrue(appDefinitions.size() == 3);
+ Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip"));
+ Assert.assertTrue(appConf.getGlobalOptions().get("application.addon.addon1").contains(
+ "addons/addon1/addon_addon1.zip"));
+ Assert.assertTrue(appConf.getGlobalOptions().get("application.addon.addon2").contains(
+ "addons/addon2/addon_addon2.zip"));
+ log.info(appConf.getGlobalOptions().get("application.addons"));
+ Assert.assertTrue(appConf.getGlobalOptions().get("application.addons").contains(
+ "application.addon.addon2,application.addon.addon1")
+ || appConf.getGlobalOptions().get("application.addons").contains(
+ "application.addon.addon1,application.addon.addon2"));
+ int seen = 0;
+ for (AppDefinitionPersister.AppDefinition adp_ad : appDefinitions) {
+ if (adp_ad.pkgName.equals("appPkg.zip")) {
+ log.info(adp_ad.toString());
+ Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith("default"));
+ Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("cluster/c1/appdef"));
+ seen++;
+ }
+ if (adp_ad.pkgName.equals("addon_addon1.zip")) {
+ log.info(adp_ad.toString());
+ Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith(tempDir.toString()));
+ Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("addons/addon1"));
+ seen++;
+ }
+ if (adp_ad.pkgName.equals("addon_addon2.zip")) {
+ log.info(adp_ad.toString());
+ Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith("metainfo.json"));
+ Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("addons/addon2"));
+ seen++;
+ }
+ }
+ Assert.assertEquals(3, seen);
+ }
+}