You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/09/10 18:10:26 UTC
[08/36] git commit: SLIDER-407. Add install package command to
install app packages
SLIDER-407. Add install package command to install app packages
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/b4c88a4b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/b4c88a4b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/b4c88a4b
Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry
Commit: b4c88a4b30309fb78f976c783932a133d9fe0058
Parents: 8da1503
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Mon Sep 8 06:36:04 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Mon Sep 8 06:36:04 2014 -0700
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 60 +++++--
.../org/apache/slider/common/SliderKeys.java | 2 +
.../common/params/AbstractActionArgs.java | 3 +-
.../common/params/ActionInstallPackageArgs.java | 58 +++++++
.../apache/slider/common/params/Arguments.java | 1 +
.../apache/slider/common/params/ClientArgs.java | 12 +-
.../slider/common/params/SliderActions.java | 2 +
.../slider/common/tools/CoreFileSystem.java | 12 ++
.../actions/TestActionInstallPackage.groovy | 158 +++++++++++++++++++
9 files changed, 296 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b4c88a4b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index ef6448d..98074ec 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -48,6 +48,7 @@ import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.params.AbstractActionArgs;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
+import org.apache.slider.common.params.ActionInstallPackageArgs;
import org.apache.slider.common.params.ActionAMSuicideArgs;
import org.apache.slider.common.params.ActionCreateArgs;
import org.apache.slider.common.params.ActionEchoArgs;
@@ -301,20 +302,22 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
int exitCode = EXIT_SUCCESS;
String clusterName = serviceArgs.getClusterName();
// actions
- if (ACTION_BUILD.equals(action)) {
+ if (ACTION_INSTALL_PACKAGE.equals(action)) {
+ exitCode = actionInstallPkg(serviceArgs.getActionInstallPackageArgs());
+ } else if (ACTION_BUILD.equals(action)) {
exitCode = actionBuild(clusterName, serviceArgs.getActionBuildArgs());
} else if (ACTION_CREATE.equals(action)) {
exitCode = actionCreate(clusterName, serviceArgs.getActionCreateArgs());
} else if (ACTION_FREEZE.equals(action)) {
exitCode = actionFreeze(clusterName,
- serviceArgs.getActionFreezeArgs());
+ serviceArgs.getActionFreezeArgs());
} else if (ACTION_THAW.equals(action)) {
exitCode = actionThaw(clusterName, serviceArgs.getActionThawArgs());
} else if (ACTION_DESTROY.equals(action)) {
exitCode = actionDestroy(clusterName);
} else if (ACTION_EXISTS.equals(action)) {
exitCode = actionExists(clusterName,
- serviceArgs.getActionExistsArgs().live);
+ serviceArgs.getActionExistsArgs().live);
} else if (ACTION_FLEX.equals(action)) {
exitCode = actionFlex(clusterName, serviceArgs.getActionFlexArgs());
} else if (ACTION_GETCONF.equals(action)) {
@@ -322,15 +325,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
} else if (ACTION_HELP.equals(action) ||
ACTION_USAGE.equals(action)) {
log.info(serviceArgs.usage());
-
} else if (ACTION_KILL_CONTAINER.equals(action)) {
exitCode = actionKillContainer(clusterName,
serviceArgs.getActionKillContainerArgs());
-
} else if (ACTION_AM_SUICIDE.equals(action)) {
exitCode = actionAmSuicide(clusterName,
serviceArgs.getActionAMSuicideArgs());
-
} else if (ACTION_LIST.equals(action)) {
exitCode = actionList(clusterName);
} else if (ACTION_REGISTRY.equals(action)) {
@@ -592,6 +592,50 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
/**
+ * Upload application package to user home directory
+ *
+ * @param installPkgInfo the arguments needed to build the cluster
+ * @throws YarnException Yarn problems
+ * @throws IOException other problems
+ * @throws BadCommandArgumentsException bad arguments.
+ */
+ public int actionInstallPkg(ActionInstallPackageArgs installPkgInfo) throws
+ YarnException,
+ IOException {
+
+ Path srcFile = null;
+ if (null == installPkgInfo.name || installPkgInfo.name.length() == 0) {
+ throw new BadCommandArgumentsException("A valid application name is required.");
+ }
+
+ if (null == installPkgInfo.packageURI || installPkgInfo.packageURI.length() == 0) {
+ throw new BadCommandArgumentsException("A valid application package is required.");
+ } else {
+ File pkgFile = new File(installPkgInfo.packageURI);
+ if (!pkgFile.exists() || pkgFile.isDirectory()) {
+ throw new BadCommandArgumentsException("Unable to access supplied pkg file at " +
+ pkgFile.getAbsolutePath());
+ } else {
+ srcFile = new Path(pkgFile.toURI());
+ }
+ }
+
+ Path pkgPath = sliderFileSystem.buildPackageDirPath(installPkgInfo.name);
+ sliderFileSystem.getFileSystem().mkdirs(pkgPath);
+
+ Path fileInFs = new Path(pkgPath, srcFile.getName());
+ log.info("Installing package {} at {} and overwrite is {}.", srcFile, fileInFs, installPkgInfo.replacePkg);
+ if (sliderFileSystem.getFileSystem().exists(fileInFs) && !installPkgInfo.replacePkg) {
+ throw new BadCommandArgumentsException("Pkg exists at " +
+ fileInFs.toUri().toString() +
+ ". Use --replacePkg true to overwrite.");
+ }
+
+ sliderFileSystem.getFileSystem().copyFromLocalFile(false, installPkgInfo.replacePkg, srcFile, fileInFs);
+ return EXIT_SUCCESS;
+ }
+
+ /**
* Update the cluster specification
*
* @param clustername cluster name
@@ -1056,8 +1100,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
clusterPerms);
- // add AM and provider specific artifacts to the resource map
- Map<String, LocalResource> providerResources;
// standard AM resources
sliderAM.prepareAMAndConfigForLaunch(sliderFileSystem,
config,
@@ -1102,7 +1144,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
clusterDirectory,
generatedConfDirPath,
clusterSecure
- );
+ );
// now add the image if it was set
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b4c88a4b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 1d0a6f5..14528b1 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -112,6 +112,8 @@ public interface SliderKeys extends SliderXmlConfKeys {
String CLUSTER_DIRECTORY = "cluster";
+ String PACKAGE_DIRECTORY = "package";
+
/**
* JVM property to define the slider configuration directory;
* this is set by the slider script: {@value}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b4c88a4b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
index d18c85b..41f6a3f 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
@@ -124,7 +124,8 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
int actionArgSize = parameters.size();
if (minArgs > actionArgSize) {
throw new BadCommandArgumentsException(
- ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS + getActionName());
+ ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS + getActionName() +
+ " Expected minimum " + minArgs + " but got " + actionArgSize);
}
int maxArgs = getMaxParams();
if (maxArgs == -1) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b4c88a4b/slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java
new file mode 100644
index 0000000..646e795
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionInstallPackageArgs.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.beust.jcommander.ParametersDelegate;
+
+@Parameters(commandNames = {SliderActions.ACTION_INSTALL_PACKAGE},
+ commandDescription = SliderActions.DESCRIBE_ACTION_INSTALL_PACKAGE)
+
+public class ActionInstallPackageArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_INSTALL_PACKAGE;
+ }
+
+ @Parameter(names = {ARG_PACKAGE},
+ description = "Path to app package on local disk")
+ public String packageURI;
+
+ @Parameter(names = {ARG_NAME},
+ description = "The type of the package")
+ public String name;
+
+ @Parameter(names = {ARG_REPLACE_PKG}, description = "Overwrite existing package")
+ public boolean replacePkg = false;
+
+ /**
+ * Get the min #of params expected
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxParams() {
+ return 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b4c88a4b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index 764ffca..7f8aa83 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -77,6 +77,7 @@ public interface Arguments {
String ARG_ZKPATH = "--zkpath";
String ARG_ZKPORT = "--zkport";
String ARG_ZKHOSTS = "--zkhosts";
+ String ARG_REPLACE_PKG = "--replacepkg";
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b4c88a4b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
index ca854f1..7173a85 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
@@ -46,6 +46,7 @@ public class ClientArgs extends CommonArgs {
private AbstractClusterBuildingActionArgs buildingActionArgs;
private final ActionAMSuicideArgs actionAMSuicideArgs = new ActionAMSuicideArgs();
private final ActionBuildArgs actionBuildArgs = new ActionBuildArgs();
+ private final ActionInstallPackageArgs actionInstallPackageArgs = new ActionInstallPackageArgs();
private final ActionUpdateArgs actionUpdateArgs = new ActionUpdateArgs();
private final ActionCreateArgs actionCreateArgs = new ActionCreateArgs();
private final ActionDestroyArgs actionDestroyArgs = new ActionDestroyArgs();
@@ -90,7 +91,8 @@ public class ClientArgs extends CommonArgs {
actionStatusArgs,
actionThawArgs,
actionHelpArgs,
- actionVersionArgs
+ actionVersionArgs,
+ actionInstallPackageArgs
);
}
@@ -122,6 +124,9 @@ public class ClientArgs extends CommonArgs {
return actionBuildArgs;
}
+ public ActionInstallPackageArgs getActionInstallPackageArgs() {
+ return actionInstallPackageArgs; }
+
public ActionUpdateArgs getActionUpdateArgs() {
return actionUpdateArgs;
}
@@ -192,7 +197,10 @@ public class ClientArgs extends CommonArgs {
} else if (SliderActions.ACTION_UPDATE.equals(action)) {
bindCoreAction(actionUpdateArgs);
- } else if (SliderActions.ACTION_FREEZE.equals(action)) {
+ } else if (SliderActions.ACTION_INSTALL_PACKAGE.equals(action)) {
+ bindCoreAction(actionInstallPackageArgs);
+
+ }else if (SliderActions.ACTION_FREEZE.equals(action)) {
bindCoreAction(actionFreezeArgs);
} else if (SliderActions.ACTION_THAW.equals(action)) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b4c88a4b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
index ba2bca9..964f184 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
@@ -44,6 +44,7 @@ public interface SliderActions {
String ACTION_THAW = "start";
String ACTION_USAGE = "usage";
String ACTION_VERSION = "version";
+ String ACTION_INSTALL_PACKAGE = "install-package";
String DESCRIBE_ACTION_AM_SUICIDE =
"Tell the Slider Application Master to simulate a process failure by terminating itself";
String DESCRIBE_ACTION_BUILD =
@@ -76,4 +77,5 @@ public interface SliderActions {
"Start a stopped application";
String DESCRIBE_ACTION_VERSION =
"Print the Slider version information";
+ String DESCRIBE_ACTION_INSTALL_PACKAGE = "Install the application package in the home directory under sub-folder packages";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b4c88a4b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index def252a..955d991 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -122,6 +122,18 @@ public class CoreFileSystem {
}
/**
+ * Build up the path string for package install location -no attempt to
+ * create the directory is made
+ *
+ * @return the path for persistent app package
+ */
+ public Path buildPackageDirPath(String packageName) {
+ Preconditions.checkNotNull(packageName);
+ Path path = getBaseApplicationPath();
+ return new Path(path, SliderKeys.PACKAGE_DIRECTORY + "/" + packageName);
+ }
+
+ /**
* Create the Slider cluster path for a named cluster and all its subdirs
* This is a directory; a mkdirs() operation is executed
* to ensure that it is there.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/b4c88a4b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy
new file mode 100644
index 0000000..4d89e9b
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionInstallPackage.groovy
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.actions
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.core.exceptions.BadCommandArgumentsException
+import org.apache.slider.core.main.ServiceLauncher
+import org.junit.Before
+import org.junit.Test
+
+/**
+ * existence tests
+ */
+@CompileStatic
+@Slf4j
+
+class TestActionInstallPackage extends AgentMiniClusterTestBase {
+
+ @Before
+ public void setup() {
+ super.setup()
+ createMiniCluster("", configuration, 1, false)
+ }
+
+ @Test
+ public void testInstallPackageFailsWithNoPackageName() throws Throwable {
+ try {
+ ServiceLauncher launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_INSTALL_PACKAGE
+ ],
+ )
+ fail("expected an exception, got a status code " + launcher.serviceExitCode)
+ } catch (BadCommandArgumentsException e) {
+ assert e.message.contains("A valid application name is required")
+ }
+ }
+
+ @Test
+ public void testInstallPackageFailsWithNoPackagePath() throws Throwable {
+ try {
+ ServiceLauncher launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_INSTALL_PACKAGE,
+ Arguments.ARG_NAME, "hbase"
+ ],
+ )
+ fail("expected an exception, got a status code " + launcher.serviceExitCode)
+ } catch (BadCommandArgumentsException e) {
+ assert e.message.contains("A valid application package is required.")
+ }
+ }
+
+ @Test
+ public void testInstallPackageFailsWithInvalidPackagePath() throws Throwable {
+ try {
+ ServiceLauncher launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_INSTALL_PACKAGE,
+ Arguments.ARG_NAME, "hbase",
+ Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties",
+ ],
+ )
+ launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_INSTALL_PACKAGE,
+ Arguments.ARG_NAME, "hbase",
+ Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties",
+ ],
+ )
+ fail("expected an exception, got a status code " + launcher.serviceExitCode)
+ } catch (BadCommandArgumentsException e) {
+ assert e.message.contains("Use --replacePkg true to overwrite")
+ }
+ }
+
+ @Test
+ public void testInstallPackageFailsWithNeedingReplaceFlag() throws Throwable {
+ try {
+ ServiceLauncher launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_INSTALL_PACKAGE,
+ Arguments.ARG_NAME, "hbase",
+ Arguments.ARG_PACKAGE, "unlikely_to_be_a_file_path",
+ ],
+ )
+ fail("expected an exception, got a status code " + launcher.serviceExitCode)
+ } catch (BadCommandArgumentsException e) {
+ assert e.message.contains("Unable to access supplied pkg file at")
+ }
+ }
+
+ @Test
+ public void testInstallPackageWithReplace() throws Throwable {
+ try {
+ ServiceLauncher launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_INSTALL_PACKAGE,
+ Arguments.ARG_NAME, "hbase",
+ Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties",
+ ],
+ )
+ launcher = launchClientAgainstMiniMR(
+ //config includes RM binding info
+ new YarnConfiguration(miniCluster.config),
+ //varargs list of command line params
+ [
+ SliderActions.ACTION_INSTALL_PACKAGE,
+ Arguments.ARG_NAME, "hbase",
+ Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties",
+ Arguments.ARG_REPLACE_PKG, true
+ ],
+ )
+ } catch (BadCommandArgumentsException e) {
+ log.info(e.message)
+ assert e.message.contains("Use --replacePkg true to overwrite")
+ }
+ }
+}