You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/11/04 22:37:08 UTC
[5/5] incubator-slider git commit: SLIDER-961 clean up SliderClient
code
SLIDER-961 clean up SliderClient code
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/227339e5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/227339e5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/227339e5
Branch: refs/heads/develop
Commit: 227339e5c661571a989aa3f5e2d7ff5ff9ac667d
Parents: 88f881b
Author: Steve Loughran <st...@apache.org>
Authored: Wed Nov 4 21:36:47 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Nov 4 21:36:47 2015 +0000
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 717 +++++++------------
.../slider/common/tools/CoreFileSystem.java | 2 +-
.../apache/slider/common/tools/SliderUtils.java | 25 +-
.../slider/core/launch/AppMasterLauncher.java | 5 +-
.../core/launch/ClasspathConstructor.java | 1 -
.../slider/core/launch/CommandLineBuilder.java | 55 ++
.../utility/AbstractSliderLaunchedService.java | 20 +-
.../agent/actions/TestActionPackage.groovy | 7 +-
8 files changed, 333 insertions(+), 499 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index de0e07e..07c915c 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -19,7 +19,6 @@
package org.apache.slider.client;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
import com.google.common.io.Files;
import org.apache.commons.collections.CollectionUtils;
@@ -65,9 +64,6 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
-import org.apache.slider.api.InternalKeys;
-import org.apache.slider.api.OptionKeys;
-import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.SliderClusterProtocol;
import org.apache.slider.api.StateValues;
import org.apache.slider.api.proto.Messages;
@@ -77,7 +73,6 @@ import org.apache.slider.client.ipc.SliderClusterOperations;
import org.apache.slider.common.Constants;
import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.params.AbstractActionArgs;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
import org.apache.slider.common.params.ActionAMSuicideArgs;
@@ -106,11 +101,9 @@ import org.apache.slider.common.params.Arguments;
import org.apache.slider.common.params.ClientArgs;
import org.apache.slider.common.params.CommonArgs;
import org.apache.slider.common.params.LaunchArgsAccessor;
-import org.apache.slider.common.params.SliderActions;
import org.apache.slider.common.tools.ConfigHelper;
import org.apache.slider.common.tools.Duration;
import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.common.tools.SliderVersionInfo;
import org.apache.slider.core.build.InstanceBuilder;
import org.apache.slider.core.build.InstanceIO;
@@ -132,7 +125,6 @@ import org.apache.slider.core.exceptions.UsageException;
import org.apache.slider.core.exceptions.WaitTimeoutException;
import org.apache.slider.core.launch.AppMasterLauncher;
import org.apache.slider.core.launch.ClasspathConstructor;
-import org.apache.slider.core.launch.CommandLineBuilder;
import org.apache.slider.core.launch.JavaCommandLineBuilder;
import org.apache.slider.core.launch.LaunchedApplication;
import org.apache.slider.core.launch.RunningApplication;
@@ -203,38 +195,13 @@ import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.ServiceRecordMarshal;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.currentUser;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.extractServiceRecords;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.listServiceRecords;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.serviceclassPath;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.statChildren;
-import static org.apache.slider.common.params.SliderActions.ACTION_AM_SUICIDE;
-import static org.apache.slider.common.params.SliderActions.ACTION_BUILD;
-import static org.apache.slider.common.params.SliderActions.ACTION_CLIENT;
-import static org.apache.slider.common.params.SliderActions.ACTION_CREATE;
-import static org.apache.slider.common.params.SliderActions.ACTION_DEPENDENCY;
-import static org.apache.slider.common.params.SliderActions.ACTION_DESTROY;
-import static org.apache.slider.common.params.SliderActions.ACTION_DIAGNOSTICS;
-import static org.apache.slider.common.params.SliderActions.ACTION_EXISTS;
-import static org.apache.slider.common.params.SliderActions.ACTION_FLEX;
-import static org.apache.slider.common.params.SliderActions.ACTION_FREEZE;
-import static org.apache.slider.common.params.SliderActions.ACTION_HELP;
-import static org.apache.slider.common.params.SliderActions.ACTION_INSTALL_KEYTAB;
-import static org.apache.slider.common.params.SliderActions.ACTION_INSTALL_PACKAGE;
-import static org.apache.slider.common.params.SliderActions.ACTION_KEYTAB;
-import static org.apache.slider.common.params.SliderActions.ACTION_KILL_CONTAINER;
-import static org.apache.slider.common.params.SliderActions.ACTION_LIST;
-import static org.apache.slider.common.params.SliderActions.ACTION_LOOKUP;
-import static org.apache.slider.common.params.SliderActions.ACTION_PACKAGE;
-import static org.apache.slider.common.params.SliderActions.ACTION_REGISTRY;
-import static org.apache.slider.common.params.SliderActions.ACTION_RESOLVE;
-import static org.apache.slider.common.params.SliderActions.ACTION_STATUS;
-import static org.apache.slider.common.params.SliderActions.ACTION_THAW;
-import static org.apache.slider.common.params.SliderActions.ACTION_UPDATE;
-import static org.apache.slider.common.params.SliderActions.ACTION_UPGRADE;
-import static org.apache.slider.common.params.SliderActions.ACTION_VERSION;
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.*;
+import static org.apache.slider.api.InternalKeys.*;
+import static org.apache.slider.api.OptionKeys.*;
+import static org.apache.slider.api.ResourceKeys.*;
+import static org.apache.slider.common.params.SliderActions.*;
+import static org.apache.slider.common.tools.SliderUtils.*;
+
/**
* Client service for Slider
@@ -287,6 +254,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
/**
* The YARN registry service
*/
+ @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
private RegistryOperations registryOperations;
/**
@@ -317,19 +285,19 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
ConfigHelper.injectSliderXMLResource();
// yarn-ify
YarnConfiguration yarnConfiguration = new YarnConfiguration(config);
- return SliderUtils.patchConfiguration(yarnConfiguration);
+ return patchConfiguration(yarnConfiguration);
}
@Override
protected void serviceInit(Configuration conf) throws Exception {
- Configuration clientConf = SliderUtils.loadSliderClientXML();
+ Configuration clientConf = loadSliderClientXML();
ConfigHelper.mergeConfigurations(conf, clientConf, SLIDER_CLIENT_XML, true);
serviceArgs.applyDefinitions(conf);
serviceArgs.applyFileSystemBinding(conf);
// init security with our conf
- if (SliderUtils.isHadoopClusterSecure(conf)) {
- SliderUtils.forceLogin();
- SliderUtils.initProcessSecurity(conf);
+ if (isHadoopClusterSecure(conf)) {
+ forceLogin();
+ initProcessSecurity(conf);
}
AbstractActionArgs coreAction = serviceArgs.getCoreAction();
if (coreAction.getHadoopServicesRequired()) {
@@ -362,9 +330,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// choose the action
String action = serviceArgs.getAction();
- if (SliderUtils.isUnset(action)) {
- throw new SliderException(EXIT_USAGE,
- serviceArgs.usage());
+ if (isUnset(action)) {
+ throw new SliderException(EXIT_USAGE, serviceArgs.usage());
}
int exitCode = EXIT_SUCCESS;
@@ -483,7 +450,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return exitCode;
}
-/**
+ /**
* Perform everything needed to init the hadoop binding.
* This assumes that the service is already in inited or started state
* @throws IOException
@@ -491,7 +458,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
*/
protected void initHadoopBinding() throws IOException, SliderException {
// validate the client
- SliderUtils.validateSliderClientEnvironment(null);
+ validateSliderClientEnvironment(null);
//create the YARN client
yarnClient = new SliderYarnClientImpl();
yarnClient.init(getConfig());
@@ -626,7 +593,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
public int actionDestroy(String clustername,
ActionDestroyArgs destroyArgs) throws YarnException, IOException {
// verify that a live cluster isn't there
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
//no=op, it is now mandatory.
verifyBindingsDefined();
verifyNoLiveClusters(clustername, "Destroy");
@@ -645,9 +612,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
throw new UsageException("Destroy will permanently delete directories and registries. "
+ "Reissue this command with the --force option if you want to proceed.");
}
- boolean deleted =
- fs.delete(clusterDirectory, true);
- if (!deleted) {
+ if (!fs.delete(clusterDirectory, true)) {
log.warn("Filesystem returned false from delete() operation");
}
@@ -743,33 +708,26 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// Otherwise the internal app config and resources states of the app will be
// unwantedly modified and the change will take effect to the running app
// immediately.
- if (template != null && resources == null) {
- throw new BadCommandArgumentsException(
+ require(!(template != null && resources == null),
"Option %s must be specified with option %s",
Arguments.ARG_RESOURCES, Arguments.ARG_TEMPLATE);
- }
- if (resources != null && template == null) {
- throw new BadCommandArgumentsException(
+
+ require(!(resources != null && template == null),
"Option %s must be specified with option %s",
Arguments.ARG_TEMPLATE, Arguments.ARG_RESOURCES);
- }
// For upgrade spec, both --template and --resources should be specified
// and neither of --containers or --components should be used
if (template != null && resources != null) {
- if (CollectionUtils.isNotEmpty(containers)) {
- throw new BadCommandArgumentsException(
+ require(CollectionUtils.isEmpty(containers),
"Option %s cannot be specified with %s or %s",
Arguments.ARG_CONTAINERS, Arguments.ARG_TEMPLATE,
Arguments.ARG_RESOURCES);
- }
- if (CollectionUtils.isNotEmpty(components)) {
- throw new BadCommandArgumentsException(
- "Option %s cannot be specified with %s or %s",
- Arguments.ARG_COMPONENTS, Arguments.ARG_TEMPLATE,
- Arguments.ARG_RESOURCES);
- }
-
+ require(CollectionUtils.isEmpty(components),
+ "Option %s cannot be specified with %s or %s",
+ Arguments.ARG_COMPONENTS, Arguments.ARG_TEMPLATE,
+ Arguments.ARG_RESOURCES);
+
// not an error to try to upgrade a stopped cluster, just return success
// code, appropriate log messages have already been dumped
if (!isAppInRunningState(clustername)) {
@@ -794,7 +752,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private int actionUpgradeContainers(String clustername,
ActionUpgradeArgs upgradeArgs) throws YarnException, IOException {
verifyBindingsDefined();
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
int waittime = upgradeArgs.getWaittime(); // ignored for now
String text = "Upgrade containers";
log.debug("actionUpgradeContainers({}, reason={}, wait={})", clustername,
@@ -892,13 +850,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return false;
}
log.debug("App to upgrade was found: {}:\n{}", clustername,
- new SliderUtils.OnDemandReportStringifier(app));
- if (app.getYarnApplicationState().ordinal() >= YarnApplicationState.FINISHED
- .ordinal()) {
- log.info(
- "Cluster {} is in a terminated state {}. Use command '{}' instead.",
- clustername, app.getYarnApplicationState(),
- SliderActions.ACTION_UPDATE);
+ new OnDemandReportStringifier(app));
+ if (app.getYarnApplicationState().ordinal() >= YarnApplicationState.FINISHED.ordinal()) {
+ log.info("Cluster {} is in a terminated state {}. Use command '{}' instead.",
+ clustername, app.getYarnApplicationState(), ACTION_UPDATE);
return false;
}
@@ -915,7 +870,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private static void checkForCredentials(Configuration conf,
ConfTree tree) throws IOException {
- if (tree.credentials == null || tree.credentials.size()==0) {
+ if (tree.credentials == null || tree.credentials.isEmpty()) {
log.info("No credentials requested");
return;
}
@@ -930,10 +885,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
Configuration c = new Configuration(conf);
c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
- CredentialProvider credentialProvider =
- CredentialProviderFactory.getProviders(c).get(0);
- Set<String> existingAliases =
- new HashSet<>(credentialProvider.getAliases());
+ CredentialProvider credentialProvider = CredentialProviderFactory.getProviders(c).get(0);
+ Set<String> existingAliases = new HashSet<>(credentialProvider.getAliases());
for (String alias : aliases) {
if (existingAliases.contains(alias.toLowerCase(Locale.ENGLISH))) {
log.info("Credentials for " + alias + " found in " + provider);
@@ -949,21 +902,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
}
} finally {
- if (br != null) {
- br.close();
- }
+ org.apache.hadoop.io.IOUtils.closeStream(br);
}
}
private static char[] readOnePassword(String alias) throws IOException {
- BufferedReader br = null;
- try {
- br = new BufferedReader(new InputStreamReader(System.in));
+ try(BufferedReader br = new BufferedReader(new InputStreamReader(System.in))) {
return readPassword(alias, br);
- } finally {
- if (br != null) {
- br.close();
- }
}
}
@@ -1049,12 +994,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
Path fileInFs = new Path(pkgPath, keytabInfo.keytab );
log.info("Deleting keytab {}", fileInFs);
- if (!sliderFileSystem.getFileSystem().exists(fileInFs)) {
- throw new BadCommandArgumentsException("No keytab to delete found at " +
- fileInFs.toUri().toString());
- }
-
- sliderFileSystem.getFileSystem().delete(fileInFs, false);
+ FileSystem sfs = sliderFileSystem.getFileSystem();
+ require(sfs.exists(fileInFs), "No keytab to delete found at %s", fileInFs.toUri());
+ sfs.delete(fileInFs, false);
return EXIT_SUCCESS;
}
@@ -1062,40 +1004,31 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private int actionInstallKeytab(ActionKeytabArgs keytabInfo)
throws BadCommandArgumentsException, IOException {
Path srcFile = null;
- if (StringUtils.isEmpty(keytabInfo.folder)) {
- throw new BadCommandArgumentsException(
- "A valid destination keytab sub-folder name is required (e.g. 'security').\n"
- + CommonArgs.usage(serviceArgs, ACTION_KEYTAB));
- }
+ require(isSet(keytabInfo.folder),
+ "A valid destination keytab sub-folder name is required (e.g. 'security').\n"
+ + CommonArgs.usage(serviceArgs, ACTION_KEYTAB));
- if (StringUtils.isEmpty(keytabInfo.keytab)) {
- throw new BadCommandArgumentsException("A valid local keytab location is required.");
- } else {
- File keytabFile = new File(keytabInfo.keytab);
- if (!keytabFile.exists() || keytabFile.isDirectory()) {
- throw new BadCommandArgumentsException("Unable to access supplied keytab file at " +
- keytabFile.getAbsolutePath());
- } else {
- srcFile = new Path(keytabFile.toURI());
- }
- }
+ requireArgumentSet(Arguments.ARG_KEYTAB, keytabInfo.keytab);
+ File keytabFile = new File(keytabInfo.keytab);
+ require(keytabFile.isFile(),
+ "Unable to access supplied keytab file at %s", keytabFile.getAbsolutePath());
+ srcFile = new Path(keytabFile.toURI());
Path pkgPath = sliderFileSystem.buildKeytabInstallationDirPath(keytabInfo.folder);
- sliderFileSystem.getFileSystem().mkdirs(pkgPath);
- sliderFileSystem.getFileSystem().setPermission(pkgPath, new FsPermission(
+ FileSystem sfs = sliderFileSystem.getFileSystem();
+ sfs.mkdirs(pkgPath);
+ sfs.setPermission(pkgPath, new FsPermission(
FsAction.ALL, FsAction.NONE, FsAction.NONE));
Path fileInFs = new Path(pkgPath, srcFile.getName());
- log.info("Installing keytab {} at {} and overwrite is {}.", srcFile, fileInFs, keytabInfo.overwrite);
- if (sliderFileSystem.getFileSystem().exists(fileInFs) && !keytabInfo.overwrite) {
- throw new BadCommandArgumentsException("Keytab exists at " +
- fileInFs.toUri().toString() +
- ". Use --overwrite to overwrite.");
- }
+ log.info("Installing keytab {} at {} and overwrite is {}.",
+ srcFile, fileInFs, keytabInfo.overwrite);
+ require(!(sfs.exists(fileInFs) && !keytabInfo.overwrite),
+ "Keytab exists at %s. Use --overwrite to overwrite.", fileInFs.toUri());
- sliderFileSystem.getFileSystem().copyFromLocalFile(false, keytabInfo.overwrite, srcFile, fileInFs);
- sliderFileSystem.getFileSystem().setPermission(fileInFs, new FsPermission(
- FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE));
+ sfs.copyFromLocalFile(false, keytabInfo.overwrite, srcFile, fileInFs);
+ sfs.setPermission(fileInFs,
+ new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE));
return EXIT_SUCCESS;
}
@@ -1111,44 +1044,29 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
public int actionInstallPkg(ActionInstallPackageArgs installPkgInfo) throws
YarnException,
IOException {
- log.warn("The " + SliderActions.ACTION_INSTALL_PACKAGE
+ log.warn("The " + ACTION_INSTALL_PACKAGE
+ " option has been deprecated. Please use '"
- + SliderActions.ACTION_PACKAGE + " " + ClientArgs.ARG_INSTALL + "'.");
- Path srcFile = null;
+ + ACTION_PACKAGE + " " + ClientArgs.ARG_INSTALL + "'.");
if (StringUtils.isEmpty(installPkgInfo.name)) {
throw new BadCommandArgumentsException(
- E_INVALID_APPLICATION_TYPE_NAME +"\n"
+ E_INVALID_APPLICATION_TYPE_NAME + "\n"
+ CommonArgs.usage(serviceArgs, ACTION_INSTALL_PACKAGE));
}
-
- if (StringUtils.isEmpty(installPkgInfo.packageURI)) {
- throw new BadCommandArgumentsException(E_INVALID_APPLICATION_PACKAGE_LOCATION);
- } else {
- File pkgFile = new File(installPkgInfo.packageURI);
- if (!pkgFile.exists() || pkgFile.isDirectory()) {
- throw new BadCommandArgumentsException(
- E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE +": "
- + pkgFile.getAbsolutePath());
- } else {
- srcFile = new Path(pkgFile.toURI());
- }
- }
+ Path srcFile = extractPackagePath(installPkgInfo.packageURI);
// Do not provide new options to install-package command as it is in
// deprecated mode. So version is kept null here. Use package --install.
Path pkgPath = sliderFileSystem.buildPackageDirPath(installPkgInfo.name,
null);
- sliderFileSystem.getFileSystem().mkdirs(pkgPath);
+ FileSystem sfs = sliderFileSystem.getFileSystem();
+ sfs.mkdirs(pkgPath);
Path fileInFs = new Path(pkgPath, srcFile.getName());
- log.info("Installing package {} at {} and overwrite is {}.", srcFile, fileInFs, installPkgInfo.replacePkg);
- if (sliderFileSystem.getFileSystem().exists(fileInFs) && !installPkgInfo.replacePkg) {
- throw new BadCommandArgumentsException(
- "Package exists at " + fileInFs.toUri().toString() +"."
- + E_USE_REPLACEPKG_TO_OVERWRITE);
- }
-
- sliderFileSystem.getFileSystem().copyFromLocalFile(false, installPkgInfo.replacePkg, srcFile, fileInFs);
+ log.info("Installing package {} at {} and overwrite is {}.",
+ srcFile, fileInFs, installPkgInfo.replacePkg);
+ require(!(sfs.exists(fileInFs) && !installPkgInfo.replacePkg),
+ "Package exists at %s. : %s", fileInFs.toUri(), E_USE_REPLACEPKG_TO_OVERWRITE);
+ sfs.copyFromLocalFile(false, installPkgInfo.replacePkg, srcFile, fileInFs);
return EXIT_SUCCESS;
}
@@ -1177,11 +1095,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
+ CommonArgs.usage(serviceArgs, ACTION_CLIENT));
}
- if (clientInfo.name == null) {
- throw new BadCommandArgumentsException("No application name specified\n"
- + CommonArgs.usage(serviceArgs,
- ACTION_CLIENT));
- }
+ requireArgumentSet(Arguments.ARG_NAME, clientInfo.name);
File storeFile = null;
SecurityStore.StoreType type;
@@ -1193,11 +1107,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
type = SecurityStore.StoreType.truststore;
}
- if (storeFile.exists()) {
- throw new BadCommandArgumentsException("File %s already exists. "
- + "Please remove that file or select a different file name.",
- storeFile.getAbsolutePath());
- }
+ require (!storeFile.exists(),
+ "File %s already exists. Please remove that file or select a different file name.",
+ storeFile.getAbsolutePath());
String hostname = null;
if (type == SecurityStore.StoreType.keystore) {
hostname = clientInfo.hostname;
@@ -1232,10 +1144,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
}
- byte[]
- keystore =
- createClusterOperations(clientInfo.name).getClientCertificateStore(
- hostname, "client", password, type.name());
+ byte[] keystore = createClusterOperations(clientInfo.name)
+ .getClientCertificateStore(hostname, "client", password, type.name());
// persist to file
IOUtils.write(keystore, new FileOutputStream(storeFile));
@@ -1245,31 +1155,20 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private int doClientInstall(ActionClientArgs clientInfo)
throws IOException, SliderException {
- if (clientInfo.installLocation == null) {
- throw new BadCommandArgumentsException(
+ require(clientInfo.installLocation != null,
E_INVALID_INSTALL_LOCATION +"\n"
+ CommonArgs.usage(serviceArgs, ACTION_CLIENT));
- } else {
- if (!clientInfo.installLocation.exists()) {
- throw new BadCommandArgumentsException(E_INSTALL_PATH_DOES_NOT_EXIST
- +": " + clientInfo.installLocation.getAbsolutePath());
- }
- if (!clientInfo.installLocation.isDirectory()) {
- throw new BadCommandArgumentsException(E_INVALID_INSTALL_PATH
- +": " + clientInfo.installLocation.getAbsolutePath());
- }
- }
+ require(clientInfo.installLocation.exists(),
+ E_INSTALL_PATH_DOES_NOT_EXIST + ": " + clientInfo.installLocation.getAbsolutePath());
+
+ require(clientInfo.installLocation.isFile(),
+ E_INVALID_INSTALL_PATH + ": " + clientInfo.installLocation.getAbsolutePath());
File pkgFile;
- if (StringUtils.isEmpty(clientInfo.packageURI)) {
- throw new BadCommandArgumentsException(E_INVALID_APPLICATION_PACKAGE_LOCATION);
- } else {
- pkgFile = new File(clientInfo.packageURI);
- if (!pkgFile.exists() || pkgFile.isDirectory()) {
- throw new BadCommandArgumentsException(E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE
- +" at " + pkgFile.getAbsolutePath());
- }
- }
+ requireArgumentSet(Arguments.ARG_PACKAGE, clientInfo.packageURI);
+ pkgFile = new File(clientInfo.packageURI);
+ require(pkgFile.isFile(),
+ E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE + " at %s", pkgFile.getAbsolutePath());
JSONObject config = null;
if(clientInfo.clientConfig != null) {
@@ -1368,7 +1267,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
Path appDefPath = null;
try {
appDefPath = new Path(
- SliderUtils.getApplicationDefinitionPath(instanceDefinition
+ getApplicationDefinitionPath(instanceDefinition
.getAppConfOperations()));
} catch (BadConfigException e) {
// Invalid cluster state, so move on to next. No need to log anything
@@ -1390,11 +1289,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
println("%-25s %15s %30s %s", clusterName, packageName,
packageVersion, appDefPathStr);
}
- } catch(IOException e) {
- if(log.isDebugEnabled()) {
- log.debug(clusterName + " application definition path "
- + appDefPathStr + " is not found.");
- }
+ } catch (IOException e) {
+ log.debug("{} application definition path {} is not found.", clusterName, appDefPathStr);
}
}
return EXIT_SUCCESS;
@@ -1403,19 +1299,19 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private int actionPackageList() throws IOException {
Path pkgPath = sliderFileSystem.buildPackageDirPath(StringUtils.EMPTY,
StringUtils.EMPTY);
- log.info("Package install path : " + pkgPath);
- if (!sliderFileSystem.getFileSystem().isDirectory(pkgPath)) {
+ log.info("Package install path : {}", pkgPath);
+ FileSystem sfs = sliderFileSystem.getFileSystem();
+ if (!sfs.isDirectory(pkgPath)) {
log.info("No package(s) installed");
return EXIT_SUCCESS;
}
- FileStatus[] fileStatus = sliderFileSystem.getFileSystem().listStatus(
- pkgPath);
+ FileStatus[] fileStatus = sfs.listStatus(pkgPath);
boolean hasPackage = false;
StringBuilder sb = new StringBuilder();
sb.append("List of installed packages:\n");
for (FileStatus fstat : fileStatus) {
if (fstat.isDirectory()) {
- sb.append("\t" + fstat.getPath().getName());
+ sb.append("\t").append(fstat.getPath().getName());
sb.append("\n");
hasPackage = true;
}
@@ -1428,53 +1324,31 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
- private int actionPackageInstall(ActionPackageArgs actionPackageArgs) throws
- YarnException,
- IOException {
-
- Path srcFile = null;
- if (StringUtils.isEmpty(actionPackageArgs.name)) {
- throw new BadCommandArgumentsException(
- "A valid application type name is required (e.g. HBASE).\n"
- + CommonArgs.usage(serviceArgs, ACTION_PACKAGE));
- }
+ private int actionPackageInstall(ActionPackageArgs actionPackageArgs)
+ throws YarnException, IOException {
+ requireArgumentSet(Arguments.ARG_NAME, actionPackageArgs.name);
- if (StringUtils.isEmpty(actionPackageArgs.packageURI)) {
- throw new BadCommandArgumentsException(
- E_INVALID_APPLICATION_PACKAGE_LOCATION);
- } else {
- File pkgFile = new File(actionPackageArgs.packageURI);
- if (!pkgFile.exists() || pkgFile.isDirectory()) {
- throw new BadCommandArgumentsException(
- E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE
- + ": " + pkgFile.getAbsolutePath());
- } else {
- srcFile = new Path(pkgFile.toURI());
- }
- }
+ Path srcFile = extractPackagePath(actionPackageArgs.packageURI);
Path pkgPath = sliderFileSystem.buildPackageDirPath(actionPackageArgs.name,
actionPackageArgs.version);
- if (!sliderFileSystem.getFileSystem().exists(pkgPath)) {
- sliderFileSystem.getFileSystem().mkdirs(pkgPath);
+ FileSystem fs = sliderFileSystem.getFileSystem();
+ if (!fs.exists(pkgPath)) {
+ fs.mkdirs(pkgPath);
}
Path fileInFs = new Path(pkgPath, srcFile.getName());
- if (sliderFileSystem.getFileSystem().exists(fileInFs)
- && !actionPackageArgs.replacePkg) {
- throw new BadCommandArgumentsException(E_PACKAGE_EXISTS +" at " +
- fileInFs.toUri() + ". Use --replacepkg to overwrite.");
- }
+ require(actionPackageArgs.replacePkg || !fs.exists(fileInFs),
+ E_PACKAGE_EXISTS +" at %s. Use --replacepkg to overwrite.", fileInFs.toUri());
log.info("Installing package {} to {} (overwrite set to {})", srcFile,
fileInFs, actionPackageArgs.replacePkg);
- sliderFileSystem.getFileSystem().copyFromLocalFile(false,
- actionPackageArgs.replacePkg, srcFile, fileInFs);
+ fs.copyFromLocalFile(false, actionPackageArgs.replacePkg, srcFile, fileInFs);
String destPathWithHomeDir = Path
.getPathWithoutSchemeAndAuthority(fileInFs).toString();
String destHomeDir = Path.getPathWithoutSchemeAndAuthority(
- sliderFileSystem.getFileSystem().getHomeDirectory()).toString();
+ fs.getHomeDirectory()).toString();
// a somewhat contrived approach to stripping out the home directory and any trailing
// separator; designed to work on windows and unix
String destPathWithoutHomeDir;
@@ -1492,23 +1366,26 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
+ private Path extractPackagePath(String packageURI)
+ throws BadCommandArgumentsException {
+ require(isSet(packageURI), E_INVALID_APPLICATION_PACKAGE_LOCATION);
+ File pkgFile = new File(packageURI);
+ require(pkgFile.isFile(),
+ E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE + ": " + pkgFile.getAbsolutePath());
+ return new Path(pkgFile.toURI());
+ }
+
private int actionPackageDelete(ActionPackageArgs actionPackageArgs) throws
YarnException, IOException {
- if (StringUtils.isEmpty(actionPackageArgs.name)) {
- throw new BadCommandArgumentsException(
- "A valid application type name is required (e.g. HBASE).\n"
- + CommonArgs.usage(serviceArgs, ACTION_PACKAGE));
- }
+ requireArgumentSet(Arguments.ARG_NAME, actionPackageArgs.name);
Path pkgPath = sliderFileSystem.buildPackageDirPath(actionPackageArgs.name,
actionPackageArgs.version);
- if (!sliderFileSystem.getFileSystem().exists(pkgPath)) {
- throw new BadCommandArgumentsException(E_PACKAGE_DOES_NOT_EXIST +": "
- + pkgPath.toUri().toString());
- }
+ FileSystem fs = sliderFileSystem.getFileSystem();
+ require(fs.exists(pkgPath), E_PACKAGE_DOES_NOT_EXIST +": %s ", pkgPath.toUri());
log.info("Deleting package {} at {}.", actionPackageArgs.name, pkgPath);
- if(sliderFileSystem.getFileSystem().delete(pkgPath, true)) {
+ if(fs.delete(pkgPath, true)) {
log.info("Deleted package {} " + actionPackageArgs.name);
return EXIT_SUCCESS;
} else {
@@ -1548,7 +1425,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
boolean liveClusterAllowed, boolean isUpgradeFlow) throws YarnException,
IOException {
// verify that a live cluster isn't there
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
verifyBindingsDefined();
if (!liveClusterAllowed) {
verifyNoLiveClusters(clustername, "Create");
@@ -1627,8 +1504,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
String count = roleEntry.getValue();
String key = roleEntry.getKey();
log.info("{} => {}", key, count);
- resources.getOrAddComponent(key)
- .put(ResourceKeys.COMPONENT_INSTANCES, count);
+ resources.getOrAddComponent(key).put(COMPONENT_INSTANCES, count);
}
//all CLI role options
@@ -1663,7 +1539,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
builder.setQueue(buildInfo.queue);
String quorum = buildInfo.getZKhosts();
- if (SliderUtils.isUnset(quorum)) {
+ if (isUnset(quorum)) {
quorum = registryQuorum;
}
if (isUnset(quorum)) {
@@ -1735,7 +1611,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
if (!SliderKeys.COMPONENT_AM.equals(componentName)) {
clientComponentInstances.put(componentName, clientResources
.getComponentOptInt(componentName,
- ResourceKeys.COMPONENT_INSTANCES, -1));
+ COMPONENT_INSTANCES, -1));
}
}
@@ -1745,8 +1621,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
} catch (LockAcquireFailedException e) {
log.warn("Failed to get a Lock on cluster resource : {}", e, e);
throw new BadClusterStateException(
- "Failed to load client resource definition " + clustername + ": "
- + e);
+ "Failed to load client resource definition " + clustername + ": " + e, e);
}
Map<String, Integer> clusterComponentInstances = new HashMap<>();
for (Map.Entry<String, Map<String, String>> component : clusterConf
@@ -1755,7 +1630,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
clusterComponentInstances.put(
component.getKey(),
Integer.decode(component.getValue().get(
- ResourceKeys.COMPONENT_INSTANCES)));
+ COMPONENT_INSTANCES)));
}
}
@@ -1763,16 +1638,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
Iterator<Map.Entry<String, Integer>> clientComponentInstanceIt = clientComponentInstances
.entrySet().iterator();
while (clientComponentInstanceIt.hasNext()) {
- Map.Entry<String, Integer> clientComponentInstanceEntry = clientComponentInstanceIt
- .next();
- if (clusterComponentInstances
- .containsKey(clientComponentInstanceEntry.getKey())) {
+ Map.Entry<String, Integer> clientComponentInstanceEntry = clientComponentInstanceIt.next();
+ if (clusterComponentInstances.containsKey(clientComponentInstanceEntry.getKey())) {
// compare instance count now and remove from both maps if they match
if (clusterComponentInstances
.get(clientComponentInstanceEntry.getKey()) == clientComponentInstanceEntry
.getValue()) {
- clusterComponentInstances
- .remove(clientComponentInstanceEntry.getKey());
+ clusterComponentInstances.remove(clientComponentInstanceEntry.getKey());
clientComponentInstanceIt.remove();
}
}
@@ -1846,8 +1718,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
public FsPermission getClusterDirectoryPermissions(Configuration conf) {
String clusterDirPermsOct =
- conf.get(CLUSTER_DIRECTORY_PERMISSIONS,
- DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS);
+ conf.get(CLUSTER_DIRECTORY_PERMISSIONS, DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS);
return new FsPermission(clusterDirPermsOct);
}
@@ -1857,9 +1728,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws BadCommandArgumentsException the exception raised on an invalid config
*/
public void verifyBindingsDefined() throws BadCommandArgumentsException {
- InetSocketAddress rmAddr = SliderUtils.getRmAddress(getConfig());
+ InetSocketAddress rmAddr = getRmAddress(getConfig());
if (!getConfig().getBoolean(YarnConfiguration.RM_HA_ENABLED, false)
- && !SliderUtils.isAddressDefined(rmAddr)) {
+ && !isAddressDefined(rmAddr)) {
throw new BadCommandArgumentsException(
E_NO_RESOURCE_MANAGER
+ " in the argument "
@@ -1923,9 +1794,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws UnknownApplicationInstanceException if the file is not found
*/
public AggregateConf loadInstanceDefinitionUnresolved(String name,
- Path clusterDirectory) throws
- IOException,
- SliderException {
+ Path clusterDirectory) throws IOException, SliderException {
try {
AggregateConf definition =
@@ -1937,7 +1806,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
throw UnknownApplicationInstanceException.unknownInstance(name, e);
}
}
- /**
+
+ /**
* Load the instance definition.
* @param name cluster name
* @param resolved flag to indicate the cluster should be resolved
@@ -1980,11 +1850,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
deployedClusterName = clustername;
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
verifyNoLiveClusters(clustername, "Launch");
Configuration config = getConfig();
lookupZKQuorum();
- boolean clusterSecure = SliderUtils.isHadoopClusterSecure(config);
+ boolean clusterSecure = isHadoopClusterSecure(config);
//create the Slider AM provider -this helps set up the AM
SliderAMClientProvider sliderAM = new SliderAMClientProvider(config);
@@ -1997,18 +1867,15 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
ConfTreeOperations appOperations = instanceDefinition.getAppConfOperations();
Path generatedConfDirPath =
createPathThatMustExist(internalOptions.getMandatoryOption(
- InternalKeys.INTERNAL_GENERATED_CONF_PATH));
+ INTERNAL_GENERATED_CONF_PATH));
Path snapshotConfPath =
createPathThatMustExist(internalOptions.getMandatoryOption(
- InternalKeys.INTERNAL_SNAPSHOT_CONF_PATH));
+ INTERNAL_SNAPSHOT_CONF_PATH));
// cluster Provider
AbstractClientProvider provider = createClientProvider(
- internalOptions.getMandatoryOption(
- InternalKeys.INTERNAL_PROVIDER_NAME));
- // make sure the conf dir is valid;
-
+ internalOptions.getMandatoryOption(INTERNAL_PROVIDER_NAME));
if (log.isDebugEnabled()) {
log.debug(instanceDefinition.toString());
}
@@ -2018,7 +1885,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// add the tags if available
Set<String> applicationTags = provider.getApplicationTags(sliderFileSystem,
- SliderUtils.getApplicationDefinitionPath(appOperations));
+ getApplicationDefinitionPath(appOperations));
AppMasterLauncher amLauncher = new AppMasterLauncher(clustername,
SliderKeys.APP_TYPE,
config,
@@ -2043,8 +1910,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
String libdir = "lib";
Path libPath = new Path(tempPath, libdir);
sliderFileSystem.getFileSystem().mkdirs(libPath);
- log.debug("FS={}, tempPath={}, libdir={}",
- sliderFileSystem, tempPath, libPath);
+ log.debug("FS={}, tempPath={}, libdir={}", sliderFileSystem, tempPath, libPath);
// set local resources for the application master
// local files or archives as needed
@@ -2055,9 +1921,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
boolean hasServerLog4jProperties = false;
Path remoteConfPath = null;
String relativeConfDir = null;
- String confdirProp =
- System.getProperty(SliderKeys.PROPERTY_CONF_DIR);
- if (confdirProp == null || confdirProp.isEmpty()) {
+ String confdirProp = System.getProperty(SliderKeys.PROPERTY_CONF_DIR);
+ if (isUnset(confdirProp)) {
log.debug("No local configuration directory provided as system property");
} else {
File confDir = new File(confdirProp);
@@ -2065,11 +1930,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
throw new BadConfigException(E_CONFIGURATION_DIRECTORY_NOT_FOUND,
confDir);
}
- Path localConfDirPath = SliderUtils.createLocalPath(confDir);
+ Path localConfDirPath = createLocalPath(confDir);
remoteConfPath = new Path(clusterDirectory, SliderKeys.SUBMITTED_CONF_DIR);
log.debug("Slider configuration directory is {}; remote to be {}",
localConfDirPath, remoteConfPath);
- SliderUtils.copyDirectory(config, localConfDirPath, remoteConfPath, null);
+ copyDirectory(config, localConfDirPath, remoteConfPath, null);
File log4jserver =
new File(confDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME);
@@ -2089,7 +1954,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
Map<String, LocalResource> submittedConfDir =
sliderFileSystem.submitDirectory(remoteConfPath,
relativeConfDir);
- SliderUtils.mergeMaps(localResources, submittedConfDir);
+ mergeMaps(localResources, submittedConfDir);
}
}
// build up the configuration
@@ -2109,7 +1974,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
Configuration clientConfExtras = new Configuration(false);
// then build up the generated path.
FsPermission clusterPerms = getClusterDirectoryPermissions(config);
- SliderUtils.copyDirectory(config, snapshotConfPath, generatedConfDirPath,
+ copyDirectory(config, snapshotConfPath, generatedConfDirPath,
clusterPerms);
@@ -2162,15 +2027,15 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// TODO: consider supporting apps that don't have an image path
Path imagePath =
- SliderUtils.extractImagePath(sliderFileSystem, internalOptions);
+ extractImagePath(sliderFileSystem, internalOptions);
if (sliderFileSystem.maybeAddImagePath(localResources, imagePath)) {
log.debug("Registered image path {}", imagePath);
}
// build the environment
amLauncher.putEnv(
- SliderUtils.buildEnvMap(sliderAMResourceComponent));
- ClasspathConstructor classpath = SliderUtils.buildClasspath(relativeConfDir,
+ buildEnvMap(sliderAMResourceComponent));
+ ClasspathConstructor classpath = buildClasspath(relativeConfDir,
libdir,
getConfig(),
sliderFileSystem,
@@ -2182,13 +2047,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
amLauncher.setEnv("LANGUAGE", "en_US.UTF-8");
amLauncher.putEnv(getAmLaunchEnv(config));
- for (Map.Entry<String, String> envs : SliderUtils.getSystemEnv().entrySet()) {
+ for (Map.Entry<String, String> envs : getSystemEnv().entrySet()) {
log.debug("System env {}={}", envs.getKey(), envs.getValue());
}
if (log.isDebugEnabled()) {
log.debug("AM classpath={}", classpath);
log.debug("Environment Map:\n{}",
- SliderUtils.stringifyMap(amLauncher.getEnv()));
+ stringifyMap(amLauncher.getEnv()));
log.debug("Files in lib path\n{}", sliderFileSystem.listFSDir(libPath));
}
@@ -2196,14 +2061,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
InetSocketAddress rmSchedulerAddress;
try {
- rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(config);
+ rmSchedulerAddress = getRmSchedulerAddress(config);
} catch (IllegalArgumentException e) {
throw new BadConfigException("%s Address invalid: %s",
- YarnConfiguration.RM_SCHEDULER_ADDRESS,
- config.get(
- YarnConfiguration.RM_SCHEDULER_ADDRESS)
- );
-
+ YarnConfiguration.RM_SCHEDULER_ADDRESS,
+ config.get(YarnConfiguration.RM_SCHEDULER_ADDRESS));
}
String rmAddr = NetUtils.getHostPortString(rmSchedulerAddress);
@@ -2244,18 +2106,14 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
/**
* pass the registry binding
*/
- addConfOptionToCLI(commandLine, config,
- RegistryConstants.KEY_REGISTRY_ZK_ROOT,
+ commandLine.addConfOptionToCLI(config, RegistryConstants.KEY_REGISTRY_ZK_ROOT,
RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
- addMandatoryConfOptionToCLI(commandLine, config,
- RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
+ commandLine.addMandatoryConfOption(config, RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
if (clusterSecure) {
// if the cluster is secure, make sure that
// the relevant security settings go over
- addConfOptionToCLI(commandLine,
- config,
- DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
+ commandLine.addConfOption(config, DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
}
// write out the path output
commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM);
@@ -2281,8 +2139,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// Set the queue to which this application is to be submitted in the RM
// Queue for App master
String amQueue = config.get(KEY_YARN_QUEUE, DEFAULT_YARN_QUEUE);
- String suppliedQueue = internalOperations.getGlobalOptions().get(InternalKeys.INTERNAL_QUEUE);
- if(!SliderUtils.isUnset(suppliedQueue)) {
+ String suppliedQueue = internalOperations.getGlobalOptions().get(INTERNAL_QUEUE);
+ if(!isUnset(suppliedQueue)) {
amQueue = suppliedQueue;
log.info("Using queue {} for the application instance.", amQueue);
}
@@ -2297,8 +2155,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
protected Map<String, String> getAmLaunchEnv(Configuration config) {
- String sliderAmLaunchEnv = config.get(SliderXmlConfKeys.KEY_AM_LAUNCH_ENV);
- log.debug("{} = {}", SliderXmlConfKeys.KEY_AM_LAUNCH_ENV, sliderAmLaunchEnv);
+ String sliderAmLaunchEnv = config.get(KEY_AM_LAUNCH_ENV);
+ log.debug("{} = {}", KEY_AM_LAUNCH_ENV, sliderAmLaunchEnv);
// Multiple env variables can be specified with a comma (,) separator
String[] envs = StringUtils.isEmpty(sliderAmLaunchEnv) ? null
: sliderAmLaunchEnv.split(",");
@@ -2343,7 +2201,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
String systemKey = placeholderKey
.substring(2, placeholderKey.length() - 1).toUpperCase(Locale.ENGLISH)
.replaceAll("\\.", "_");
- String placeholderValue = SliderUtils.getSystemEnv(systemKey);
+ String placeholderValue = getSystemEnv(systemKey);
log.debug("Placeholder {}={}", placeholderKey, placeholderValue);
placeholderKeyValueMap.put(placeholderKey, placeholderValue);
}
@@ -2353,10 +2211,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private void propagatePythonExecutable(Configuration config,
AggregateConf instanceDefinition) {
String pythonExec = config.get(
- SliderXmlConfKeys.PYTHON_EXECUTABLE_PATH);
+ PYTHON_EXECUTABLE_PATH);
if (pythonExec != null) {
instanceDefinition.getAppConfOperations().getGlobalOptions().putIfUnset(
- SliderXmlConfKeys.PYTHON_EXECUTABLE_PATH,
+ PYTHON_EXECUTABLE_PATH,
pythonExec);
}
}
@@ -2386,7 +2244,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
new Duration(acceptWaitMillis));
// may have failed, so check that
- if (SliderUtils.hasAppFinished(report)) {
+ if (hasAppFinished(report)) {
exitCode = buildExitCode(report);
} else {
// exit unless there is a wait
@@ -2421,68 +2279,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
AggregateConf clusterSpec) {
String dfsPrincipal = config.get(DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
if (dfsPrincipal != null) {
- String siteDfsPrincipal = OptionKeys.SITE_XML_PREFIX +
- DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
+ String siteDfsPrincipal = SITE_XML_PREFIX + DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
clusterSpec.getAppConfOperations().getGlobalOptions().putIfUnset(
siteDfsPrincipal,
dfsPrincipal);
}
}
-
- private boolean addConfOptionToCLI(CommandLineBuilder cmdLine,
- Configuration conf,
- String key) {
- String val = conf.get(key);
- return defineIfSet(cmdLine, key, val);
- }
-
- private String addConfOptionToCLI(CommandLineBuilder cmdLine,
- Configuration conf,
- String key,
- String defVal) {
- String val = conf.get(key, defVal);
- define(cmdLine, key, val);
- return val;
- }
-
- /**
- * Add a <code>-D key=val</code> command to the CLI
- * @param cmdLine command line
- * @param key key
- * @param val value
- */
- private void define(CommandLineBuilder cmdLine, String key, String val) {
- Preconditions.checkArgument(key != null, "null key");
- Preconditions.checkArgument(val != null, "null value");
- cmdLine.add(Arguments.ARG_DEFINE, key + "=" + val);
- }
-
- /**
- * Add a <code>-D key=val</code> command to the CLI if <code>val</code>
- * is not null
- * @param cmdLine command line
- * @param key key
- * @param val value
- */
- private boolean defineIfSet(CommandLineBuilder cmdLine, String key, String val) {
- Preconditions.checkArgument(key != null, "null key");
- if (val != null) {
- define(cmdLine, key, val);
- return true;
- } else {
- return false;
- }
- }
-
- private void addMandatoryConfOptionToCLI(CommandLineBuilder cmdLine,
- Configuration conf,
- String key) throws BadConfigException {
- if (!addConfOptionToCLI(cmdLine, conf, key)) {
- throw new BadConfigException("Missing configuration option: " + key);
- }
- }
-
/**
* Create a path that must exist in the cluster fs
* @param uri uri to create
@@ -2781,9 +2584,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// and those the RM knows about
List<ApplicationReport> instances = listSliderInstances(null);
- SliderUtils.sortApplicationsByMostRecent(instances);
+ sortApplicationsByMostRecent(instances);
Map<String, ApplicationReport> reportMap =
- SliderUtils.buildApplicationReportMap(instances, min, max);
+ buildApplicationReportMap(instances, min, max);
log.debug("Persisted {} deployed {} filtered[{}-{}] & de-duped to {}",
persistentInstances.size(),
instances.size(),
@@ -2816,7 +2619,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// a report
listedInstances.add(name);
// containers will be non-null when only one instance is requested
- String details = SliderUtils.instanceDetailsToString(name, report,
+ String details = instanceDetailsToString(name, report,
containers, version, components, verbose);
print(details);
}
@@ -2899,7 +2702,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
@VisibleForTesting
public int actionFlex(String name, ActionFlexArgs args) throws YarnException, IOException {
verifyBindingsDefined();
- SliderUtils.validateClusterName(name);
+ validateClusterName(name);
log.debug("actionFlex({})", name);
Map<String, Integer> roleInstances = new HashMap<>();
Map<String, String> roleMap = args.getComponentMap();
@@ -2926,7 +2729,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
public int actionExists(String name, ActionExistsArgs args) throws YarnException, IOException {
verifyBindingsDefined();
- SliderUtils.validateClusterName(name);
+ validateClusterName(name);
boolean checkLive = args.live;
log.debug("actionExists({}, {}, {})", name, checkLive, args.state);
@@ -2936,7 +2739,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
throw unknownClusterException(name);
}
String state = args.state;
- if (!checkLive && SliderUtils.isUnset(state)) {
+ if (!checkLive && isUnset(state)) {
log.info("Application {} exists", name);
return EXIT_SUCCESS;
}
@@ -2970,8 +2773,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
}
- SliderUtils.OnDemandReportStringifier report =
- new SliderUtils.OnDemandReportStringifier(instance);
+ OnDemandReportStringifier report =
+ new OnDemandReportStringifier(instance);
if (!inDesiredState) {
//cluster in the list of apps but not running
log.info("Application {} found but is in wrong state {}", name,
@@ -2991,7 +2794,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
public int actionKillContainer(String name,
ActionKillContainerArgs args) throws YarnException, IOException {
String id = args.id;
- if (SliderUtils.isUnset(id)) {
+ if (isUnset(id)) {
throw new BadCommandArgumentsException("Missing container id");
}
log.info("killingContainer {}:{}", name, id);
@@ -3087,7 +2890,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
YarnException,
IOException {
verifyBindingsDefined();
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
String outfile = statusArgs.getOutput();
ClusterDescription status = getClusterDescription(clustername);
String text = status.toJsonString();
@@ -3109,7 +2912,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
public int actionFreeze(String clustername,
ActionFreezeArgs freezeArgs) throws YarnException, IOException {
verifyBindingsDefined();
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
int waittime = freezeArgs.getWaittime();
String text = freezeArgs.message;
boolean forcekill = freezeArgs.force;
@@ -3128,7 +2931,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
log.debug("App to stop was found: {}:\n{}", clustername,
- new SliderUtils.OnDemandReportStringifier(app));
+ new OnDemandReportStringifier(app));
if (app.getYarnApplicationState().ordinal() >=
YarnApplicationState.FINISHED.ordinal()) {
log.info("Cluster {} is in a terminated state {}", clustername,
@@ -3195,7 +2998,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
@Override
public int actionThaw(String clustername, ActionThawArgs thaw) throws YarnException, IOException {
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
verifyBindingsDefined();
// see if it is actually running and bail out;
verifyNoLiveClusters(clustername, "Start");
@@ -3215,7 +3018,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
public int flex(String clustername, Map<String, Integer> roleInstances)
throws YarnException, IOException {
verifyBindingsDefined();
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(
clustername,
@@ -3226,7 +3029,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
for (Map.Entry<String, Integer> entry : roleInstances.entrySet()) {
String role = entry.getKey();
int count = entry.getValue();
- resources.getOrAddComponent(role).put(ResourceKeys.COMPONENT_INSTANCES,
+ resources.getOrAddComponent(role).put(COMPONENT_INSTANCES,
Integer.toString(count));
log.debug("Flexed cluster specification ( {} -> {}) : \n{}",
@@ -3237,7 +3040,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
SliderAMClientProvider sliderAM = new SliderAMClientProvider(getConfig());
AbstractClientProvider provider = createClientProvider(
instanceDefinition.getInternalOperations().getGlobalOptions().getMandatoryOption(
- InternalKeys.INTERNAL_PROVIDER_NAME));
+ INTERNAL_PROVIDER_NAME));
// slider provider to validate what there is
validateInstanceDefinition(sliderAM, instanceDefinition, sliderFileSystem);
validateInstanceDefinition(provider, instanceDefinition, sliderFileSystem);
@@ -3548,7 +3351,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
} else {
String filename = RegistryPathUtils.lastPathEntry(name) + ".json";
File jsonFile = new File(destDir, filename);
- SliderUtils.write(jsonFile,
+ write(jsonFile,
serviceRecordMarshal.toBytes(instance),
true);
}
@@ -3561,7 +3364,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
outFile = new File(args.destdir, RegistryPathUtils.lastPathEntry(path));
}
if (outFile != null) {
- SliderUtils.write(outFile, serviceRecordMarshal.toBytes(instance), true);
+ write(outFile, serviceRecordMarshal.toBytes(instance), true);
} else {
println(serviceRecordMarshal.toJson(instance));
}
@@ -3592,12 +3395,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
} else if (registryArgs.listExports) {
// list the exports
actionRegistryListExports(registryArgs);
- } else if (SliderUtils.isSet(registryArgs.getConf)) {
+ } else if (isSet(registryArgs.getConf)) {
// get a configuration
PublishedConfiguration publishedConfiguration =
actionRegistryGetConfig(registryArgs);
outputConfig(publishedConfiguration, registryArgs);
- } else if (SliderUtils.isSet(registryArgs.getExport)) {
+ } else if (isSet(registryArgs.getExport)) {
// get a export group
PublishedExports publishedExports =
actionRegistryGetExport(registryArgs);
@@ -3609,11 +3412,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
// JDK7
} catch (FileNotFoundException e) {
- log.info("{}", e);
+ log.info("{}", e.toString());
log.debug("{}", e, e);
return EXIT_NOT_FOUND;
} catch (PathNotFoundException e) {
- log.info("{}", e);
+ log.info("{}", e.toString());
log.debug("{}", e, e);
return EXIT_NOT_FOUND;
}
@@ -3637,10 +3440,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
RegistryOperations operations = getRegistryOperations();
Collection<ServiceRecord> serviceRecords;
if (StringUtils.isEmpty(name)) {
- String path =
- serviceclassPath(
- currentUser(),
- serviceType);
+ String path = serviceclassPath(currentUser(), serviceType);
try {
Map<String, ServiceRecord> recordMap =
@@ -3698,18 +3498,15 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// application name after --application option and member variable
// cluster name has to be put behind action
String clusterName = diagnosticArgs.name;
- if(SliderUtils.isUnset(clusterName)){
- throw new BadCommandArgumentsException("application name must be provided with --name option");
- }
-
+ requireArgumentSet(Arguments.ARG_NAME, clusterName);
+
try {
- SliderUtils.validateClientConfigFile();
+ validateClientConfigFile();
log.info("Slider-client.xml is accessible");
} catch (IOException e) {
// we are catching exceptions here because those are indication of
// validation result, and we need to print them here
- log.error(
- "validation of slider-client.xml fails because: " + e.toString(), e);
+ log.error("validation of slider-client.xml fails because: " + e, e);
return;
}
SliderClusterOperations clusterOperations = createClusterOperations(clusterName);
@@ -3722,7 +3519,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
AggregateConf instanceDefinition = clusterOperations
.getInstanceDefinition();
String imagePath = instanceDefinition.getInternalOperations().get(
- InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ INTERNAL_APPLICATION_IMAGE_PATH);
// if null, that means slider uploaded the agent tarball for the user
// and we need to use where slider has put
if (imagePath == null) {
@@ -3733,26 +3530,27 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
+ "/agent");
imagePath = subPath.toString();
}
+ String pathStr = imagePath + "/" + AGENT_TAR;
try {
- SliderUtils.validateHDFSFile(sliderFileSystem, imagePath + "/" + AGENT_TAR);
+ validateHDFSFile(sliderFileSystem, pathStr);
log.info("Slider agent package is properly installed");
} catch (FileNotFoundException e) {
- log.error("can not find agent package: " + e.toString());
+ log.error("can not find agent package: {}", pathStr);
+ log.debug("can not find agent package: {}", pathStr, e);
return;
} catch (IOException e) {
- log.error("can not open agent package: " + e.toString());
+ log.error("can not open agent package: {}", pathStr, e);
return;
}
- String pkgTarballPath = SliderUtils
- .getApplicationDefinitionPath(instanceDefinition
+ String pkgTarballPath = getApplicationDefinitionPath(instanceDefinition
.getAppConfOperations());
try {
- SliderUtils.validateHDFSFile(sliderFileSystem, pkgTarballPath);
+ validateHDFSFile(sliderFileSystem, pkgTarballPath);
log.info("Application package is properly installed");
} catch (FileNotFoundException e) {
- log.error("can not find application package: {}", e);
+ log.error("can not find application package: {}", pkgTarballPath, e);
} catch (IOException e) {
- log.error("can not open application package: {} ", e);
+ log.error("can not open application package: {} ", pkgTarballPath, e);
}
}
}
@@ -3769,11 +3567,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private void actionDiagnosticCredentials() throws BadConfigException,
IOException {
- if (SliderUtils.isHadoopClusterSecure(SliderUtils
- .loadSliderClientXML())) {
+ if (isHadoopClusterSecure(loadSliderClientXML())) {
String credentialCacheFileDescription = null;
try {
- credentialCacheFileDescription = SliderUtils.checkCredentialCacheFile();
+ credentialCacheFileDescription = checkCredentialCacheFile();
} catch (BadConfigException e) {
log.error("The credential config is not valid: " + e.toString());
throw e;
@@ -3835,25 +3632,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// application name after --application option and member variable
// cluster name has to be put behind action
String clusterName = diagnosticArgs.name;
- if(SliderUtils.isUnset(clusterName)){
+ if(isUnset(clusterName)){
throw new BadCommandArgumentsException("application name must be provided with --name option");
}
- SliderClusterOperations clusterOperations;
- AggregateConf instanceDefinition = null;
- try {
- clusterOperations = createClusterOperations(clusterName);
- instanceDefinition = clusterOperations.getInstanceDefinition();
- } catch (YarnException e) {
- log.error("Exception happened when retrieving instance definition from YARN: "
- + e.toString());
- throw e;
- } catch (IOException e) {
- log.error("Network problem happened when retrieving instance definition from YARN: "
- + e.toString());
- throw e;
- }
+ AggregateConf instanceDefinition = fetchInstanceDefinition(clusterName);
String imagePath = instanceDefinition.getInternalOperations().get(
- InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ INTERNAL_APPLICATION_IMAGE_PATH);
// if null, it will be uploaded by Slider and thus at slider's path
if (imagePath == null) {
ApplicationReport appReport = findInstance(clusterName);
@@ -3865,58 +3649,54 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
log.info("The path of slider agent tarball on HDFS is: " + imagePath);
}
- private void actionDiagnosticApplication(ActionDiagnosticArgs diagnosticArgs)
+ private AggregateConf fetchInstanceDefinition(String clusterName)
throws YarnException, IOException {
- // not using member variable clustername because we want to place
- // application name after --application option and member variable
- // cluster name has to be put behind action
- String clusterName = diagnosticArgs.name;
- if(SliderUtils.isUnset(clusterName)){
- throw new BadCommandArgumentsException("application name must be provided with --name option");
- }
SliderClusterOperations clusterOperations;
AggregateConf instanceDefinition = null;
try {
clusterOperations = createClusterOperations(clusterName);
instanceDefinition = clusterOperations.getInstanceDefinition();
- } catch (YarnException e) {
- log.error("Exception happened when retrieving instance definition from YARN: "
- + e.toString());
- throw e;
- } catch (IOException e) {
- log.error("Network problem happened when retrieving instance definition from YARN: "
+ } catch (YarnException | IOException e) {
+ log.error("Failed to retrieve instance definition from YARN: "
+ e.toString());
throw e;
}
+ return instanceDefinition;
+ }
+
+ private void actionDiagnosticApplication(ActionDiagnosticArgs diagnosticArgs)
+ throws YarnException, IOException {
+ // not using member variable clustername because we want to place
+ // application name after --application option and member variable
+ // cluster name has to be put behind action
+ String clusterName = diagnosticArgs.name;
+ requireArgumentSet(Arguments.ARG_NAME, clusterName);
+ AggregateConf instanceDefinition = fetchInstanceDefinition(clusterName);
String clusterDir = instanceDefinition.getAppConfOperations()
.getGlobalOptions().get(AgentKeys.APP_ROOT);
- String pkgTarball = SliderUtils
- .getApplicationDefinitionPath(instanceDefinition.getAppConfOperations());
+ String pkgTarball = getApplicationDefinitionPath(instanceDefinition.getAppConfOperations());
String runAsUser = instanceDefinition.getAppConfOperations()
.getGlobalOptions().get(AgentKeys.RUNAS_USER);
- log.info("The location of the cluster instance directory in HDFS is: "
- + clusterDir);
- log.info("The name of the application package tarball on HDFS is: "
- + pkgTarball);
- log.info("The runas user of the application in the cluster is: "
- + runAsUser);
+ log.info("The location of the cluster instance directory in HDFS is: {}", clusterDir);
+ log.info("The name of the application package tarball on HDFS is: {}",pkgTarball);
+ log.info("The runas user of the application in the cluster is: {}",runAsUser);
if (diagnosticArgs.verbose) {
- log.info("App config of the application: "
- + instanceDefinition.getAppConf().toJson());
- log.info("Resource config of the application: "
- + instanceDefinition.getResources().toJson());
+ log.info("App config of the application:\n{}",
+ instanceDefinition.getAppConf().toJson());
+ log.info("Resource config of the application:\n{}",
+ instanceDefinition.getResources().toJson());
}
}
private void actionDiagnosticClient(ActionDiagnosticArgs diagnosticArgs)
throws SliderException, IOException {
try {
- String currentCommandPath = SliderUtils.getCurrentCommandPath();
+ String currentCommandPath = getCurrentCommandPath();
SliderVersionInfo.loadAndPrintVersionInfo(log);
- String clientConfigPath = SliderUtils.getClientConfigPath();
- String jdkInfo = SliderUtils.getJDKInfo();
+ String clientConfigPath = getClientConfigPath();
+ String jdkInfo = getJDKInfo();
println("The slider command path: %s", currentCommandPath);
println("The slider-client.xml used by current running command path: %s",
clientConfigPath);
@@ -3924,7 +3704,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// security info
Configuration config = getConfig();
- if (SliderUtils.isHadoopClusterSecure(config)) {
+ if (isHadoopClusterSecure(config)) {
println("Hadoop Cluster is secure");
println("Login user is %s", UserGroupInformation.getLoginUser());
println("Current user is %s", UserGroupInformation.getCurrentUser());
@@ -3936,7 +3716,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// verbose?
if (diagnosticArgs.verbose) {
// do the environment
- Map<String, String> env = SliderUtils.getSystemEnv();
+ Map<String, String> env = getSystemEnv();
Set<String> envList = ConfigHelper.sortedConfigKeys(env.entrySet());
StringBuilder builder = new StringBuilder("Environment variables:\n");
for (String key : envList) {
@@ -3947,7 +3727,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// Java properties
builder = new StringBuilder("JVM Properties\n");
Map<String, String> props =
- SliderUtils.sortedMap(SliderUtils.toMap(System.getProperties()));
+ sortedMap(toMap(System.getProperties()));
for (Entry<String, String> entry : props.entrySet()) {
builder.append(entry.getKey()).append("=")
.append(entry.getValue()).append("\n");
@@ -3956,12 +3736,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
println(builder.toString());
// then the config
- println("Slider client configuration:\n"
- + ConfigHelper.dumpConfigToString(config));
-
+ println("Slider client configuration:\n" + ConfigHelper.dumpConfigToString(config));
}
- SliderUtils.validateSliderClientEnvironment(log);
+ validateSliderClientEnvironment(log);
} catch (SliderException | IOException e) {
log.error(e.toString());
throw e;
@@ -4122,8 +3900,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
RegistryRetriever retriever = new RegistryRetriever(getConfig(), instance);
boolean external = !registryArgs.internal;
- PublishedExportsSet exports =
- retriever.getExports(external);
+ PublishedExportsSet exports = retriever.getExports(external);
PublishedExports published = retriever.retrieveExports(exports,
registryArgs.getExport,
@@ -4395,7 +4172,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_FALSE;
}
- String version = SliderUtils.getSliderVersion();
+ String version = getSliderVersion();
Path dependencyLibTarGzip = sliderFileSystem.getDependencyTarGzip();
// Check if dependency has already been uploaded, in which case log
@@ -4409,23 +4186,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
String libDir = System.getProperty(SliderKeys.PROPERTY_LIB_DIR);
- if (SliderUtils.isSet(libDir)) {
+ if (isSet(libDir)) {
File srcFolder = new File(libDir);
File tempLibTarGzipFile = File.createTempFile(
SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME + "_",
SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT);
// copy all jars except slider-core-<version>.jar
- FilenameFilter jarFilter = new FilenameFilter() {
- public boolean accept(File dir, String name) {
- String lowercaseName = name.toLowerCase();
- if (lowercaseName.endsWith(".jar")) {
- return true;
- } else {
- return false;
- }
- }
- };
- SliderUtils.tarGzipFolder(srcFolder, tempLibTarGzipFile, jarFilter);
+ tarGzipFolder(srcFolder, tempLibTarGzipFile, createJarFilter());
log.info("Uploading dependency for AM (version {}) from {} to {}",
version, tempLibTarGzipFile.toURI(), dependencyLibTarGzip.toUri());
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index d34dd2a..a1ad690 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -159,7 +159,7 @@ public class CoreFileSystem {
Preconditions.checkNotNull(packageName);
Path path = getBaseApplicationPath();
path = new Path(path, SliderKeys.PACKAGE_DIRECTORY + "/" + packageName);
- if (StringUtils.isNotEmpty(packageVersion)) {
+ if (SliderUtils.isSet(packageVersion)) {
path = new Path(path, packageVersion);
}
return path;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index f380ec6..1f97982 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -1034,7 +1034,7 @@ public final class SliderUtils {
* something other than 0.0.0.0
*/
public static boolean isAddressDefined(InetSocketAddress address) {
- return !(address.getHostName().equals("0.0.0.0"));
+ return !(address.getHostString().equals("0.0.0.0"));
}
public static void setRmAddress(Configuration conf, String rmAddr) {
@@ -1352,16 +1352,7 @@ public final class SliderUtils {
log.info("Loading all dependencies from {}", srcPath);
if (SliderUtils.isSet(srcPath)) {
File srcFolder = new File(srcPath);
- FilenameFilter jarFilter = new FilenameFilter() {
- public boolean accept(File dir, String name) {
- String lowercaseName = name.toLowerCase();
- if (lowercaseName.endsWith(".jar")) {
- return true;
- } else {
- return false;
- }
- }
- };
+ FilenameFilter jarFilter = createJarFilter();
File[] listOfJars = srcFolder.listFiles(jarFilter);
for (File jarFile : listOfJars) {
LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName());
@@ -1371,6 +1362,18 @@ public final class SliderUtils {
}
/**
+ * Accept all filenames ending with {@code .jar}
+ * @return a filename filter
+ */
+ public static FilenameFilter createJarFilter() {
+ return new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.toLowerCase(Locale.ENGLISH).endsWith(".jar");
+ }
+ };
+ }
+
+ /**
* Submit the AM tar.gz containing all dependencies and map it
* @param providerResources provider map to build up
* @param sliderFileSystem remote fs
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
index b4ca791..06dbfea 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
@@ -220,8 +220,7 @@ public class AppMasterLauncher extends AbstractLauncher {
}
Token<? extends TokenIdentifier>[] tokens = null;
- boolean tokensProvided =
- this.getConf().get(MAPREDUCE_JOB_CREDENTIALS_BINARY) != null;
+ boolean tokensProvided = getConf().get(MAPREDUCE_JOB_CREDENTIALS_BINARY) != null;
if (!tokensProvided) {
// For now, only getting tokens for the default file-system.
FileSystem fs = coreFileSystem.getFileSystem();
@@ -232,7 +231,7 @@ public class AppMasterLauncher extends AbstractLauncher {
if (tokens != null && tokens.length > 0) {
AbstractDelegationTokenIdentifier id =
(AbstractDelegationTokenIdentifier)tokens[0].decodeIdentifier();
- Date d = new Date(id.getIssueDate() + 24*60*60*1000);
+ Date d = new Date(id.getIssueDate() + 24 * 60 * 60 * 1000);
log.info("HDFS delegation tokens for AM launch context require renewal by {}",
DateFormat.getDateTimeInstance().format(d));
} else {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
index 3ba0181..6eb4058 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
@@ -38,7 +38,6 @@ import java.util.List;
public class ClasspathConstructor {
public static final String CLASS_PATH_SEPARATOR = ApplicationConstants.CLASS_PATH_SEPARATOR;
-// public static final String CLASS_PATH_SEPARATOR = File.pathSeparator;
private final List<String> pathElements = new ArrayList<>();
public ClasspathConstructor() {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java b/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
index dbaa981..57b8965 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/CommandLineBuilder.java
@@ -19,8 +19,11 @@
package org.apache.slider.core.launch;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.slider.common.params.Arguments;
import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
import java.util.ArrayList;
import java.util.List;
@@ -101,4 +104,56 @@ public class CommandLineBuilder {
return argumentList;
}
+ public boolean addConfOption(Configuration conf, String key) {
+ String val = conf.get(key);
+ return defineIfSet(key, val);
+ }
+
+ public String addConfOptionToCLI(Configuration conf,
+ String key,
+ String defVal) {
+ String val = conf.get(key, defVal);
+ define(key, val);
+ return val;
+ }
+
+ /**
+ * Add a <code>-D key=val</code> command to the CLI
+ * @param key key
+ * @param val value
+ */
+ public void define(String key, String val) {
+ Preconditions.checkArgument(key != null, "null key");
+ Preconditions.checkArgument(val != null, "null value");
+ add(Arguments.ARG_DEFINE, key + "=" + val);
+ }
+
+ /**
+ * Add a <code>-D key=val</code> command to the CLI if <code>val</code>
+ * is not null
+ * @param key key
+ * @param val value
+ */
+ public boolean defineIfSet(String key, String val) {
+ Preconditions.checkArgument(key != null, "null key");
+ if (val != null) {
+ define(key, val);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Add a mandatory config option
+ * @param conf configuration
+ * @param key key
+ * @throws BadConfigException if the key is missing
+ */
+ public void addMandatoryConfOption(Configuration conf,
+ String key) throws BadConfigException {
+ if (!addConfOption(conf, key)) {
+ throw new BadConfigException("Missing configuration option: " + key);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
index 7f3931b..1622309 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
@@ -98,11 +98,23 @@ public abstract class AbstractSliderLaunchedService extends
*/
protected static void requireArgumentSet(String argname, String value)
throws BadCommandArgumentsException {
- if (isUnset(value)) {
- throw new BadCommandArgumentsException(
- "Required argument " + argname + " missing");
- }
+ require(isSet(value), "Required argument %s missing", argname );
}
+ /**
+ * Require a condition to hold; throw {@link BadCommandArgumentsException} if not.
+ * The exception text is the formatted message.
+ * @param condition condition
+ * @param message string to format
+ * @param args list of arguments to format.
+ * @throws BadCommandArgumentsException
+ */
+ protected static void require(boolean condition, String message,
+ Object... args)
+ throws BadCommandArgumentsException {
+ if (!condition) {
+ throw new BadCommandArgumentsException(message, args);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/227339e5/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy
index 8245dae..7e3f55a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy
@@ -41,8 +41,7 @@ import org.junit.Test
class TestActionPackage extends AgentMiniClusterTestBase {
- public static final String E_INVALID_APP_TYPE =
- "A valid application type name is required (e.g. HBASE)"
+ public static final String E_NO_NAME = "Required argument --name missing"
String s = File.separator
File packageFile = new File("src${s}test${s}resources${s}log4j.properties")
@@ -70,7 +69,7 @@ class TestActionPackage extends AgentMiniClusterTestBase {
} catch (BadCommandArgumentsException e) {
assertExceptionDetails(e,
LauncherExitCodes.EXIT_COMMAND_ARGUMENT_ERROR,
- E_INVALID_APP_TYPE)
+ E_NO_NAME)
}
}
@@ -285,7 +284,7 @@ class TestActionPackage extends AgentMiniClusterTestBase {
} catch (BadCommandArgumentsException e) {
assertExceptionDetails(e,
LauncherExitCodes.EXIT_COMMAND_ARGUMENT_ERROR,
- E_INVALID_APP_TYPE)
+ E_NO_NAME)
}
}
}