You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bi...@apache.org on 2017/11/04 19:32:01 UTC
hadoop git commit: YARN-7423. Improve service client loggings.
Contributed by Jian He
Repository: hadoop
Updated Branches:
refs/heads/yarn-native-services f2c7f4ee6 -> d81335927
YARN-7423. Improve service client loggings. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d8133592
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d8133592
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d8133592
Branch: refs/heads/yarn-native-services
Commit: d81335927c5c80fc4ac52369370bab41de171b90
Parents: f2c7f4e
Author: Billie Rinaldi <bi...@apache.org>
Authored: Sat Nov 4 12:27:42 2017 -0700
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Sat Nov 4 12:27:42 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/service/ServiceMaster.java | 4 +-
.../yarn/service/api/records/Configuration.java | 8 +-
.../yarn/service/client/ServiceClient.java | 50 +-
.../yarn/service/component/Component.java | 10 +-
.../component/instance/ComponentInstance.java | 4 +-
.../containerlaunch/AbstractLauncher.java | 6 +-
.../containerlaunch/ClasspathConstructor.java | 4 +-
.../containerlaunch/CommandLineBuilder.java | 4 +-
.../containerlaunch/JavaCommandLineBuilder.java | 4 +-
.../yarn/service/monitor/probe/HttpProbe.java | 4 +-
.../yarn/service/monitor/probe/PortProbe.java | 4 +-
.../provider/AbstractClientProvider.java | 8 +-
.../provider/AbstractProviderService.java | 4 +-
.../yarn/service/provider/ProviderUtils.java | 10 +-
.../registry/YarnRegistryViewForProviders.java | 6 +-
.../yarn/service/utils/CoreFileSystem.java | 2 +-
.../hadoop/yarn/service/utils/PortScanner.java | 4 +-
.../yarn/service/utils/ServiceApiUtil.java | 2 +-
.../hadoop/yarn/service/utils/ServiceUtils.java | 545 ++++++++++++++++++
.../hadoop/yarn/service/utils/SliderUtils.java | 564 -------------------
.../yarn/service/utils/ZookeeperUtils.java | 4 +-
.../service/conf/TestValidateServiceNames.java | 1 -
22 files changed, 614 insertions(+), 638 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8133592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
index 2d86a5a..b0b4f06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
@@ -39,7 +39,7 @@ import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
import org.apache.hadoop.yarn.service.monitor.ServiceMonitor;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -112,7 +112,7 @@ public class ServiceMaster extends CompositeService {
}
protected ContainerId getAMContainerId() throws BadClusterStateException {
- return ContainerId.fromString(SliderUtils.mandatoryEnvVariable(
+ return ContainerId.fromString(ServiceUtils.mandatoryEnvVariable(
ApplicationConstants.Environment.CONTAINER_ID.name()));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8133592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
index dc6dff8..859c1ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
@@ -24,7 +24,7 @@ import io.swagger.annotations.ApiModelProperty;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import java.io.Serializable;
import java.util.ArrayList;
@@ -201,9 +201,9 @@ public class Configuration implements Serializable {
* this ConfigFile.
*/
public synchronized void mergeFrom(Configuration that) {
- SliderUtils.mergeMapsIgnoreDuplicateKeys(this.properties, that
+ ServiceUtils.mergeMapsIgnoreDuplicateKeys(this.properties, that
.getProperties());
- SliderUtils.mergeMapsIgnoreDuplicateKeys(this.env, that.getEnv());
+ ServiceUtils.mergeMapsIgnoreDuplicateKeys(this.env, that.getEnv());
Map<String, ConfigFile> thatMap = new HashMap<>();
for (ConfigFile file : that.getFiles()) {
@@ -212,7 +212,7 @@ public class Configuration implements Serializable {
for (ConfigFile thisFile : files) {
if(thatMap.containsKey(thisFile.getDestFile())) {
ConfigFile thatFile = thatMap.get(thisFile.getDestFile());
- SliderUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProperties(),
+ ServiceUtils.mergeMapsIgnoreDuplicateKeys(thisFile.getProperties(),
thatFile.getProperties());
thatMap.remove(thisFile.getDestFile());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8133592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 775a642..11cd30d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -69,7 +69,7 @@ import org.apache.hadoop.yarn.service.provider.ProviderUtils;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.apache.hadoop.yarn.service.utils.ZookeeperUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.Times;
@@ -80,19 +80,13 @@ import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.text.MessageFormat;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import static org.apache.hadoop.yarn.api.records.YarnApplicationState.*;
import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*;
import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser;
-import static org.apache.hadoop.yarn.service.utils.SliderUtils.*;
+import static org.apache.hadoop.yarn.service.utils.ServiceUtils.*;
@InterfaceAudience.Public
@InterfaceStability.Unstable
@@ -169,7 +163,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
fileName);
}
Path filePath = new Path(file.getAbsolutePath());
- LOG.info("Loading service definition from: " + filePath);
+ LOG.info("Loading service definition from local FS: " + filePath);
Service service = jsonSerDeser
.load(FileSystem.getLocal(getConfig()), filePath);
if (!StringUtils.isEmpty(serviceName)) {
@@ -482,7 +476,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
getConfig().get(RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
// though if neither is set: trouble
- if (SliderUtils.isUnset(registryQuorum)) {
+ if (ServiceUtils.isUnset(registryQuorum)) {
throw new BadConfigException(
"No Zookeeper quorum provided in the" + " configuration property "
+ RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
@@ -505,7 +499,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
types.add(YarnServiceConstants.APP_TYPE);
Set<String> tags = null;
if (serviceName != null) {
- tags = Collections.singleton(SliderUtils.createNameTag(serviceName));
+ tags = Collections.singleton(ServiceUtils.createNameTag(serviceName));
}
GetApplicationsRequest request = GetApplicationsRequest.newInstance();
request.setApplicationTypes(types);
@@ -652,7 +646,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
// write out the path output
CLI.addOutAndErrFiles(STDOUT_AM, STDERR_AM);
String cmdStr = CLI.build();
- LOG.info("AM launch command: {}", cmdStr);
+ LOG.debug("AM launch command: {}", cmdStr);
return cmdStr;
}
@@ -671,12 +665,12 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
}
if (!UserGroupInformation.isSecurityEnabled()) {
String userName = UserGroupInformation.getCurrentUser().getUserName();
- LOG.info("Run as user " + userName);
+ LOG.debug("Run as user " + userName);
// HADOOP_USER_NAME env is used by UserGroupInformation when log in
// This env makes AM run as this user
env.put("HADOOP_USER_NAME", userName);
}
- LOG.info("AM env: \n{}", stringifyMap(env));
+ LOG.debug("AM env: \n{}", stringifyMap(env));
return env;
}
@@ -689,11 +683,14 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
libPath, "lib", false);
Path dependencyLibTarGzip = fs.getDependencyTarGzip();
if (fs.isFile(dependencyLibTarGzip)) {
- LOG.info("Loading lib tar from " + fs.getFileSystem().getScheme() + ":/"
+ LOG.debug("Loading lib tar from " + fs.getFileSystem().getScheme() + ":/"
+ dependencyLibTarGzip);
- SliderUtils.putAmTarGzipAndUpdate(localResources, fs);
+ fs.submitTarGzipAndUpdate(localResources);
} else {
- String[] libs = SliderUtils.getLibDirs();
+ String[] libs = ServiceUtils.getLibDirs();
+ LOG.info("Uploading all dependency jars to HDFS. For faster submission of" +
+ " apps, pre-upload dependency jars to HDFS "
+ + "using command: yarn app -enableFastLaunch");
for (String libDirProp : libs) {
ProviderUtils.addAllDependencyJars(localResources, fs, libPath, "lib",
libDirProp);
@@ -740,7 +737,8 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
ApplicationId appId = submitApp(service);
service.setId(appId.toString());
// write app definition on to hdfs
- createDirAndPersistApp(appDir, service);
+ Path appJson = persistAppDef(appDir, service);
+ LOG.info("Persisted service " + service.getName() + " at " + appJson);
return 0;
}
@@ -763,16 +761,14 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
throws IOException, SliderException {
FsPermission appDirPermission = new FsPermission("750");
fs.createWithPermissions(appDir, appDirPermission);
- persistAppDef(appDir, service);
+ Path appJson = persistAppDef(appDir, service);
+ LOG.info("Persisted service " + service.getName() + " at " + appJson);
}
- private void persistAppDef(Path appDir, Service service)
- throws IOException {
+ private Path persistAppDef(Path appDir, Service service) throws IOException {
Path appJson = new Path(appDir, service.getName() + ".json");
- jsonSerDeser
- .save(fs.getFileSystem(), appJson, service, true);
- LOG.info(
- "Persisted service " + service.getName() + " at " + appJson);
+ jsonSerDeser.save(fs.getFileSystem(), appJson, service, true);
+ return appJson;
}
private void addKeytabResourceIfSecure(SliderFileSystem fileSystem,
@@ -922,7 +918,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
return EXIT_SUCCESS;
}
- String[] libDirs = SliderUtils.getLibDirs();
+ String[] libDirs = ServiceUtils.getLibDirs();
if (libDirs.length > 0) {
File tempLibTarGzipFile = File.createTempFile(
YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME + "_",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8133592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
index 7667a53..7208f39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -40,7 +40,7 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.Apps;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.apache.hadoop.yarn.service.monitor.probe.MonitorUtils;
import org.apache.hadoop.yarn.service.monitor.probe.Probe;
import org.slf4j.Logger;
@@ -419,7 +419,7 @@ public class Component implements EventHandler<ComponentEvent> {
public boolean areDependenciesReady() {
List<String> dependencies = componentSpec.getDependencies();
- if (SliderUtils.isEmpty(dependencies)) {
+ if (ServiceUtils.isEmpty(dependencies)) {
return true;
}
for (String dependency : dependencies) {
@@ -445,7 +445,7 @@ public class Component implements EventHandler<ComponentEvent> {
public Map<String, String> getDependencyHostIpTokens() {
Map<String, String> tokens = new HashMap<>();
List<String> dependencies = componentSpec.getDependencies();
- if (SliderUtils.isEmpty(dependencies)) {
+ if (ServiceUtils.isEmpty(dependencies)) {
return tokens;
}
for (String dependency : dependencies) {
@@ -455,8 +455,8 @@ public class Component implements EventHandler<ComponentEvent> {
if (instance.getContainerStatus() == null) {
continue;
}
- if (SliderUtils.isEmpty(instance.getContainerStatus().getIPs()) ||
- SliderUtils.isUnset(instance.getContainerStatus().getHost())) {
+ if (ServiceUtils.isEmpty(instance.getContainerStatus().getIPs()) ||
+ ServiceUtils.isUnset(instance.getContainerStatus().getHost())) {
continue;
}
String ip = instance.getContainerStatus().getIPs().get(0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8133592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
index 2492454..9e5f98c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
@@ -44,7 +44,7 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.BoundedAppender;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
import org.apache.hadoop.yarn.service.monitor.probe.ProbeStatus;
import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
@@ -503,7 +503,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
+ nodeId + ", will try again", e);
return;
}
- if (SliderUtils.isEmpty(status.getIPs()) || SliderUtils
+ if (ServiceUtils.isEmpty(status.getIPs()) || ServiceUtils
.isUnset(status.getHost())) {
return;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8133592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
index e4eae20..2d7c3bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
@@ -28,7 +28,7 @@ import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.utils.CoreFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -132,7 +132,7 @@ public class AbstractLauncher {
*/
public ContainerLaunchContext completeContainerLaunch() throws IOException {
- String cmdStr = SliderUtils.join(commands, " ", false);
+ String cmdStr = ServiceUtils.join(commands, " ", false);
log.debug("Completed setting up container command {}", cmdStr);
containerLaunchContext.setCommands(commands);
@@ -205,7 +205,7 @@ public class AbstractLauncher {
String key = entry.getKey();
LocalResource val = entry.getValue();
- log.debug(key + "=" + SliderUtils.stringify(val.getResource()));
+ log.debug(key + "=" + ServiceUtils.stringify(val.getResource()));
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8133592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
index 22b3877..711abb2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ClasspathConstructor.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import java.util.ArrayList;
import java.util.Arrays;
@@ -63,7 +63,7 @@ public class ClasspathConstructor {
}
public String buildClasspath() {
- return SliderUtils.join(pathElements,
+ return ServiceUtils.join(pathElements,
CLASS_PATH_SEPARATOR,
false);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8133592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
index 7baa284..5ed56e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.service.containerlaunch;
import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import java.util.ArrayList;
import java.util.List;
@@ -81,6 +81,6 @@ public class CommandLineBuilder {
* @return the command line
*/
public String build() {
- return SliderUtils.join(argumentList, " ");
+ return ServiceUtils.join(argumentList, " ");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8133592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
index cbcb0d6..7f6cc04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
@@ -22,7 +22,7 @@ package org.apache.hadoop.yarn.service.containerlaunch;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
import java.util.Map;
@@ -53,7 +53,7 @@ public class JavaCommandLineBuilder extends CommandLineBuilder {
* trimmed.
*/
public void setJVMHeap(String heap) {
- if (SliderUtils.isSet(heap)) {
+ if (ServiceUtils.isSet(heap)) {
add("-Xmx" + heap.trim());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8133592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
index 1923086..1ed13a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/HttpProbe.java
@@ -21,7 +21,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -75,7 +75,7 @@ public class HttpProbe extends Probe {
public ProbeStatus ping(ComponentInstance instance) {
ProbeStatus status = new ProbeStatus();
ContainerStatus containerStatus = instance.getContainerStatus();
- if (containerStatus == null || SliderUtils.isEmpty(containerStatus.getIPs())
+ if (containerStatus == null || ServiceUtils.isEmpty(containerStatus.getIPs())
|| StringUtils.isEmpty(containerStatus.getHost())) {
status.fail(this, new IOException("IP is not available yet"));
return status;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8133592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
index aba5859..85569f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.service.monitor.probe;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,7 +67,7 @@ public class PortProbe extends Probe {
public ProbeStatus ping(ComponentInstance instance) {
ProbeStatus status = new ProbeStatus();
- if (instance.getContainerStatus() == null || SliderUtils
+ if (instance.getContainerStatus() == null || ServiceUtils
.isEmpty(instance.getContainerStatus().getIPs())) {
status.fail(this, new IOException(
instance.getCompInstanceName() + ": IP is not available yet"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8133592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
index 75c647b..fc8953c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractClientProvider.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.service.api.records.Artifact;
import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import java.io.IOException;
import java.nio.file.Paths;
@@ -53,12 +53,12 @@ public abstract class AbstractClientProvider {
public static final Set<String> createApplicationTags(String appName,
String appVersion, String appDescription) {
Set<String> tags = new HashSet<>();
- tags.add(SliderUtils.createNameTag(appName));
+ tags.add(ServiceUtils.createNameTag(appName));
if (appVersion != null) {
- tags.add(SliderUtils.createVersionTag(appVersion));
+ tags.add(ServiceUtils.createVersionTag(appVersion));
}
if (appDescription != null) {
- tags.add(SliderUtils.createDescriptionTag(appDescription));
+ tags.add(ServiceUtils.createDescriptionTag(appDescription));
}
return tags;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8133592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
index 6ffb84d..6d74061 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
import org.apache.hadoop.yarn.service.containerlaunch.CommandLineBuilder;
@@ -70,7 +70,7 @@ public abstract class AbstractProviderService implements ProviderService,
.initCompTokensForSubstitute(instance);
tokensForSubstitution.putAll(globalTokens);
// Set the environment variables in launcher
- launcher.putEnv(SliderUtils
+ launcher.putEnv(ServiceUtils
.buildEnvMap(component.getConfiguration(), tokensForSubstitution));
launcher.setEnv("WORK_DIR", ApplicationConstants.Environment.PWD.$());
launcher.setEnv("LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8133592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
index 582c5ac..e074dd7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -41,7 +41,7 @@ import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.apache.hadoop.yarn.service.utils.PublishedConfiguration;
import org.apache.hadoop.yarn.service.utils.PublishedConfigurationOutputter;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -94,7 +94,7 @@ public class ProviderUtils implements YarnServiceConstants {
IOException,
SliderException {
try {
- SliderUtils.putJar(providerResources,
+ ServiceUtils.putJar(providerResources,
sliderFileSystem,
providerClass,
tempPath,
@@ -127,14 +127,14 @@ public class ProviderUtils implements YarnServiceConstants {
String libDir,
String libLocalSrcDir)
throws IOException, SliderException {
- if (SliderUtils.isSet(libLocalSrcDir)) {
+ if (ServiceUtils.isSet(libLocalSrcDir)) {
File file = new File(libLocalSrcDir);
if (!file.exists() || !file.isDirectory()) {
throw new BadCommandArgumentsException(
"Supplied lib src dir %s is not valid", libLocalSrcDir);
}
}
- SliderUtils.putAllJars(providerResources, sliderFileSystem, tempPath,
+ ServiceUtils.putAllJars(providerResources, sliderFileSystem, tempPath,
libDir, libLocalSrcDir);
}
@@ -174,7 +174,7 @@ public class ProviderUtils implements YarnServiceConstants {
Configuration conf = service.getConfiguration();
String keytabPathOnHost =
conf.getProperty(YarnServiceConf.KEY_AM_KEYTAB_LOCAL_PATH);
- if (SliderUtils.isUnset(keytabPathOnHost)) {
+ if (ServiceUtils.isUnset(keytabPathOnHost)) {
String amKeytabName =
conf.getProperty(YarnServiceConf.KEY_AM_LOGIN_KEYTAB_NAME);
String keytabDir =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8133592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
index d418b59..cecca5f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,9 +70,9 @@ public class YarnRegistryViewForProviders {
Preconditions.checkArgument(registryOperations != null,
"null registry operations");
Preconditions.checkArgument(user != null, "null user");
- Preconditions.checkArgument(SliderUtils.isSet(serviceClass),
+ Preconditions.checkArgument(ServiceUtils.isSet(serviceClass),
"unset service class");
- Preconditions.checkArgument(SliderUtils.isSet(instanceName),
+ Preconditions.checkArgument(ServiceUtils.isSet(instanceName),
"instanceName");
Preconditions.checkArgument(applicationAttemptId != null,
"null applicationAttemptId");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8133592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
index 281e1dfe..6b9d4d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
@@ -409,7 +409,7 @@ public class CoreFileSystem {
*/
public LocalResource submitJarWithClass(Class clazz, Path tempPath, String subdir, String jarName)
throws IOException, SliderException {
- File localFile = SliderUtils.findContainingJarOrFail(clazz);
+ File localFile = ServiceUtils.findContainingJarOrFail(clazz);
return submitFile(localFile, tempPath, subdir, jarName);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8133592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
index 2dbf37f..1d64ed6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/PortScanner.java
@@ -85,7 +85,7 @@ public class PortScanner {
if (remainingPortsToCheck != null) {
return getAvailablePortViaPortArray();
} else {
- return SliderUtils.getOpenPort();
+ return ServiceUtils.getOpenPort();
}
}
@@ -95,7 +95,7 @@ public class PortScanner {
Iterator<Integer> portsToCheck = this.remainingPortsToCheck.iterator();
while (portsToCheck.hasNext() && !found) {
int portToCheck = portsToCheck.next();
- found = SliderUtils.isPortAvailable(portToCheck);
+ found = ServiceUtils.isPortAvailable(portToCheck);
if (found) {
availablePort = portToCheck;
portsToCheck.remove();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8133592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
index 68db0bb..b58cea8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
@@ -369,7 +369,7 @@ public class ServiceApiUtil {
continue;
}
boolean dependenciesAlreadySorted = true;
- if (!SliderUtils.isEmpty(component.getDependencies())) {
+ if (!ServiceUtils.isEmpty(component.getDependencies())) {
for (String dependency : component.getDependencies()) {
if (!sortedComponents.containsKey(dependency)) {
dependenciesAlreadySorted = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8133592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
new file mode 100644
index 0000000..e18bcae
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceUtils.java
@@ -0,0 +1,545 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPOutputStream;
+
+/**
+ * These are slider-specific Util methods
+ */
+public final class ServiceUtils {
+
+ private static final Logger log = LoggerFactory.getLogger(ServiceUtils.class);
+
+ private ServiceUtils() {
+ }
+
+ /**
+ * Implementation of set-ness, groovy definition of true/false for a string
+ * @param s string
+ * @return true iff the string is neither null nor empty
+ */
+ public static boolean isUnset(String s) {
+ return s == null || s.isEmpty();
+ }
+
+ public static boolean isSet(String s) {
+ return !isUnset(s);
+ }
+
+ public static boolean isEmpty(Collection l) {
+ return l == null || l.isEmpty();
+ }
+
+ /**
+ * Find a containing JAR
+ * @param clazz class to find
+ * @return the file
+ * @throws IOException any IO problem, including the class not having a
+ * classloader
+ * @throws FileNotFoundException if the class did not resolve to a file
+ */
+ public static File findContainingJarOrFail(Class clazz) throws IOException {
+ File localFile = ServiceUtils.findContainingJar(clazz);
+ if (null == localFile) {
+ throw new FileNotFoundException("Could not find JAR containing " + clazz);
+ }
+ return localFile;
+ }
+
+
+ /**
+ * Find a containing JAR
+ * @param my_class class to find
+ * @return the file or null if it is not found
+ * @throws IOException any IO problem, including the class not having a
+ * classloader
+ */
+ public static File findContainingJar(Class my_class) throws IOException {
+ ClassLoader loader = my_class.getClassLoader();
+ if (loader == null) {
+ throw new IOException(
+ "Class " + my_class + " does not have a classloader!");
+ }
+ String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
+ Enumeration<URL> urlEnumeration = loader.getResources(class_file);
+ for (; urlEnumeration.hasMoreElements(); ) {
+ URL url = urlEnumeration.nextElement();
+ if ("jar".equals(url.getProtocol())) {
+ String toReturn = url.getPath();
+ if (toReturn.startsWith("file:")) {
+ toReturn = toReturn.substring("file:".length());
+ }
+ // URLDecoder is a misnamed class, since it actually decodes
+ // x-www-form-urlencoded MIME type rather than actual
+ // URL encoding (which the file path has). Therefore it would
+ // decode +s to ' 's which is incorrect (spaces are actually
+ // either unencoded or encoded as "%20"). Replace +s first, so
+ // that they are kept sacred during the decoding process.
+ toReturn = toReturn.replaceAll("\\+", "%2B");
+ toReturn = URLDecoder.decode(toReturn, "UTF-8");
+ String jarFilePath = toReturn.replaceAll("!.*$", "");
+ return new File(jarFilePath);
+ } else {
+ log.info("could not locate JAR containing {} URL={}", my_class, url);
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Copy a file to a new FS -both paths must be qualified.
+ * @param conf conf file
+ * @param srcFile src file
+ * @param destFile dest file
+ */
+ @SuppressWarnings("deprecation")
+ public static void copy(Configuration conf,
+ Path srcFile,
+ Path destFile) throws
+ IOException,
+ BadClusterStateException {
+ FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf);
+ //list all paths in the src.
+ if (!srcFS.exists(srcFile)) {
+ throw new FileNotFoundException("Source file not found " + srcFile);
+ }
+ if (!srcFS.isFile(srcFile)) {
+ throw new FileNotFoundException(
+ "Source file not a file " + srcFile);
+ }
+ FileSystem destFS = FileSystem.get(destFile.toUri(), conf);
+ FileUtil.copy(srcFS, srcFile, destFS, destFile, false, true, conf);
+ }
+
+ /**
+ * Take a collection, return a list containing the string value of every
+ * element in the collection.
+ * @param c collection
+ * @return a stringified list
+ */
+ public static List<String> collectionToStringList(Collection c) {
+ List<String> l = new ArrayList<>(c.size());
+ for (Object o : c) {
+ l.add(o.toString());
+ }
+ return l;
+ }
+
+ /**
+ * Join an collection of objects with a separator that appears after every
+ * instance in the list -including at the end
+ * @param collection collection to call toString() on each element
+ * @param separator separator string
+ * @return the joined entries
+ */
+ public static String join(Collection collection, String separator) {
+ return join(collection, separator, true);
+ }
+
+ /**
+ * Join an collection of objects with a separator that appears after every
+ * instance in the list -optionally at the end
+ * @param collection collection to call toString() on each element
+ * @param separator separator string
+ * @param trailing add a trailing entry or not
+ * @return the joined entries
+ */
+ public static String join(Collection collection,
+ String separator,
+ boolean trailing) {
+ StringBuilder b = new StringBuilder();
+ // fast return on empty collection
+ if (collection.isEmpty()) {
+ return trailing ? separator : "";
+ }
+ for (Object o : collection) {
+ b.append(o);
+ b.append(separator);
+ }
+ int length = separator.length();
+ String s = b.toString();
+ return (trailing || s.isEmpty()) ?
+ s : (b.substring(0, b.length() - length));
+ }
+
+ /**
+ * Join an array of strings with a separator that appears after every
+ * instance in the list -optionally at the end
+ * @param collection strings
+ * @param separator separator string
+ * @param trailing add a trailing entry or not
+ * @return the joined entries
+ */
+ public static String join(String[] collection, String separator,
+ boolean trailing) {
+ return join(Arrays.asList(collection), separator, trailing);
+ }
+
+ /**
+ * Resolve a mandatory environment variable
+ * @param key env var
+ * @return the resolved value
+ * @throws BadClusterStateException
+ */
+ public static String mandatoryEnvVariable(String key) throws
+ BadClusterStateException {
+ String v = System.getenv(key);
+ if (v == null) {
+ throw new BadClusterStateException("Missing Environment variable " + key);
+ }
+ return v;
+ }
+
+ /**
+ * Generic map merge logic
+ * @param first first map
+ * @param second second map
+ * @param <T1> key type
+ * @param <T2> value type
+ * @return 'first' merged with the second
+ */
+ public static <T1, T2> Map<T1, T2> mergeMapsIgnoreDuplicateKeys(Map<T1, T2> first,
+ Map<T1, T2> second) {
+ Preconditions.checkArgument(first != null, "Null 'first' value");
+ Preconditions.checkArgument(second != null, "Null 'second' value");
+ for (Map.Entry<T1, T2> entry : second.entrySet()) {
+ T1 key = entry.getKey();
+ if (!first.containsKey(key)) {
+ first.put(key, entry.getValue());
+ }
+ }
+ return first;
+ }
+
+ /**
+ * Convert a map to a multi-line string for printing
+ * @param map map to stringify
+ * @return a string representation of the map
+ */
+ public static String stringifyMap(Map<String, String> map) {
+ StringBuilder builder = new StringBuilder();
+ for (Map.Entry<String, String> entry : map.entrySet()) {
+ builder.append(entry.getKey())
+ .append("=\"")
+ .append(entry.getValue())
+ .append("\"\n");
+
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Convert a YARN URL into a string value of a normal URL
+ * @param url URL
+ * @return string representatin
+ */
+ public static String stringify(org.apache.hadoop.yarn.api.records.URL url) {
+ StringBuilder builder = new StringBuilder();
+ builder.append(url.getScheme()).append("://");
+ if (url.getHost() != null) {
+ builder.append(url.getHost()).append(":").append(url.getPort());
+ }
+ builder.append(url.getFile());
+ return builder.toString();
+ }
+
+ /**
+ * Get a random open port
+ * @return true if the port was available for listening on
+ */
+ public static int getOpenPort() throws IOException {
+ ServerSocket socket = null;
+ try {
+ socket = new ServerSocket(0);
+ return socket.getLocalPort();
+ } finally {
+ if (socket != null) {
+ socket.close();
+ }
+ }
+ }
+
+ /**
+ * See if a port is available for listening on by trying to listen
+ * on it and seeing if that works or fails.
+ * @param port port to listen to
+ * @return true if the port was available for listening on
+ */
+ public static boolean isPortAvailable(int port) {
+ try {
+ ServerSocket socket = new ServerSocket(port);
+ socket.close();
+ return true;
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ // Build env map: key -> value;
+ // value will be replaced by the corresponding value in tokenMap, if any.
+ public static Map<String, String> buildEnvMap(
+ org.apache.hadoop.yarn.service.api.records.Configuration conf,
+ Map<String,String> tokenMap) {
+ if (tokenMap == null) {
+ return conf.getEnv();
+ }
+ Map<String, String> env = new HashMap<>();
+ for (Map.Entry<String, String> entry : conf.getEnv().entrySet()) {
+ String key = entry.getKey();
+ String val = entry.getValue();
+ for (Map.Entry<String,String> token : tokenMap.entrySet()) {
+ val = val.replaceAll(Pattern.quote(token.getKey()),
+ token.getValue());
+ }
+ env.put(key,val);
+ }
+ return env;
+ }
+
+ public static String[] getLibDirs() {
+ String libDirStr = System.getProperty(YarnServiceConstants.PROPERTY_LIB_DIR);
+ if (isUnset(libDirStr)) {
+ return ArrayUtils.EMPTY_STRING_ARRAY;
+ }
+ return StringUtils.split(libDirStr, ',');
+ }
+
+ /**
+ * Submit a JAR containing a specific class and map it
+ * @param providerResources provider map to build up
+ * @param sliderFileSystem remote fs
+ * @param clazz class to look for
+ * @param libdir lib directory
+ * @param jarName <i>At the destination</i>
+ * @return the local resource ref
+ * @throws IOException trouble copying to HDFS
+ */
+ public static LocalResource putJar(Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Class clazz,
+ Path tempPath,
+ String libdir,
+ String jarName
+ )
+ throws IOException, SliderException {
+ LocalResource res = sliderFileSystem.submitJarWithClass(
+ clazz,
+ tempPath,
+ libdir,
+ jarName);
+ providerResources.put(libdir + "/" + jarName, res);
+ return res;
+ }
+
+ /**
+ * Submit a JAR containing and map it
+ * @param providerResources provider map to build up
+ * @param sliderFileSystem remote fs
+ * @param libDir lib directory
+ * @param srcPath copy jars from
+ */
+ public static void putAllJars(Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Path tempPath,
+ String libDir,
+ String srcPath) throws IOException, SliderException {
+ log.debug("Loading all dependencies from {}", srcPath);
+ if (ServiceUtils.isSet(srcPath)) {
+ File srcFolder = new File(srcPath);
+ FilenameFilter jarFilter = createJarFilter();
+ File[] listOfJars = srcFolder.listFiles(jarFilter);
+ if (listOfJars == null || listOfJars.length == 0) {
+ return;
+ }
+ for (File jarFile : listOfJars) {
+ LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName());
+ providerResources.put(libDir + "/" + jarFile.getName(), res);
+ }
+ }
+ }
+
+ /**
+ * Accept all filenames ending with {@code .jar}
+ * @return a filename filter
+ */
+ public static FilenameFilter createJarFilter() {
+ return new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.toLowerCase(Locale.ENGLISH).endsWith(".jar");
+ }
+ };
+ }
+
+ /**
+ * Create a file:// path from a local file
+ * @param file file to point the path
+ * @return a new Path
+ */
+ public static Path createLocalPath(File file) {
+ return new Path(file.toURI());
+ }
+
+ /**
+ * Build up the classpath for execution
+ * -behaves very differently on a mini test cluster vs a production
+ * production one.
+ *
+ * @param sliderConfDir relative path to the dir containing slider config
+ * options to put on the classpath -or null
+ * @param libdir directory containing the JAR files
+ * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use
+ * (and hence the current classpath should be used, not anything built up)
+ * @return a classpath
+ */
+ public static ClasspathConstructor buildClasspath(String sliderConfDir,
+ String libdir,
+ SliderFileSystem sliderFileSystem,
+ boolean usingMiniMRCluster) {
+
+ ClasspathConstructor classpath = new ClasspathConstructor();
+ classpath.append(YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
+
+ // add the runtime classpath needed for tests to work
+ if (usingMiniMRCluster) {
+ // for mini cluster we pass down the java CP properties
+ // and nothing else
+ classpath.appendAll(classpath.localJVMClasspath());
+ } else {
+ if (sliderConfDir != null) {
+ classpath.addClassDirectory(sliderConfDir);
+ }
+ classpath.addLibDir(libdir);
+ if (sliderFileSystem.isFile(sliderFileSystem.getDependencyTarGzip())) {
+ classpath.addLibDir(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK);
+ }
+ classpath.addRemoteClasspathEnvVar();
+ classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$());
+ }
+ return classpath;
+ }
+
+ /**
+ * Given a source folder create a tar.gz file
+ *
+ * @param libDirs
+ * @param tarGzipFile
+ *
+ * @throws IOException
+ */
+ public static void tarGzipFolder(String[] libDirs, File tarGzipFile,
+ FilenameFilter filter) throws IOException {
+ log.info("Tar-gzipping folders {} to {}", libDirs,
+ tarGzipFile.getAbsolutePath());
+
+ try(TarArchiveOutputStream taos =
+ new TarArchiveOutputStream(new GZIPOutputStream(
+ new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) {
+ for (String libDir : libDirs) {
+ File srcFolder = new File(libDir);
+ List<String> files = new ArrayList<>();
+ generateFileList(files, srcFolder, srcFolder, true, filter);
+ for (String file : files) {
+ File srcFile = new File(srcFolder, file);
+ TarArchiveEntry tarEntry = new TarArchiveEntry(
+ srcFile, file);
+ taos.putArchiveEntry(tarEntry);
+ try(FileInputStream in = new FileInputStream(srcFile)) {
+ org.apache.commons.io.IOUtils.copy(in, taos);
+ }
+ taos.flush();
+ taos.closeArchiveEntry();
+ }
+ }
+ }
+ }
+
+ private static void generateFileList(List<String> fileList, File node,
+ File rootFolder, Boolean relative, FilenameFilter filter) {
+ if (node.isFile()) {
+ String fileFullPath = node.toString();
+ if (relative) {
+ fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1,
+ fileFullPath.length()));
+ } else {
+ fileList.add(fileFullPath);
+ }
+ }
+
+ if (node.isDirectory()) {
+ String[] subNode = node.list(filter);
+ if (subNode == null || subNode.length == 0) {
+ return;
+ }
+ for (String filename : subNode) {
+ generateFileList(fileList, new File(node, filename), rootFolder,
+ relative, filter);
+ }
+ }
+ }
+
+ public static String createNameTag(String name) {
+ return "Name: " + name;
+ }
+
+ public static String createVersionTag(String version) {
+ return "Version: " + version;
+ }
+
+ public static String createDescriptionTag(String description) {
+ return "Description: " + description;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8133592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
deleted file mode 100644
index 2809dfb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
+++ /dev/null
@@ -1,564 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
-import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.net.ServerSocket;
-import java.net.URL;
-import java.net.URLDecoder;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.regex.Pattern;
-import java.util.zip.GZIPOutputStream;
-
-/**
- * These are slider-specific Util methods
- */
-public final class SliderUtils {
-
- private static final Logger log = LoggerFactory.getLogger(SliderUtils.class);
-
- private SliderUtils() {
- }
-
- /**
- * Implementation of set-ness, groovy definition of true/false for a string
- * @param s string
- * @return true iff the string is neither null nor empty
- */
- public static boolean isUnset(String s) {
- return s == null || s.isEmpty();
- }
-
- public static boolean isSet(String s) {
- return !isUnset(s);
- }
-
- public static boolean isEmpty(Collection l) {
- return l == null || l.isEmpty();
- }
-
- /**
- * Find a containing JAR
- * @param clazz class to find
- * @return the file
- * @throws IOException any IO problem, including the class not having a
- * classloader
- * @throws FileNotFoundException if the class did not resolve to a file
- */
- public static File findContainingJarOrFail(Class clazz) throws IOException {
- File localFile = SliderUtils.findContainingJar(clazz);
- if (null == localFile) {
- throw new FileNotFoundException("Could not find JAR containing " + clazz);
- }
- return localFile;
- }
-
-
- /**
- * Find a containing JAR
- * @param my_class class to find
- * @return the file or null if it is not found
- * @throws IOException any IO problem, including the class not having a
- * classloader
- */
- public static File findContainingJar(Class my_class) throws IOException {
- ClassLoader loader = my_class.getClassLoader();
- if (loader == null) {
- throw new IOException(
- "Class " + my_class + " does not have a classloader!");
- }
- String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
- Enumeration<URL> urlEnumeration = loader.getResources(class_file);
- for (; urlEnumeration.hasMoreElements(); ) {
- URL url = urlEnumeration.nextElement();
- if ("jar".equals(url.getProtocol())) {
- String toReturn = url.getPath();
- if (toReturn.startsWith("file:")) {
- toReturn = toReturn.substring("file:".length());
- }
- // URLDecoder is a misnamed class, since it actually decodes
- // x-www-form-urlencoded MIME type rather than actual
- // URL encoding (which the file path has). Therefore it would
- // decode +s to ' 's which is incorrect (spaces are actually
- // either unencoded or encoded as "%20"). Replace +s first, so
- // that they are kept sacred during the decoding process.
- toReturn = toReturn.replaceAll("\\+", "%2B");
- toReturn = URLDecoder.decode(toReturn, "UTF-8");
- String jarFilePath = toReturn.replaceAll("!.*$", "");
- return new File(jarFilePath);
- } else {
- log.info("could not locate JAR containing {} URL={}", my_class, url);
- }
- }
- return null;
- }
-
- /**
- * Copy a file to a new FS -both paths must be qualified.
- * @param conf conf file
- * @param srcFile src file
- * @param destFile dest file
- */
- @SuppressWarnings("deprecation")
- public static void copy(Configuration conf,
- Path srcFile,
- Path destFile) throws
- IOException,
- BadClusterStateException {
- FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf);
- //list all paths in the src.
- if (!srcFS.exists(srcFile)) {
- throw new FileNotFoundException("Source file not found " + srcFile);
- }
- if (!srcFS.isFile(srcFile)) {
- throw new FileNotFoundException(
- "Source file not a file " + srcFile);
- }
- FileSystem destFS = FileSystem.get(destFile.toUri(), conf);
- FileUtil.copy(srcFS, srcFile, destFS, destFile, false, true, conf);
- }
-
- /**
- * Take a collection, return a list containing the string value of every
- * element in the collection.
- * @param c collection
- * @return a stringified list
- */
- public static List<String> collectionToStringList(Collection c) {
- List<String> l = new ArrayList<>(c.size());
- for (Object o : c) {
- l.add(o.toString());
- }
- return l;
- }
-
- /**
- * Join an collection of objects with a separator that appears after every
- * instance in the list -including at the end
- * @param collection collection to call toString() on each element
- * @param separator separator string
- * @return the joined entries
- */
- public static String join(Collection collection, String separator) {
- return join(collection, separator, true);
- }
-
- /**
- * Join an collection of objects with a separator that appears after every
- * instance in the list -optionally at the end
- * @param collection collection to call toString() on each element
- * @param separator separator string
- * @param trailing add a trailing entry or not
- * @return the joined entries
- */
- public static String join(Collection collection,
- String separator,
- boolean trailing) {
- StringBuilder b = new StringBuilder();
- // fast return on empty collection
- if (collection.isEmpty()) {
- return trailing ? separator : "";
- }
- for (Object o : collection) {
- b.append(o);
- b.append(separator);
- }
- int length = separator.length();
- String s = b.toString();
- return (trailing || s.isEmpty()) ?
- s : (b.substring(0, b.length() - length));
- }
-
- /**
- * Join an array of strings with a separator that appears after every
- * instance in the list -optionally at the end
- * @param collection strings
- * @param separator separator string
- * @param trailing add a trailing entry or not
- * @return the joined entries
- */
- public static String join(String[] collection, String separator,
- boolean trailing) {
- return join(Arrays.asList(collection), separator, trailing);
- }
-
- /**
- * Resolve a mandatory environment variable
- * @param key env var
- * @return the resolved value
- * @throws BadClusterStateException
- */
- public static String mandatoryEnvVariable(String key) throws
- BadClusterStateException {
- String v = System.getenv(key);
- if (v == null) {
- throw new BadClusterStateException("Missing Environment variable " + key);
- }
- return v;
- }
-
- /**
- * Generic map merge logic
- * @param first first map
- * @param second second map
- * @param <T1> key type
- * @param <T2> value type
- * @return 'first' merged with the second
- */
- public static <T1, T2> Map<T1, T2> mergeMapsIgnoreDuplicateKeys(Map<T1, T2> first,
- Map<T1, T2> second) {
- Preconditions.checkArgument(first != null, "Null 'first' value");
- Preconditions.checkArgument(second != null, "Null 'second' value");
- for (Map.Entry<T1, T2> entry : second.entrySet()) {
- T1 key = entry.getKey();
- if (!first.containsKey(key)) {
- first.put(key, entry.getValue());
- }
- }
- return first;
- }
-
- /**
- * Convert a map to a multi-line string for printing
- * @param map map to stringify
- * @return a string representation of the map
- */
- public static String stringifyMap(Map<String, String> map) {
- StringBuilder builder = new StringBuilder();
- for (Map.Entry<String, String> entry : map.entrySet()) {
- builder.append(entry.getKey())
- .append("=\"")
- .append(entry.getValue())
- .append("\"\n");
-
- }
- return builder.toString();
- }
-
- /**
- * Convert a YARN URL into a string value of a normal URL
- * @param url URL
- * @return string representatin
- */
- public static String stringify(org.apache.hadoop.yarn.api.records.URL url) {
- StringBuilder builder = new StringBuilder();
- builder.append(url.getScheme()).append("://");
- if (url.getHost() != null) {
- builder.append(url.getHost()).append(":").append(url.getPort());
- }
- builder.append(url.getFile());
- return builder.toString();
- }
-
- /**
- * Get a random open port
- * @return true if the port was available for listening on
- */
- public static int getOpenPort() throws IOException {
- ServerSocket socket = null;
- try {
- socket = new ServerSocket(0);
- return socket.getLocalPort();
- } finally {
- if (socket != null) {
- socket.close();
- }
- }
- }
-
- /**
- * See if a port is available for listening on by trying to listen
- * on it and seeing if that works or fails.
- * @param port port to listen to
- * @return true if the port was available for listening on
- */
- public static boolean isPortAvailable(int port) {
- try {
- ServerSocket socket = new ServerSocket(port);
- socket.close();
- return true;
- } catch (IOException e) {
- return false;
- }
- }
-
- // Build env map: key -> value;
- // value will be replaced by the corresponding value in tokenMap, if any.
- public static Map<String, String> buildEnvMap(
- org.apache.hadoop.yarn.service.api.records.Configuration conf,
- Map<String,String> tokenMap) {
- if (tokenMap == null) {
- return conf.getEnv();
- }
- Map<String, String> env = new HashMap<>();
- for (Map.Entry<String, String> entry : conf.getEnv().entrySet()) {
- String key = entry.getKey();
- String val = entry.getValue();
- for (Map.Entry<String,String> token : tokenMap.entrySet()) {
- val = val.replaceAll(Pattern.quote(token.getKey()),
- token.getValue());
- }
- env.put(key,val);
- }
- return env;
- }
-
- public static String[] getLibDirs() {
- String libDirStr = System.getProperty(YarnServiceConstants.PROPERTY_LIB_DIR);
- if (isUnset(libDirStr)) {
- return ArrayUtils.EMPTY_STRING_ARRAY;
- }
- return StringUtils.split(libDirStr, ',');
- }
-
- /**
- * Submit a JAR containing a specific class and map it
- * @param providerResources provider map to build up
- * @param sliderFileSystem remote fs
- * @param clazz class to look for
- * @param libdir lib directory
- * @param jarName <i>At the destination</i>
- * @return the local resource ref
- * @throws IOException trouble copying to HDFS
- */
- public static LocalResource putJar(Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem,
- Class clazz,
- Path tempPath,
- String libdir,
- String jarName
- )
- throws IOException, SliderException {
- LocalResource res = sliderFileSystem.submitJarWithClass(
- clazz,
- tempPath,
- libdir,
- jarName);
- providerResources.put(libdir + "/" + jarName, res);
- return res;
- }
-
- /**
- * Submit a JAR containing and map it
- * @param providerResources provider map to build up
- * @param sliderFileSystem remote fs
- * @param libDir lib directory
- * @param srcPath copy jars from
- */
- public static void putAllJars(Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem,
- Path tempPath,
- String libDir,
- String srcPath) throws IOException, SliderException {
- log.info("Loading all dependencies from {}", srcPath);
- if (SliderUtils.isSet(srcPath)) {
- File srcFolder = new File(srcPath);
- FilenameFilter jarFilter = createJarFilter();
- File[] listOfJars = srcFolder.listFiles(jarFilter);
- if (listOfJars == null || listOfJars.length == 0) {
- return;
- }
- for (File jarFile : listOfJars) {
- LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName());
- providerResources.put(libDir + "/" + jarFile.getName(), res);
- }
- }
- }
-
- /**
- * Accept all filenames ending with {@code .jar}
- * @return a filename filter
- */
- public static FilenameFilter createJarFilter() {
- return new FilenameFilter() {
- public boolean accept(File dir, String name) {
- return name.toLowerCase(Locale.ENGLISH).endsWith(".jar");
- }
- };
- }
-
- /**
- * Submit the AM tar.gz containing all dependencies and map it
- * @param providerResources provider map to build up
- * @param sliderFileSystem remote fs
- */
- public static void putAmTarGzipAndUpdate(
- Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem
- ) throws IOException, SliderException {
- log.info("Loading all dependencies from {}{}",
- YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME,
- YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
- sliderFileSystem.submitTarGzipAndUpdate(providerResources);
- }
-
- /**
- * Create a file:// path from a local file
- * @param file file to point the path
- * @return a new Path
- */
- public static Path createLocalPath(File file) {
- return new Path(file.toURI());
- }
-
- /**
- * Build up the classpath for execution
- * -behaves very differently on a mini test cluster vs a production
- * production one.
- *
- * @param sliderConfDir relative path to the dir containing slider config
- * options to put on the classpath -or null
- * @param libdir directory containing the JAR files
- * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use
- * (and hence the current classpath should be used, not anything built up)
- * @return a classpath
- */
- public static ClasspathConstructor buildClasspath(String sliderConfDir,
- String libdir,
- SliderFileSystem sliderFileSystem,
- boolean usingMiniMRCluster) {
-
- ClasspathConstructor classpath = new ClasspathConstructor();
- classpath.append(YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
-
- // add the runtime classpath needed for tests to work
- if (usingMiniMRCluster) {
- // for mini cluster we pass down the java CP properties
- // and nothing else
- classpath.appendAll(classpath.localJVMClasspath());
- } else {
- if (sliderConfDir != null) {
- classpath.addClassDirectory(sliderConfDir);
- }
- classpath.addLibDir(libdir);
- if (sliderFileSystem.isFile(sliderFileSystem.getDependencyTarGzip())) {
- classpath.addLibDir(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK);
- } else {
- log.info(
- "For faster submission of apps, upload dependencies using cmd " +
- "enableFastLaunch");
- }
- classpath.addRemoteClasspathEnvVar();
- classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$());
- }
- return classpath;
- }
-
- /**
- * Given a source folder create a tar.gz file
- *
- * @param libDirs
- * @param tarGzipFile
- *
- * @throws IOException
- */
- public static void tarGzipFolder(String[] libDirs, File tarGzipFile,
- FilenameFilter filter) throws IOException {
- log.info("Tar-gzipping folders {} to {}", libDirs,
- tarGzipFile.getAbsolutePath());
-
- try(TarArchiveOutputStream taos =
- new TarArchiveOutputStream(new GZIPOutputStream(
- new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) {
- for (String libDir : libDirs) {
- File srcFolder = new File(libDir);
- List<String> files = new ArrayList<>();
- generateFileList(files, srcFolder, srcFolder, true, filter);
- for (String file : files) {
- File srcFile = new File(srcFolder, file);
- TarArchiveEntry tarEntry = new TarArchiveEntry(
- srcFile, file);
- taos.putArchiveEntry(tarEntry);
- try(FileInputStream in = new FileInputStream(srcFile)) {
- org.apache.commons.io.IOUtils.copy(in, taos);
- }
- taos.flush();
- taos.closeArchiveEntry();
- }
- }
- }
- }
-
- private static void generateFileList(List<String> fileList, File node,
- File rootFolder, Boolean relative, FilenameFilter filter) {
- if (node.isFile()) {
- String fileFullPath = node.toString();
- if (relative) {
- fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1,
- fileFullPath.length()));
- } else {
- fileList.add(fileFullPath);
- }
- }
-
- if (node.isDirectory()) {
- String[] subNode = node.list(filter);
- if (subNode == null || subNode.length == 0) {
- return;
- }
- for (String filename : subNode) {
- generateFileList(fileList, new File(node, filename), rootFolder,
- relative, filter);
- }
- }
- }
-
- public static String createNameTag(String name) {
- return "Name: " + name;
- }
-
- public static String createVersionTag(String version) {
- return "Version: " + version;
- }
-
- public static String createDescriptionTag(String description) {
- return "Description: " + description;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8133592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
index 1fa07ce..c6e8525 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
@@ -32,7 +32,7 @@ public class ZookeeperUtils {
String zkPort = Integer.toString(port);
//parse the hosts
String[] hostlist = zkHosts.split(",", 0);
- String quorum = SliderUtils.join(hostlist, ":" + zkPort + ",", false);
+ String quorum = ServiceUtils.join(hostlist, ":" + zkPort + ",", false);
return quorum;
}
@@ -117,7 +117,7 @@ public class ZookeeperUtils {
for (HostAndPort hostAndPort : hostAndPorts) {
entries.add(buildQuorumEntry(hostAndPort, defaultPort));
}
- return SliderUtils.join(entries, ",", false);
+ return ServiceUtils.join(entries, ",", false);
}
public static String convertToHostsOnlyList(String quorum) throws
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8133592/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
index 6159215..d7fa9a04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.service.conf;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
import org.junit.Assert;
import org.junit.Test;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org