You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/09/12 00:35:37 UTC
[35/84] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d44876af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
new file mode 100644
index 0000000..415392a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
@@ -0,0 +1,1699 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.GlobFilter;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.client.params.Arguments;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.zookeeper.server.util.KerberosUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPOutputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+/**
+ * These are slider-specific Util methods
+ */
+public final class SliderUtils {
+
+ private static final Logger log = LoggerFactory.getLogger(SliderUtils.class);
+
+ /**
+ * Atomic bool to track whether or not process security has already been
+ * turned on (prevents re-entrancy)
+ */
+ private static final AtomicBoolean processSecurityAlreadyInitialized =
+ new AtomicBoolean(false);
+ public static final String JAVA_SECURITY_KRB5_REALM =
+ "java.security.krb5.realm";
+ public static final String JAVA_SECURITY_KRB5_KDC = "java.security.krb5.kdc";
+
+ /**
+ * Winutils
+ */
+ public static final String WINUTILS = "WINUTILS.EXE";
+ /**
+ * name of openssl program
+ */
+ public static final String OPENSSL = "openssl";
+
+ /**
+ * name of python program
+ */
+ public static final String PYTHON = "python";
+
+ /**
+ * type of docker standalone application
+ */
+ public static final String DOCKER = "docker";
+ /**
+ * type of docker on yarn application
+ */
+ public static final String DOCKER_YARN = "yarn_docker";
+
+ public static final int NODE_LIST_LIMIT = 10;
+
+ private SliderUtils() {
+ }
+
+ /**
+ * Implementation of set-ness, groovy definition of true/false for a string
+ * @param s string
+ * @return true iff the string is neither null nor empty
+ */
+ public static boolean isUnset(String s) {
+ return s == null || s.isEmpty();
+ }
+
+ public static boolean isSet(String s) {
+ return !isUnset(s);
+ }
+
+ public static boolean isEmpty(Collection l) {
+ return l == null || l.isEmpty();
+ }
+
+ /**
+ * Probe for a collection existing and not being empty
+ * @param l collection
+ * @return true if the reference is valid and it contains entries
+ */
+
+ public static boolean isNotEmpty(Collection l) {
+ return l != null && !l.isEmpty();
+ }
+
+ /**
+ * Probe for a map existing and not being empty
+ * @param m map
+ * @return true if the reference is valid and it contains map entries
+ */
+ public static boolean isNotEmpty(Map m) {
+ return m != null && !m.isEmpty();
+ }
+
+ /*
+ * Validates whether num is an integer
+ * @param num
+ * @param msg the message to be shown in exception
+ */
+ @SuppressWarnings("ResultOfMethodCallIgnored")
+ private static void validateNumber(String num, String msg) throws
+ BadConfigException {
+ try {
+ Integer.parseInt(num);
+ } catch (NumberFormatException nfe) {
+ throw new BadConfigException(msg + num);
+ }
+ }
+
+ /*
+ * Translates the trailing JVM heapsize unit: g, G, m, M
+ * This assumes designated unit of 'm'
+ * @param heapsize
+ * @return heapsize in MB
+ */
+ public static String translateTrailingHeapUnit(String heapsize) throws
+ BadConfigException {
+ String errMsg = "Bad heapsize: ";
+ if (heapsize.endsWith("m") || heapsize.endsWith("M")) {
+ String num = heapsize.substring(0, heapsize.length() - 1);
+ validateNumber(num, errMsg);
+ return num;
+ }
+ if (heapsize.endsWith("g") || heapsize.endsWith("G")) {
+ String num = heapsize.substring(0, heapsize.length() - 1) + "000";
+ validateNumber(num, errMsg);
+ return num;
+ }
+ // check if specified heap size is a number
+ validateNumber(heapsize, errMsg);
+ return heapsize;
+ }
+
+ /**
+ * recursive directory delete
+ * @param dir dir to delete
+ * @throws IOException on any problem
+ */
+ public static void deleteDirectoryTree(File dir) throws IOException {
+ if (dir.exists()) {
+ if (dir.isDirectory()) {
+ log.info("Cleaning up {}", dir);
+ //delete the children
+ File[] files = dir.listFiles();
+ if (files == null) {
+ throw new IOException("listfiles() failed for " + dir);
+ }
+ for (File file : files) {
+ log.info("deleting {}", file);
+ if (!file.delete()) {
+ log.warn("Unable to delete " + file);
+ }
+ }
+ if (!dir.delete()) {
+ log.warn("Unable to delete " + dir);
+ }
+ } else {
+ throw new IOException("Not a directory " + dir);
+ }
+ } else {
+ //not found, do nothing
+ log.debug("No output dir yet");
+ }
+ }
+
+ /**
+ * Find a containing JAR
+ * @param clazz class to find
+ * @return the file
+ * @throws IOException any IO problem, including the class not having a
+ * classloader
+ * @throws FileNotFoundException if the class did not resolve to a file
+ */
+ public static File findContainingJarOrFail(Class clazz) throws IOException {
+ File localFile = SliderUtils.findContainingJar(clazz);
+ if (null == localFile) {
+ throw new FileNotFoundException("Could not find JAR containing " + clazz);
+ }
+ return localFile;
+ }
+
+
+ /**
+ * Find a containing JAR
+ * @param my_class class to find
+ * @return the file or null if it is not found
+ * @throws IOException any IO problem, including the class not having a
+ * classloader
+ */
+ public static File findContainingJar(Class my_class) throws IOException {
+ ClassLoader loader = my_class.getClassLoader();
+ if (loader == null) {
+ throw new IOException(
+ "Class " + my_class + " does not have a classloader!");
+ }
+ String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
+ Enumeration<URL> urlEnumeration = loader.getResources(class_file);
+ for (; urlEnumeration.hasMoreElements(); ) {
+ URL url = urlEnumeration.nextElement();
+ if ("jar".equals(url.getProtocol())) {
+ String toReturn = url.getPath();
+ if (toReturn.startsWith("file:")) {
+ toReturn = toReturn.substring("file:".length());
+ }
+ // URLDecoder is a misnamed class, since it actually decodes
+ // x-www-form-urlencoded MIME type rather than actual
+ // URL encoding (which the file path has). Therefore it would
+ // decode +s to ' 's which is incorrect (spaces are actually
+ // either unencoded or encoded as "%20"). Replace +s first, so
+ // that they are kept sacred during the decoding process.
+ toReturn = toReturn.replaceAll("\\+", "%2B");
+ toReturn = URLDecoder.decode(toReturn, "UTF-8");
+ String jarFilePath = toReturn.replaceAll("!.*$", "");
+ return new File(jarFilePath);
+ } else {
+ log.info("could not locate JAR containing {} URL={}", my_class, url);
+ }
+ }
+ return null;
+ }
+
+ public static void checkPort(String hostname, int port, int connectTimeout)
+ throws IOException {
+ InetSocketAddress addr = new InetSocketAddress(hostname, port);
+ checkPort(hostname, addr, connectTimeout);
+ }
+
+ @SuppressWarnings("SocketOpenedButNotSafelyClosed")
+ public static void checkPort(String name,
+ InetSocketAddress address,
+ int connectTimeout)
+ throws IOException {
+ try(Socket socket = new Socket()) {
+ socket.connect(address, connectTimeout);
+ } catch (Exception e) {
+ throw new IOException("Failed to connect to " + name
+ + " at " + address
+ + " after " + connectTimeout + "milliseconds"
+ + ": " + e,
+ e);
+ }
+ }
+
+ public static void checkURL(String name, String url, int timeout) throws
+ IOException {
+ InetSocketAddress address = NetUtils.createSocketAddr(url);
+ checkPort(name, address, timeout);
+ }
+
+ /**
+ * A required file
+ * @param role role of the file (for errors)
+ * @param filename the filename
+ * @throws ExitUtil.ExitException if the file is missing
+ * @return the file
+ */
+ public static File requiredFile(String filename, String role) throws
+ IOException {
+ if (filename.isEmpty()) {
+ throw new ExitUtil.ExitException(-1, role + " file not defined");
+ }
+ File file = new File(filename);
+ if (!file.exists()) {
+ throw new ExitUtil.ExitException(-1,
+ role + " file not found: " +
+ file.getCanonicalPath());
+ }
+ return file;
+ }
+
+ private static final PatternValidator clusternamePattern
+ = new PatternValidator("[a-z][a-z0-9_-]*");
+
+ private static final PatternValidator compNamePattern
+ = new PatternValidator("[a-z][a-z0-9-]*");
+
+ public static void validateCompName(String compName) {
+ compNamePattern.validate(compName);
+ }
+
+ /**
+ * Normalize a cluster name then verify that it is valid
+ * @param name proposed cluster name
+ * @return true iff it is valid
+ */
+ public static boolean isClusternameValid(String name) {
+ return name != null && clusternamePattern.matches(name);
+ }
+
+ public static boolean oldIsClusternameValid(String name) {
+ if (name == null || name.isEmpty()) {
+ return false;
+ }
+ int first = name.charAt(0);
+ if (0 == (Character.getType(first) & Character.LOWERCASE_LETTER)) {
+ return false;
+ }
+
+ for (int i = 0; i < name.length(); i++) {
+ int elt = (int) name.charAt(i);
+ int t = Character.getType(elt);
+ if (0 == (t & Character.LOWERCASE_LETTER)
+ && 0 == (t & Character.DECIMAL_DIGIT_NUMBER)
+ && elt != '-'
+ && elt != '_') {
+ return false;
+ }
+ if (!Character.isLetterOrDigit(elt) && elt != '-' && elt != '_') {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Copy a directory to a new FS -both paths must be qualified. If
+ * a directory needs to be created, supplied permissions can override
+ * the default values. Existing directories are not touched
+ * @param conf conf file
+ * @param srcDirPath src dir
+ * @param destDirPath dest dir
+ * @param permission permission for the dest directory; null means "default"
+ * @return # of files copies
+ */
+ @SuppressWarnings("deprecation")
+ public static int copyDirectory(Configuration conf,
+ Path srcDirPath,
+ Path destDirPath,
+ FsPermission permission) throws
+ IOException,
+ BadClusterStateException {
+ FileSystem srcFS = FileSystem.get(srcDirPath.toUri(), conf);
+ FileSystem destFS = FileSystem.get(destDirPath.toUri(), conf);
+ //list all paths in the src.
+ if (!srcFS.exists(srcDirPath)) {
+ throw new FileNotFoundException("Source dir not found " + srcDirPath);
+ }
+ if (!srcFS.isDirectory(srcDirPath)) {
+ throw new FileNotFoundException(
+ "Source dir not a directory " + srcDirPath);
+ }
+ GlobFilter dotFilter = new GlobFilter("[!.]*");
+ FileStatus[] entries = srcFS.listStatus(srcDirPath, dotFilter);
+ int srcFileCount = entries.length;
+ if (srcFileCount == 0) {
+ return 0;
+ }
+ if (permission == null) {
+ permission = FsPermission.getDirDefault();
+ }
+ if (!destFS.exists(destDirPath)) {
+ new SliderFileSystem(destFS, conf).createWithPermissions(destDirPath,
+ permission);
+ }
+ Path[] sourcePaths = new Path[srcFileCount];
+ for (int i = 0; i < srcFileCount; i++) {
+ FileStatus e = entries[i];
+ Path srcFile = e.getPath();
+ if (srcFS.isDirectory(srcFile)) {
+ String msg = "Configuration dir " + srcDirPath
+ + " contains a directory " + srcFile;
+ log.warn(msg);
+ throw new IOException(msg);
+ }
+ log.debug("copying src conf file {}", srcFile);
+ sourcePaths[i] = srcFile;
+ }
+ log.debug("Copying {} files from {} to dest {}", srcFileCount,
+ srcDirPath,
+ destDirPath);
+ FileUtil.copy(srcFS, sourcePaths, destFS, destDirPath, false, true, conf);
+ return srcFileCount;
+ }
+
+ /**
+ * Copy a file to a new FS -both paths must be qualified.
+ * @param conf conf file
+ * @param srcFile src file
+ * @param destFile dest file
+ */
+ @SuppressWarnings("deprecation")
+ public static void copy(Configuration conf,
+ Path srcFile,
+ Path destFile) throws
+ IOException,
+ BadClusterStateException {
+ FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf);
+ //list all paths in the src.
+ if (!srcFS.exists(srcFile)) {
+ throw new FileNotFoundException("Source file not found " + srcFile);
+ }
+ if (!srcFS.isFile(srcFile)) {
+ throw new FileNotFoundException(
+ "Source file not a file " + srcFile);
+ }
+ FileSystem destFS = FileSystem.get(destFile.toUri(), conf);
+ FileUtil.copy(srcFS, srcFile, destFS, destFile, false, true, conf);
+ }
+
+ /**
+ * Take a collection, return a list containing the string value of every
+ * element in the collection.
+ * @param c collection
+ * @return a stringified list
+ */
+ public static List<String> collectionToStringList(Collection c) {
+ List<String> l = new ArrayList<>(c.size());
+ for (Object o : c) {
+ l.add(o.toString());
+ }
+ return l;
+ }
+
+ /**
+ * Join an collection of objects with a separator that appears after every
+ * instance in the list -including at the end
+ * @param collection collection to call toString() on each element
+ * @param separator separator string
+ * @return the joined entries
+ */
+ public static String join(Collection collection, String separator) {
+ return join(collection, separator, true);
+ }
+
+ /**
+ * Join an collection of objects with a separator that appears after every
+ * instance in the list -optionally at the end
+ * @param collection collection to call toString() on each element
+ * @param separator separator string
+ * @param trailing add a trailing entry or not
+ * @return the joined entries
+ */
+ public static String join(Collection collection,
+ String separator,
+ boolean trailing) {
+ StringBuilder b = new StringBuilder();
+ // fast return on empty collection
+ if (collection.isEmpty()) {
+ return trailing ? separator : "";
+ }
+ for (Object o : collection) {
+ b.append(o);
+ b.append(separator);
+ }
+ int length = separator.length();
+ String s = b.toString();
+ return (trailing || s.isEmpty()) ?
+ s : (b.substring(0, b.length() - length));
+ }
+
+ /**
+ * Join an array of strings with a separator that appears after every
+ * instance in the list -including at the end
+ * @param collection strings
+ * @param separator separator string
+ * @return the joined entries
+ */
+ public static String join(String[] collection, String separator) {
+ return join(collection, separator, true);
+
+
+ }
+
+ /**
+ * Join an array of strings with a separator that appears after every
+ * instance in the list -optionally at the end
+ * @param collection strings
+ * @param separator separator string
+ * @param trailing add a trailing entry or not
+ * @return the joined entries
+ */
+ public static String join(String[] collection, String separator,
+ boolean trailing) {
+ return join(Arrays.asList(collection), separator, trailing);
+ }
+
+ /**
+ * Join an array of strings with a separator that appears after every
+ * instance in the list -except at the end
+ * @param collection strings
+ * @param separator separator string
+ * @return the list
+ */
+ public static String joinWithInnerSeparator(String separator,
+ Object... collection) {
+ StringBuilder b = new StringBuilder();
+ boolean first = true;
+
+ for (Object o : collection) {
+ if (first) {
+ first = false;
+ } else {
+ b.append(separator);
+ }
+ b.append(o.toString());
+ b.append(separator);
+ }
+ return b.toString();
+ }
+
+ /**
+ * Resolve a mandatory environment variable
+ * @param key env var
+ * @return the resolved value
+ * @throws BadClusterStateException
+ */
+ public static String mandatoryEnvVariable(String key) throws
+ BadClusterStateException {
+ String v = System.getenv(key);
+ if (v == null) {
+ throw new BadClusterStateException("Missing Environment variable " + key);
+ }
+ return v;
+ }
+
+ public static String appReportToString(ApplicationReport r,
+ String separator) {
+ StringBuilder builder = new StringBuilder(512);
+ builder.append("application ")
+ .append(
+ r.getName())
+ .append("/")
+ .append(r.getApplicationType())
+ .append(separator);
+ Set<String> tags = r.getApplicationTags();
+ if (!tags.isEmpty()) {
+ for (String tag : tags) {
+ builder.append(tag).append(separator);
+ }
+ }
+ DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm");
+ dateFormat.setTimeZone(TimeZone.getDefault());
+ builder.append("state: ").append(r.getYarnApplicationState());
+ String trackingUrl = r.getTrackingUrl();
+ if (isSet(trackingUrl)) {
+ builder.append(separator).append("URL: ").append(trackingUrl);
+ }
+ builder.append(separator)
+ .append("Started: ")
+ .append(dateFormat.format(new Date(r.getStartTime())));
+ long finishTime = r.getFinishTime();
+ if (finishTime > 0) {
+ builder.append(separator)
+ .append("Finished: ")
+ .append(dateFormat.format(new Date(finishTime)));
+ }
+ String rpcHost = r.getHost();
+ if (!isSet(rpcHost)) {
+ builder.append(separator)
+ .append("RPC :")
+ .append(rpcHost)
+ .append(':')
+ .append(r.getRpcPort());
+ }
+ String diagnostics = r.getDiagnostics();
+ if (!isSet(diagnostics)) {
+ builder.append(separator).append("Diagnostics :").append(diagnostics);
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Filter a string value given a single filter
+ *
+ * @param value
+ * the string value to check
+ * @param filter
+ * a single string filter
+ * @return return true if value should be trapped, false if it should be let
+ * through
+ */
+ public static boolean filter(String value, String filter) {
+ return !(StringUtils.isEmpty(filter) || filter.equals(value));
+ }
+
+ /**
+ * Filter a string value given a set of filters
+ *
+ * @param value
+ * the string value to check
+ * @param filters
+ * a set of string filters
+ * @return return true if value should be trapped, false if it should be let
+ * through
+ */
+ public static boolean filter(String value, Set<String> filters) {
+ return !(filters.isEmpty() || filters.contains(value));
+ }
+
+ /**
+ * Sorts the given list of application reports, most recently started
+ * or finished instance first.
+ *
+ * @param instances list of instances
+ */
+ public static void sortApplicationsByMostRecent(List<ApplicationReport> instances) {
+ Collections.sort(instances, new MostRecentlyStartedOrFinishedFirst());
+ }
+
+ /**
+ * Sorts the given list of application reports
+ * Finished instances are ordered by finished time and running/accepted instances are
+ * ordered by start time
+ * Finally Instance are order by finished instances coming after running instances
+ *
+ * @param instances list of instances
+ */
+ public static void sortApplicationReport(List<ApplicationReport> instances) {
+ if (instances.size() <= 1) {
+ return;
+ }
+ List<ApplicationReport> nonLiveInstance =
+ new ArrayList<>(instances.size());
+ List<ApplicationReport> liveInstance =
+ new ArrayList<>(instances.size());
+
+ for (ApplicationReport report : instances) {
+ if (report.getYarnApplicationState() == YarnApplicationState.RUNNING
+ ||
+ report.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
+ liveInstance.add(report);
+ } else {
+ nonLiveInstance.add(report);
+ }
+ }
+
+ if (liveInstance.size() > 1) {
+ Collections.sort(liveInstance, new MostRecentlyStartedAppFirst());
+ }
+ if (nonLiveInstance.size() > 1) {
+ Collections.sort(nonLiveInstance, new MostRecentAppFinishFirst());
+ }
+ instances.clear();
+ instances.addAll(liveInstance);
+ instances.addAll(nonLiveInstance);
+ }
+
+ /**
+ * Merge in one map to another -all entries in the second map are
+ * merged into the first -overwriting any duplicate keys.
+ * @param first first map -the updated one.
+ * @param second the map that is merged in
+ * @return the first map
+ */
+ public static Map<String, String> mergeMap(Map<String, String> first,
+ Map<String, String> second) {
+ first.putAll(second);
+ return first;
+ }
+
+ /**
+ * Merge a set of entries into a map. This will take the entryset of
+ * a map, or a Hadoop collection itself
+ * @param dest destination
+ * @param entries entries
+ * @return dest -with the entries merged in
+ */
+ public static Map<String, String> mergeEntries(Map<String, String> dest,
+ Iterable<Map.Entry<String, String>> entries) {
+ for (Map.Entry<String, String> entry : entries) {
+ dest.put(entry.getKey(), entry.getValue());
+ }
+ return dest;
+ }
+
+ /**
+ * Generic map merge logic
+ * @param first first map
+ * @param second second map
+ * @param <T1> key type
+ * @param <T2> value type
+ * @return 'first' merged with the second
+ */
+ public static <T1, T2> Map<T1, T2> mergeMaps(Map<T1, T2> first,
+ Map<T1, T2> second) {
+ first.putAll(second);
+ return first;
+ }
+
+ /**
+ * Generic map merge logic
+ * @param first first map
+ * @param second second map
+ * @param <T1> key type
+ * @param <T2> value type
+ * @return 'first' merged with the second
+ */
+ public static <T1, T2> Map<T1, T2> mergeMapsIgnoreDuplicateKeys(Map<T1, T2> first,
+ Map<T1, T2> second) {
+ Preconditions.checkArgument(first != null, "Null 'first' value");
+ Preconditions.checkArgument(second != null, "Null 'second' value");
+ for (Map.Entry<T1, T2> entry : second.entrySet()) {
+ T1 key = entry.getKey();
+ if (!first.containsKey(key)) {
+ first.put(key, entry.getValue());
+ }
+ }
+ return first;
+ }
+
+ /**
+ * Convert a map to a multi-line string for printing
+ * @param map map to stringify
+ * @return a string representation of the map
+ */
+ public static String stringifyMap(Map<String, String> map) {
+ StringBuilder builder = new StringBuilder();
+ for (Map.Entry<String, String> entry : map.entrySet()) {
+ builder.append(entry.getKey())
+ .append("=\"")
+ .append(entry.getValue())
+ .append("\"\n");
+
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Parse an int value, replacing it with defval if undefined;
+ * @param errorKey key to use in exceptions
+ * @param defVal default value to use if the key is not in the map
+ * @param min min value or -1 for do not check
+ * @param max max value or -1 for do not check
+ * @return the int value the integer value
+ * @throws BadConfigException if the value could not be parsed
+ */
+ public static int parseAndValidate(String errorKey,
+ String valS,
+ int defVal,
+ int min, int max) throws
+ BadConfigException {
+ if (valS == null) {
+ valS = Integer.toString(defVal);
+ }
+ String trim = valS.trim();
+ int val;
+ try {
+ val = Integer.decode(trim);
+ } catch (NumberFormatException e) {
+ throw new BadConfigException("Failed to parse value of "
+ + errorKey + ": \"" + trim + "\"");
+ }
+ if (min >= 0 && val < min) {
+ throw new BadConfigException("Value of "
+ + errorKey + ": " + val + ""
+ + "is less than the minimum of " + min);
+ }
+ if (max >= 0 && val > max) {
+ throw new BadConfigException("Value of "
+ + errorKey + ": " + val + ""
+ + "is more than the maximum of " + max);
+ }
+ return val;
+ }
+
+ public static InetSocketAddress getRmAddress(Configuration conf) {
+ return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_PORT);
+ }
+
+ public static InetSocketAddress getRmSchedulerAddress(Configuration conf) {
+ return conf.getSocketAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
+ }
+
+ /**
+ * probe to see if the RM scheduler is defined
+ * @param conf config
+ * @return true if the RM scheduler address is set to
+ * something other than 0.0.0.0
+ */
+ public static boolean isRmSchedulerAddressDefined(Configuration conf) {
+ InetSocketAddress address = getRmSchedulerAddress(conf);
+ return isAddressDefined(address);
+ }
+
+ /**
+ * probe to see if the address
+ * @param address network address
+ * @return true if the scheduler address is set to
+ * something other than 0.0.0.0
+ */
+ public static boolean isAddressDefined(InetSocketAddress address) {
+ if (address == null || address.getHostString() == null) {
+ return false;
+ }
+ return !(address.getHostString().equals("0.0.0.0"));
+ }
+
+ public static void setRmAddress(Configuration conf, String rmAddr) {
+ conf.set(YarnConfiguration.RM_ADDRESS, rmAddr);
+ }
+
+ public static void setRmSchedulerAddress(Configuration conf, String rmAddr) {
+ conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rmAddr);
+ }
+
+ public static boolean hasAppFinished(ApplicationReport report) {
+ return report == null ||
+ report.getYarnApplicationState().ordinal() >=
+ YarnApplicationState.FINISHED.ordinal();
+ }
+
+ public static String containerToString(Container container) {
+ if (container == null) {
+ return "null container";
+ }
+ return String.format(Locale.ENGLISH,
+ "ContainerID=%s nodeID=%s http=%s priority=%s resource=%s",
+ container.getId(),
+ container.getNodeId(),
+ container.getNodeHttpAddress(),
+ container.getPriority(),
+ container.getResource());
+ }
+
+ /**
+ * convert an AM report to a string for diagnostics
+ * @param report the report
+ * @return the string value
+ */
+ public static String reportToString(ApplicationReport report) {
+ if (report == null) {
+ return "Null application report";
+ }
+
+ return "App " + report.getName() + "/" + report.getApplicationType() +
+ "# " +
+ report.getApplicationId() + " user " + report.getUser() +
+ " is in state " + report.getYarnApplicationState() +
+ " RPC: " + report.getHost() + ":" + report.getRpcPort() +
+ " URL: " + report.getOriginalTrackingUrl();
+ }
+
+ /**
+ * Convert a YARN URL into a string value of a normal URL
+ * @param url URL
+ * @return string representatin
+ */
+ public static String stringify(org.apache.hadoop.yarn.api.records.URL url) {
+ StringBuilder builder = new StringBuilder();
+ builder.append(url.getScheme()).append("://");
+ if (url.getHost() != null) {
+ builder.append(url.getHost()).append(":").append(url.getPort());
+ }
+ builder.append(url.getFile());
+ return builder.toString();
+ }
+
+ /**
+ * Get a random open port
+ * @return true if the port was available for listening on
+ */
+ public static int getOpenPort() throws IOException {
+ ServerSocket socket = null;
+ try {
+ socket = new ServerSocket(0);
+ return socket.getLocalPort();
+ } finally {
+ if (socket != null) {
+ socket.close();
+ }
+ }
+ }
+
+ /**
+ * See if a port is available for listening on by trying to listen
+ * on it and seeing if that works or fails.
+ * @param port port to listen to
+ * @return true if the port was available for listening on
+ */
+ public static boolean isPortAvailable(int port) {
+ try {
+ ServerSocket socket = new ServerSocket(port);
+ socket.close();
+ return true;
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ // Build env map: key -> value;
+ // value will be replaced by the corresponding value in tokenMap, if any.
+ public static Map<String, String> buildEnvMap(
+ org.apache.hadoop.yarn.service.api.records.Configuration conf,
+ Map<String,String> tokenMap) {
+ if (tokenMap == null) {
+ return conf.getEnv();
+ }
+ Map<String, String> env = new HashMap<>();
+ for (Map.Entry<String, String> entry : conf.getEnv().entrySet()) {
+ String key = entry.getKey();
+ String val = entry.getValue();
+ for (Map.Entry<String,String> token : tokenMap.entrySet()) {
+ val = val.replaceAll(Pattern.quote(token.getKey()),
+ token.getValue());
+ }
+ env.put(key,val);
+ }
+ return env;
+ }
+
+ /**
+ * Apply a set of command line options to a cluster role map
+ * @param clusterRoleMap cluster role map to merge onto
+ * @param commandOptions command opts
+ */
+ public static void applyCommandLineRoleOptsToRoleMap(
+ Map<String, Map<String, String>> clusterRoleMap,
+ Map<String, Map<String, String>> commandOptions) {
+ for (Map.Entry<String, Map<String, String>> entry : commandOptions.entrySet()) {
+ String key = entry.getKey();
+ Map<String, String> optionMap = entry.getValue();
+ Map<String, String> existingMap = clusterRoleMap.get(key);
+ if (existingMap == null) {
+ existingMap = new HashMap<String, String>();
+ }
+ log.debug("Overwriting role options with command line values {}",
+ stringifyMap(optionMap));
+ mergeMap(existingMap, optionMap);
+ //set or overwrite the role
+ clusterRoleMap.put(key, existingMap);
+ }
+ }
+
+ /**
+ * verify that the supplied cluster name is valid
+ * @param clustername cluster name
+ * @throws BadCommandArgumentsException if it is invalid
+ */
+ public static void validateClusterName(String clustername)
+ throws BadCommandArgumentsException {
+ if (!isClusternameValid(clustername)) {
+ throw new BadCommandArgumentsException(
+ "Illegal cluster name: " + clustername);
+ }
+ }
+
+ /**
+ * Verify that a Kerberos principal has been set -if not fail
+ * with an error message that actually tells you what is missing
+ * @param conf configuration to look at
+ * @param principal key of principal
+ * @throws BadConfigException if the key is not set
+ */
+ public static void verifyPrincipalSet(Configuration conf,
+ String principal) throws
+ BadConfigException {
+ String principalName = conf.get(principal);
+ if (principalName == null) {
+ throw new BadConfigException("Unset Kerberos principal : %s",
+ principal);
+ }
+ log.debug("Kerberos princial {}={}", principal, principalName);
+ }
+
+ /**
+ * Flag to indicate whether the cluster is in secure mode
+ * @param conf configuration to look at
+ * @return true if the slider client/service should be in secure mode
+ */
+ public static boolean isHadoopClusterSecure(Configuration conf) {
+ return SecurityUtil.getAuthenticationMethod(conf) !=
+ UserGroupInformation.AuthenticationMethod.SIMPLE;
+ }
+
+ /**
+ * Init security if the cluster configuration declares the cluster is secure
+ * @param conf configuration to look at
+ * @return true if the cluster is secure
+ * @throws IOException cluster is secure
+ * @throws SliderException the configuration/process is invalid
+ */
+ public static boolean maybeInitSecurity(Configuration conf) throws
+ IOException,
+ SliderException {
+ boolean clusterSecure = isHadoopClusterSecure(conf);
+ if (clusterSecure) {
+ log.debug("Enabling security");
+ initProcessSecurity(conf);
+ }
+ return clusterSecure;
+ }
+
+ /**
+ * Turn on security. This is setup to only run once.
+ * @param conf configuration to build up security
+ * @return true if security was initialized in this call
+ * @throws IOException IO/Net problems
+ * @throws BadConfigException the configuration and system state are inconsistent
+ */
+ public static boolean initProcessSecurity(Configuration conf) throws
+ IOException,
+ SliderException {
+
+ if (processSecurityAlreadyInitialized.compareAndSet(true, true)) {
+ //security is already inited
+ return false;
+ }
+
+ log.info("JVM initialized into secure mode with kerberos realm {}",
+ SliderUtils.getKerberosRealm());
+ //this gets UGI to reset its previous world view (i.e simple auth)
+ //security
+ log.debug("java.security.krb5.realm={}",
+ System.getProperty(JAVA_SECURITY_KRB5_REALM, ""));
+ log.debug("java.security.krb5.kdc={}",
+ System.getProperty(JAVA_SECURITY_KRB5_KDC, ""));
+ log.debug("hadoop.security.authentication={}",
+ conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION));
+ log.debug("hadoop.security.authorization={}",
+ conf.get(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION));
+ UserGroupInformation.setConfiguration(conf);
+ UserGroupInformation authUser = UserGroupInformation.getCurrentUser();
+ log.debug("Authenticating as {}", authUser);
+ log.debug("Login user is {}", UserGroupInformation.getLoginUser());
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ throw new SliderException(LauncherExitCodes.EXIT_UNAUTHORIZED,
+ "Although secure mode is enabled," +
+ "the application has already set up its user as an insecure entity %s",
+ authUser);
+ }
+ if (authUser.getAuthenticationMethod() ==
+ UserGroupInformation.AuthenticationMethod.SIMPLE) {
+ throw new BadConfigException("Auth User is not Kerberized %s" +
+ " -security has already been set up with the wrong authentication method. "
+ + "This can occur if a file system has already been created prior to the loading of "
+ + "the security configuration.",
+ authUser);
+
+ }
+
+ SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
+ SliderUtils.verifyPrincipalSet(conf, "dfs.namenode.kerberos.principal");
+ return true;
+ }
+
+ /**
+ * Force an early login: This catches any auth problems early rather than
+ * in RPC operations
+ * @throws IOException if the login fails
+ */
+ public static void forceLogin() throws IOException {
+ if (UserGroupInformation.isSecurityEnabled()) {
+ if (UserGroupInformation.isLoginKeytabBased()) {
+ UserGroupInformation.getLoginUser().reloginFromKeytab();
+ } else {
+ UserGroupInformation.getLoginUser().reloginFromTicketCache();
+ }
+ }
+ }
+
+ public static String getLibDir() {
+ String[] libDirs = getLibDirs();
+ if (libDirs == null || libDirs.length == 0) {
+ return null;
+ }
+ return libDirs[0];
+ }
+
+ public static String[] getLibDirs() {
+ String libDirStr = System.getProperty(YarnServiceConstants.PROPERTY_LIB_DIR);
+ if (isUnset(libDirStr)) {
+ return ArrayUtils.EMPTY_STRING_ARRAY;
+ }
+ return StringUtils.split(libDirStr, ',');
+ }
+
+ /**
+ * Submit a JAR containing a specific class and map it
+ * @param providerResources provider map to build up
+ * @param sliderFileSystem remote fs
+ * @param clazz class to look for
+ * @param libdir lib directory
+ * @param jarName <i>At the destination</i>
+ * @return the local resource ref
+ * @throws IOException trouble copying to HDFS
+ */
+ public static LocalResource putJar(Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Class clazz,
+ Path tempPath,
+ String libdir,
+ String jarName
+ )
+ throws IOException, SliderException {
+ LocalResource res = sliderFileSystem.submitJarWithClass(
+ clazz,
+ tempPath,
+ libdir,
+ jarName);
+ providerResources.put(libdir + "/" + jarName, res);
+ return res;
+ }
+
+ /**
+ * Submit a JAR containing and map it
+ * @param providerResources provider map to build up
+ * @param sliderFileSystem remote fs
+ * @param libDir lib directory
+ * @param srcPath copy jars from
+ */
+ public static void putAllJars(Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem,
+ Path tempPath,
+ String libDir,
+ String srcPath) throws IOException, SliderException {
+ log.info("Loading all dependencies from {}", srcPath);
+ if (SliderUtils.isSet(srcPath)) {
+ File srcFolder = new File(srcPath);
+ FilenameFilter jarFilter = createJarFilter();
+ File[] listOfJars = srcFolder.listFiles(jarFilter);
+ if (listOfJars == null || listOfJars.length == 0) {
+ return;
+ }
+ for (File jarFile : listOfJars) {
+ LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName());
+ providerResources.put(libDir + "/" + jarFile.getName(), res);
+ }
+ }
+ }
+
+ /**
+ * Accept all filenames ending with {@code .jar}
+ * @return a filename filter
+ */
+ public static FilenameFilter createJarFilter() {
+ return new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.toLowerCase(Locale.ENGLISH).endsWith(".jar");
+ }
+ };
+ }
+
+ /**
+ * Submit the AM tar.gz containing all dependencies and map it
+ * @param providerResources provider map to build up
+ * @param sliderFileSystem remote fs
+ */
+ public static void putAmTarGzipAndUpdate(
+ Map<String, LocalResource> providerResources,
+ SliderFileSystem sliderFileSystem
+ ) throws IOException, SliderException {
+ log.info("Loading all dependencies from {}{}",
+ YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME,
+ YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
+ sliderFileSystem.submitTarGzipAndUpdate(providerResources);
+ }
+
+ public static Map<String, Map<String, String>> deepClone(Map<String, Map<String, String>> src) {
+ Map<String, Map<String, String>> dest = new HashMap<>();
+ for (Map.Entry<String, Map<String, String>> entry : src.entrySet()) {
+ dest.put(entry.getKey(), stringMapClone(entry.getValue()));
+ }
+ return dest;
+ }
+
+ public static Map<String, String> stringMapClone(Map<String, String> src) {
+ Map<String, String> dest = new HashMap<>();
+ return mergeEntries(dest, src.entrySet());
+ }
+
+ /**
+ * List a directory in the local filesystem
+ * @param dir directory
+ * @return a listing, one to a line
+ */
+ public static String listDir(File dir) {
+ if (dir == null) {
+ return "";
+ }
+ String[] confDirEntries = dir.list();
+ if (confDirEntries == null) {
+ return "";
+ }
+ StringBuilder builder = new StringBuilder();
+ for (String entry : confDirEntries) {
+ builder.append(entry).append("\n");
+ }
+ return builder.toString();
+ }
+
+ /**
+ * Create a file:// path from a local file
+ * @param file file to point the path
+ * @return a new Path
+ */
+ public static Path createLocalPath(File file) {
+ return new Path(file.toURI());
+ }
+
+ public static String getKerberosRealm() {
+ try {
+ return KerberosUtil.getDefaultRealm();
+ } catch (Exception e) {
+ log.debug("introspection into JVM internals failed", e);
+ return "(unknown)";
+
+ }
+ }
+
+ /**
+ * Build up the classpath for execution
+ * -behaves very differently on a mini test cluster vs a production
+ * production one.
+ *
+ * @param sliderConfDir relative path to the dir containing slider config
+ * options to put on the classpath -or null
+ * @param libdir directory containing the JAR files
+ * @param usingMiniMRCluster flag to indicate the MiniMR cluster is in use
+ * (and hence the current classpath should be used, not anything built up)
+ * @return a classpath
+ */
+ public static ClasspathConstructor buildClasspath(String sliderConfDir,
+ String libdir,
+ SliderFileSystem sliderFileSystem,
+ boolean usingMiniMRCluster) {
+
+ ClasspathConstructor classpath = new ClasspathConstructor();
+ classpath.append(YarnServiceConstants.YARN_SERVICE_LOG4J_FILENAME);
+
+ // add the runtime classpath needed for tests to work
+ if (usingMiniMRCluster) {
+ // for mini cluster we pass down the java CP properties
+ // and nothing else
+ classpath.appendAll(classpath.localJVMClasspath());
+ } else {
+ if (sliderConfDir != null) {
+ classpath.addClassDirectory(sliderConfDir);
+ }
+ classpath.addLibDir(libdir);
+ if (sliderFileSystem.isFile(sliderFileSystem.getDependencyTarGzip())) {
+ classpath.addLibDir(YarnServiceConstants.DEPENDENCY_LOCALIZED_DIR_LINK);
+ } else {
+ log.info(
+ "For faster submission of apps, upload dependencies using cmd {} {}",
+ SliderActions.ACTION_DEPENDENCY, Arguments.ARG_UPLOAD);
+ }
+ classpath.addRemoteClasspathEnvVar();
+ classpath.append(ApplicationConstants.Environment.HADOOP_CONF_DIR.$$());
+ }
+ return classpath;
+ }
+
+ /**
+ * Verify that a path refers to a directory. If not
+ * logs the parent dir then throws an exception
+ * @param dir the directory
+ * @param errorlog log for output on an error
+ * @throws FileNotFoundException if it is not a directory
+ */
+ public static void verifyIsDir(File dir, Logger errorlog) throws
+ FileNotFoundException {
+ if (!dir.exists()) {
+ errorlog.warn("contents of {}: {}", dir,
+ listDir(dir.getParentFile()));
+ throw new FileNotFoundException(dir.toString());
+ }
+ if (!dir.isDirectory()) {
+ errorlog.info("contents of {}: {}", dir,
+ listDir(dir.getParentFile()));
+ throw new FileNotFoundException(
+ "Not a directory: " + dir);
+ }
+ }
+
+ /**
+ * Verify that a file exists
+ * @param file file
+ * @param errorlog log for output on an error
+ * @throws FileNotFoundException
+ */
+ public static void verifyFileExists(File file, Logger errorlog) throws
+ FileNotFoundException {
+ if (!file.exists()) {
+ errorlog.warn("contents of {}: {}", file,
+ listDir(file.getParentFile()));
+ throw new FileNotFoundException(file.toString());
+ }
+ if (!file.isFile()) {
+ throw new FileNotFoundException("Not a file: " + file.toString());
+ }
+ }
+
+ /**
+ * verify that a config option is set
+ * @param configuration config
+ * @param key key
+ * @return the value, in case it needs to be verified too
+ * @throws BadConfigException if the key is missing
+ */
+ public static String verifyOptionSet(Configuration configuration, String key,
+ boolean allowEmpty) throws BadConfigException {
+ String val = configuration.get(key);
+ if (val == null) {
+ throw new BadConfigException(
+ "Required configuration option \"%s\" not defined ", key);
+ }
+ if (!allowEmpty && val.isEmpty()) {
+ throw new BadConfigException(
+ "Configuration option \"%s\" must not be empty", key);
+ }
+ return val;
+ }
+
+ /**
+ * Verify that a keytab property is defined and refers to a non-empty file
+ *
+ * @param siteConf configuration
+ * @param prop property to look for
+ * @return the file referenced
+ * @throws BadConfigException on a failure
+ */
+ public static File verifyKeytabExists(Configuration siteConf,
+ String prop) throws
+ BadConfigException {
+ String keytab = siteConf.get(prop);
+ if (keytab == null) {
+ throw new BadConfigException("Missing keytab property %s",
+ prop);
+
+ }
+ File keytabFile = new File(keytab);
+ if (!keytabFile.exists()) {
+ throw new BadConfigException("Missing keytab file %s defined in %s",
+ keytabFile,
+ prop);
+ }
+ if (keytabFile.length() == 0 || !keytabFile.isFile()) {
+ throw new BadConfigException("Invalid keytab file %s defined in %s",
+ keytabFile,
+ prop);
+ }
+ return keytabFile;
+ }
+
+ /**
+ * Add a subpath to an existing URL. This extends
+ * the path, inserting a / between all entries
+ * if needed.
+ * @param base base path/URL
+ * @param path subpath
+ * @return base+"/"+subpath
+ */
+ public static String appendToURL(String base, String path) {
+ StringBuilder fullpath = new StringBuilder(base);
+ if (!base.endsWith("/")) {
+ fullpath.append("/");
+ }
+ if (path.startsWith("/")) {
+ fullpath.append(path.substring(1));
+ } else {
+ fullpath.append(path);
+ }
+ return fullpath.toString();
+ }
+
+ /**
+ * Truncate the given string to a maximum length provided
+ * with a pad (...) added to the end if expected size if more than 10.
+ * @param toTruncate string to truncate; may be null
+ * @param maxSize maximum size
+ * @return the truncated/padded string.
+ */
+ public static String truncate(String toTruncate, int maxSize) {
+ if (toTruncate == null || maxSize < 1
+ || toTruncate.length() <= maxSize) {
+ return toTruncate;
+ }
+
+ String pad = "...";
+ if (maxSize < 10) {
+ pad = "";
+ }
+ return toTruncate.substring(0, maxSize - pad.length()).concat(pad);
+ }
+
+ /**
+ * Given a source folder create zipped file
+ *
+ * @param srcFolder
+ * @param zipFile
+ *
+ * @throws IOException
+ */
+ public static void zipFolder(File srcFolder, File zipFile) throws IOException {
+ log.info("Zipping folder {} to {}", srcFolder.getAbsolutePath(), zipFile.getAbsolutePath());
+ List<String> files = new ArrayList<>();
+ generateFileList(files, srcFolder, srcFolder, true);
+
+ byte[] buffer = new byte[1024];
+
+ try (FileOutputStream fos = new FileOutputStream(zipFile)) {
+ try (ZipOutputStream zos = new ZipOutputStream(fos)) {
+
+ for (String file : files) {
+ ZipEntry ze = new ZipEntry(file);
+ zos.putNextEntry(ze);
+ try (FileInputStream in = new FileInputStream(srcFolder + File.separator + file)) {
+ int len;
+ while ((len = in.read(buffer)) > 0) {
+ zos.write(buffer, 0, len);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Given a source folder create a tar.gz file
+ *
+ * @param libDirs
+ * @param tarGzipFile
+ *
+ * @throws IOException
+ */
+ public static void tarGzipFolder(String[] libDirs, File tarGzipFile,
+ FilenameFilter filter) throws IOException {
+ log.info("Tar-gzipping folders {} to {}", libDirs,
+ tarGzipFile.getAbsolutePath());
+
+ try(TarArchiveOutputStream taos =
+ new TarArchiveOutputStream(new GZIPOutputStream(
+ new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) {
+ for (String libDir : libDirs) {
+ File srcFolder = new File(libDir);
+ List<String> files = new ArrayList<>();
+ generateFileList(files, srcFolder, srcFolder, true, filter);
+ for (String file : files) {
+ File srcFile = new File(srcFolder, file);
+ TarArchiveEntry tarEntry = new TarArchiveEntry(
+ srcFile, file);
+ taos.putArchiveEntry(tarEntry);
+ try(FileInputStream in = new FileInputStream(srcFile)) {
+ org.apache.commons.io.IOUtils.copy(in, taos);
+ }
+ taos.flush();
+ taos.closeArchiveEntry();
+ }
+ }
+ }
+ }
+
+ private static void generateFileList(List<String> fileList, File node,
+ File rootFolder, Boolean relative) {
+ generateFileList(fileList, node, rootFolder, relative, null);
+ }
+
+ private static void generateFileList(List<String> fileList, File node,
+ File rootFolder, Boolean relative, FilenameFilter filter) {
+ if (node.isFile()) {
+ String fileFullPath = node.toString();
+ if (relative) {
+ fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1,
+ fileFullPath.length()));
+ } else {
+ fileList.add(fileFullPath);
+ }
+ }
+
+ if (node.isDirectory()) {
+ String[] subNode = node.list(filter);
+ if (subNode == null || subNode.length == 0) {
+ return;
+ }
+ for (String filename : subNode) {
+ generateFileList(fileList, new File(node, filename), rootFolder,
+ relative, filter);
+ }
+ }
+ }
+
+ /**
+ * Check for any needed libraries being present. On Unix none are needed;
+ * on windows they must be present
+ * @return true if all is well
+ */
+ public static String checkForRequiredNativeLibraries() {
+
+ if (!Shell.WINDOWS) {
+ return "";
+ }
+ StringBuilder errorText = new StringBuilder("");
+ if (!NativeIO.isAvailable()) {
+ errorText.append("No native IO library. ");
+ }
+ try {
+ String path = Shell.getQualifiedBinPath(WINUTILS);
+ log.debug("winutils is at {}", path);
+ } catch (IOException e) {
+ errorText.append("No " + WINUTILS);
+ log.warn("No winutils: {}", e, e);
+ }
+ try {
+ File target = new File("target");
+ FileUtil.canRead(target);
+ } catch (UnsatisfiedLinkError e) {
+ log.warn("Failing to link to native IO methods: {}", e, e);
+ errorText.append("No native IO methods");
+ }
+ return errorText.toString();
+ }
+
+ /**
+ * Strictly verify that windows utils is present.
+ * Checks go as far as opening the file and looking for
+ * the headers.
+ * @throws IOException on any problem reading the file
+ * @throws FileNotFoundException if the file is not considered valid
+ */
+ public static void maybeVerifyWinUtilsValid() throws
+ IOException,
+ SliderException {
+ String errorText = SliderUtils.checkForRequiredNativeLibraries();
+ if (!errorText.isEmpty()) {
+ throw new BadClusterStateException(errorText);
+ }
+ }
+
+ /**
+ * Write bytes to a file
+ * @param outfile output file
+ * @param data data to write
+ * @throws IOException on any IO problem
+ */
+ public static void write(File outfile, byte[] data)
+ throws IOException {
+ File parentDir = outfile.getCanonicalFile().getParentFile();
+ if (parentDir == null) {
+ throw new IOException(outfile.getPath() + " has no parent dir");
+ }
+ if (!parentDir.exists()) {
+ if(!parentDir.mkdirs()) {
+ throw new IOException("Failed to create parent directory " + parentDir);
+ }
+ }
+ SliderUtils.verifyIsDir(parentDir, log);
+ try(FileOutputStream out = new FileOutputStream(outfile)) {
+ out.write(data);
+ }
+ }
+
+ /**
+ * Compare the times of two applications: most recent app comes first
+ * Specifically: the one whose start time value is greater.
+ */
+ private static class MostRecentlyStartedAppFirst
+ implements Comparator<ApplicationReport>, Serializable {
+ @Override
+ public int compare(ApplicationReport r1, ApplicationReport r2) {
+ long x = r1.getStartTime();
+ long y = r2.getStartTime();
+ return compareTwoLongsReverse(x, y);
+ }
+ }
+
+ /**
+ * Compare the times of two applications: most recent app comes first.
+ * "Recent"== the app whose start time <i>or finish time</i> is the greatest.
+ */
+ private static class MostRecentlyStartedOrFinishedFirst
+ implements Comparator<ApplicationReport>, Serializable {
+ @Override
+ public int compare(ApplicationReport r1, ApplicationReport r2) {
+ long started1 = r1.getStartTime();
+ long started2 = r2.getStartTime();
+ long finished1 = r1.getFinishTime();
+ long finished2 = r2.getFinishTime();
+ long lastEvent1 = Math.max(started1, finished1);
+ long lastEvent2 = Math.max(started2, finished2);
+ return compareTwoLongsReverse(lastEvent1, lastEvent2);
+ }
+ }
+
+ /**
+ * Compare the times of two applications: most recently finished app comes first
+ * Specifically: the one whose finish time value is greater.
+ */
+ private static class MostRecentAppFinishFirst
+ implements Comparator<ApplicationReport>, Serializable {
+ @Override
+ public int compare(ApplicationReport r1, ApplicationReport r2) {
+ long x = r1.getFinishTime();
+ long y = r2.getFinishTime();
+ return compareTwoLongsReverse(x, y);
+ }
+ }
+
+ /**
+ * Compare two long values for sorting. As the return value for
+ * comparators must be int, the simple value of <code>x-y</code>
+ * is inapplicable
+ * @param x x value
+ * @param y y value
+ * @return +ve if x is less than y, -ve if y is greater than x; 0 for equality
+ */
+ public static int compareTwoLongsReverse(long x, long y) {
+ return (x < y) ? 1 : ((x == y) ? 0 : -1);
+ }
+
+ public static String createNameTag(String name) {
+ return "Name: " + name;
+ }
+
+ public static String createVersionTag(String version) {
+ return "Version: " + version;
+ }
+
+ public static String createDescriptionTag(String description) {
+ return "Description: " + description;
+ }
+
+ public static final String DAYS = ".days";
+ public static final String HOURS = ".hours";
+ public static final String MINUTES = ".minutes";
+ public static final String SECONDS = ".seconds";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d44876af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
new file mode 100644
index 0000000..1fa07ce
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.net.HostAndPort;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ZookeeperUtils {
+ public static final int DEFAULT_PORT = 2181;
+
+ public static String buildConnectionString(String zkHosts, int port) {
+ String zkPort = Integer.toString(port);
+ //parse the hosts
+ String[] hostlist = zkHosts.split(",", 0);
+ String quorum = SliderUtils.join(hostlist, ":" + zkPort + ",", false);
+ return quorum;
+ }
+
+ /**
+ * Take a quorum list and split it to (trimmed) pairs
+ * @param hostPortQuorumList list of form h1:port, h2:port2,...
+ * @return a possibly empty list of values between commas. They may not be
+ * valid hostname:port pairs
+ */
+ public static List<String> splitToPairs(String hostPortQuorumList) {
+ // split an address hot
+ String[] strings = StringUtils.getStrings(hostPortQuorumList);
+ int len = 0;
+ if (strings != null) {
+ len = strings.length;
+ }
+ List<String> tuples = new ArrayList<String>(len);
+ if (strings != null) {
+ for (String s : strings) {
+ tuples.add(s.trim());
+ }
+ }
+ return tuples;
+ }
+
+ /**
+ * Split a quorum list into a list of hostnames and ports
+ * @param hostPortQuorumList split to a list of hosts and ports
+ * @return a list of values
+ */
+ public static List<HostAndPort> splitToHostsAndPorts(String hostPortQuorumList) {
+ // split an address hot
+ String[] strings = StringUtils.getStrings(hostPortQuorumList);
+ int len = 0;
+ if (strings != null) {
+ len = strings.length;
+ }
+ List<HostAndPort> list = new ArrayList<HostAndPort>(len);
+ if (strings != null) {
+ for (String s : strings) {
+ list.add(HostAndPort.fromString(s.trim()).withDefaultPort(DEFAULT_PORT));
+ }
+ }
+ return list;
+ }
+
+ /**
+ * Build up to a hosts only list
+ * @param hostAndPorts
+ * @return a list of the hosts only
+ */
+ public static String buildHostsOnlyList(List<HostAndPort> hostAndPorts) {
+ StringBuilder sb = new StringBuilder();
+ for (HostAndPort hostAndPort : hostAndPorts) {
+ sb.append(hostAndPort.getHostText()).append(",");
+ }
+ if (sb.length() > 0) {
+ sb.delete(sb.length() - 1, sb.length());
+ }
+ return sb.toString();
+ }
+
+ public static String buildQuorumEntry(HostAndPort hostAndPort,
+ int defaultPort) {
+ String s = hostAndPort.toString();
+ if (hostAndPort.hasPort()) {
+ return s;
+ } else {
+ return s + ":" + defaultPort;
+ }
+ }
+
+ /**
+ * Build a quorum list, injecting a ":defaultPort" ref if needed on
+ * any entry without one
+ * @param hostAndPorts
+ * @param defaultPort
+ * @return
+ */
+ public static String buildQuorum(List<HostAndPort> hostAndPorts, int defaultPort) {
+ List<String> entries = new ArrayList<String>(hostAndPorts.size());
+ for (HostAndPort hostAndPort : hostAndPorts) {
+ entries.add(buildQuorumEntry(hostAndPort, defaultPort));
+ }
+ return SliderUtils.join(entries, ",", false);
+ }
+
+ public static String convertToHostsOnlyList(String quorum) throws
+ BadConfigException {
+ List<HostAndPort> hostAndPorts = splitToHostsAndPortsStrictly(quorum);
+ return ZookeeperUtils.buildHostsOnlyList(hostAndPorts);
+ }
+
+ public static List<HostAndPort> splitToHostsAndPortsStrictly(String quorum) throws
+ BadConfigException {
+ List<HostAndPort> hostAndPorts =
+ ZookeeperUtils.splitToHostsAndPorts(quorum);
+ if (hostAndPorts.isEmpty()) {
+ throw new BadConfigException("empty zookeeper quorum");
+ }
+ return hostAndPorts;
+ }
+
+ public static int getFirstPort(String quorum, int defVal) throws
+ BadConfigException {
+ List<HostAndPort> hostAndPorts = splitToHostsAndPortsStrictly(quorum);
+ int port = hostAndPorts.get(0).getPortOrDefault(defVal);
+ return port;
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d44876af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/Slider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/Slider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/Slider.java
deleted file mode 100644
index 5fc8618..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/Slider.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider;
-
-import org.apache.slider.client.SliderClient;
-import org.apache.slider.core.main.ServiceLauncher;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * This is just the entry point class
- */
-public class Slider extends SliderClient {
-
-
- public static final String SERVICE_CLASSNAME = "org.apache.slider.Slider";
-
- /**
- * This is the main entry point for the service launcher.
- * @param args command line arguments.
- */
- public static void main(String[] args) {
-
- //turn the args to a list
- List<String> argsList = Arrays.asList(args);
- //create a new list, as the ArrayList type doesn't push() on an insert
- List<String> extendedArgs = new ArrayList<String>(argsList);
- //insert the service name
- extendedArgs.add(0, SERVICE_CLASSNAME);
- //now have the service launcher do its work
- ServiceLauncher.serviceMain(extendedArgs);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d44876af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterNode.java
deleted file mode 100644
index 8b0a563..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterNode.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.api.proto.Messages;
-import org.codehaus.jackson.JsonParseException;
-import org.codehaus.jackson.annotate.JsonIgnore;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.JsonMappingException;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-/**
- * Describe a specific node in the cluster
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL )
-public final class ClusterNode implements Cloneable {
- protected static final Logger
- LOG = LoggerFactory.getLogger(ClusterNode.class);
-
- @JsonIgnore
- public ContainerId containerId;
-
- /**
- * server name
- */
- public String name;
-
-
- /**
- * UUID of container used in Slider RPC to refer to instances
- */
- public String id;
-
- public String role;
-
- public int roleId;
-
- public long createTime;
- public long startTime;
- /**
- * flag set when it is released, to know if it has
- * already been targeted for termination
- */
- public boolean released;
- public String host;
- public String ip;
- public String hostname;
- public String hostUrl;
-
- /**
- * state from {@link ClusterDescription}
- */
- public int state;
-
- /**
- * Exit code: only valid if the state >= STOPPED
- */
- public int exitCode;
-
- /**
- * what was the command executed?
- */
- public String command;
-
- /**
- * Any diagnostics
- */
- public String diagnostics;
-
- /**
- * What is the tail output from the executed process (or [] if not started
- * or the log cannot be picked up
- */
- public String[] output;
-
- /**
- * Any environment details
- */
- public String[] environment;
-
- /**
- * server-side ctor takes the container ID and builds the name from it
- * @param containerId container ID; can be null
- */
- public ClusterNode(ContainerId containerId) {
- if (containerId != null) {
- this.containerId = containerId;
- this.name = containerId.toString();
- }
- }
-
- /**
- * ctor for deserialization
- */
- public ClusterNode() {
- }
-
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder();
- builder.append(name).append(": ");
- builder.append(state).append("\n");
- builder.append("state: ").append(state).append("\n");
- builder.append("role: ").append(role).append("\n");
- append(builder, "host", host);
- append(builder, "hostURL", hostUrl);
- append(builder, "command", command);
- if (output != null) {
- for (String line : output) {
- builder.append(line).append("\n");
- }
- }
- append(builder, "diagnostics", diagnostics);
- return builder.toString();
- }
-
- private void append(StringBuilder builder, String key, Object val) {
- if (val != null) {
- builder.append(key).append(": ").append(val.toString()).append("\n");
- }
- }
-
- /**
- * Convert to a JSON string
- * @return a JSON string description
- * @throws IOException Problems mapping/writing the object
- */
- public String toJsonString() throws IOException {
- ObjectMapper mapper = new ObjectMapper();
- return mapper.writeValueAsString(this);
- }
-
-
- /**
- * Convert from JSON
- * @param json input
- * @return the parsed JSON
- * @throws IOException IO
- */
- public static ClusterNode fromJson(String json)
- throws IOException, JsonParseException, JsonMappingException {
- ObjectMapper mapper = new ObjectMapper();
- try {
- return mapper.readValue(json, ClusterNode.class);
- } catch (IOException e) {
- LOG.error("Exception while parsing json : {}\n{}", e , json, e);
- throw e;
- }
- }
-
- /**
- * Build from a protobuf response
- * @param message
- * @return the deserialized node
- */
- public static ClusterNode fromProtobuf(Messages.RoleInstanceState message) {
- ClusterNode node = new ClusterNode();
- node.name = message.getName();
- node.command = message.getCommand();
- node.diagnostics = message.getDiagnostics();
- String[] arr;
- int environmentCount = message.getEnvironmentCount();
- if (environmentCount > 0) {
- arr = new String[environmentCount];
- node.environment = message.getEnvironmentList().toArray(arr);
- }
- node.exitCode = message.getExitCode();
- int outputCount = message.getOutputCount();
- if (outputCount > 0) {
- arr = new String[outputCount];
- node.output = message.getOutputList().toArray(arr);
- }
- node.role = message.getRole();
- node.roleId = message.getRoleId();
- node.state = message.getState();
- node.host = message.getHost();
- node.hostUrl = message.getHostURL();
- node.createTime = message.getCreateTime();
- node.startTime = message.getStartTime();
- node.released = message.getReleased();
- return node;
- }
-
- @Override
- public Object clone() throws CloneNotSupportedException {
- return super.clone();
- }
-
- public ClusterNode doClone() {
- try {
- return (ClusterNode)clone();
- } catch (CloneNotSupportedException e) {
- //not going to happen. This is a final class
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d44876af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
deleted file mode 100644
index 0e3b535..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api;
-
-/**
- * Keys for internal use, go into `internal.json` and not intended for normal
- * use except when tuning Slider AM operations
- */
-public interface InternalKeys {
-
-
- /**
- * Home dir of the app: {@value}
- * If set, implies there is a home dir to use
- */
- String INTERNAL_APPLICATION_HOME = "internal.application.home";
- /**
- * Path to an image file containing the app: {@value}
- */
- String INTERNAL_APPLICATION_IMAGE_PATH = "internal.application.image.path";
- /**
- * Time in milliseconds to wait after forking any in-AM
- * process before attempting to start up the containers: {@value}
- *
- * A shorter value brings the cluster up faster, but means that if the
- * in AM process fails (due to a bad configuration), then time
- * is wasted starting containers on a cluster that isn't going to come
- * up
- */
- String INTERNAL_CONTAINER_STARTUP_DELAY = "internal.container.startup.delay";
- /**
- * internal temp directory: {@value}
- */
- String INTERNAL_AM_TMP_DIR = "internal.am.tmp.dir";
- /**
- * internal temp directory: {@value}
- */
- String INTERNAL_TMP_DIR = "internal.tmp.dir";
- /**
- * where a snapshot of the original conf dir is: {@value}
- */
- String INTERNAL_SNAPSHOT_CONF_PATH = "internal.snapshot.conf.path";
- /**
- * where a snapshot of the original conf dir is: {@value}
- */
- String INTERNAL_GENERATED_CONF_PATH = "internal.generated.conf.path";
- /**
- * where a snapshot of the original conf dir is: {@value}
- */
- String INTERNAL_PROVIDER_NAME = "internal.provider.name";
- /**
- * where a snapshot of the original conf dir is: {@value}
- */
- String INTERNAL_DATA_DIR_PATH = "internal.data.dir.path";
- /**
- * where the app def is stored
- */
- String INTERNAL_APPDEF_DIR_PATH = "internal.appdef.dir.path";
- /**
- * where addons for the app are stored
- */
- String INTERNAL_ADDONS_DIR_PATH = "internal.addons.dir.path";
- /**
- * Time in milliseconds to wait after forking any in-AM
- * process before attempting to start up the containers: {@value}
- *
- * A shorter value brings the cluster up faster, but means that if the
- * in AM process fails (due to a bad configuration), then time
- * is wasted starting containers on a cluster that isn't going to come
- * up
- */
- int DEFAULT_INTERNAL_CONTAINER_STARTUP_DELAY = 5000;
- /**
- * Time in seconds before a container is considered long-lived.
- * Shortlived containers are interpreted as a problem with the role
- * and/or the host: {@value}
- */
- String INTERNAL_CONTAINER_FAILURE_SHORTLIFE =
- "internal.container.failure.shortlife";
- /**
- * Default short life threshold: {@value}
- */
- int DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE = 60;
-
- /**
- * Version of the app: {@value}
- */
- String KEYTAB_LOCATION = "internal.keytab.location";
-
- /**
- * Queue used to deploy the app: {@value}
- */
- String INTERNAL_QUEUE = "internal.queue";
-
- /**
- * Flag to indicate whether or not the chaos monkey is enabled:
- * {@value}
- */
- String CHAOS_MONKEY_ENABLED = "internal.chaos.monkey.enabled";
- boolean DEFAULT_CHAOS_MONKEY_ENABLED = false;
-
-
- /**
- * Rate
- */
-
- String CHAOS_MONKEY_INTERVAL = "internal.chaos.monkey.interval";
- String CHAOS_MONKEY_INTERVAL_DAYS = CHAOS_MONKEY_INTERVAL + ".days";
- String CHAOS_MONKEY_INTERVAL_HOURS = CHAOS_MONKEY_INTERVAL + ".hours";
- String CHAOS_MONKEY_INTERVAL_MINUTES = CHAOS_MONKEY_INTERVAL + ".minutes";
- String CHAOS_MONKEY_INTERVAL_SECONDS = CHAOS_MONKEY_INTERVAL + ".seconds";
-
- long DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS = 0;
- long DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS = 0;
- long DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES = 0;
-
- String CHAOS_MONKEY_DELAY = "internal.chaos.monkey.delay";
- String CHAOS_MONKEY_DELAY_DAYS = CHAOS_MONKEY_DELAY + ".days";
- String CHAOS_MONKEY_DELAY_HOURS = CHAOS_MONKEY_DELAY + ".hours";
- String CHAOS_MONKEY_DELAY_MINUTES = CHAOS_MONKEY_DELAY + ".minutes";
- String CHAOS_MONKEY_DELAY_SECONDS = CHAOS_MONKEY_DELAY + ".seconds";
-
- int DEFAULT_CHAOS_MONKEY_STARTUP_DELAY = 0;
-
- /**
- * Prefix for all chaos monkey probabilities
- */
- String CHAOS_MONKEY_PROBABILITY =
- "internal.chaos.monkey.probability";
- /**
- * Probabilies are out of 10000 ; 100==1%
- */
-
- /**
- * Probability of a monkey check killing the AM: {@value}
- */
- String CHAOS_MONKEY_PROBABILITY_AM_FAILURE =
- CHAOS_MONKEY_PROBABILITY + ".amfailure";
-
- /**
- * Default probability of a monkey check killing the AM: {@value}
- */
- int DEFAULT_CHAOS_MONKEY_PROBABILITY_AM_FAILURE = 0;
-
- /**
- * Probability of a monkey check killing the AM: {@value}
- */
- String CHAOS_MONKEY_PROBABILITY_AM_LAUNCH_FAILURE =
- CHAOS_MONKEY_PROBABILITY + ".amlaunchfailure";
-
- /**
- * Probability of a monkey check killing a container: {@value}
- */
-
- String CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE =
- CHAOS_MONKEY_PROBABILITY + ".containerfailure";
-
- /**
- * Default probability of a monkey check killing the a container: {@value}
- */
- int DEFAULT_CHAOS_MONKEY_PROBABILITY_CONTAINER_FAILURE = 0;
-
-
- /**
- * 1% of chaos
- */
- int PROBABILITY_PERCENT_1 = 100;
-
- /**
- * 100% for chaos values
- */
- int PROBABILITY_PERCENT_100 = 100 * PROBABILITY_PERCENT_1;
-
- /**
- * interval between checks for escalation: {@value}
- */
- String ESCALATION_CHECK_INTERVAL = "escalation.check.interval.seconds";
-
- /**
- * default value: {@value}
- */
- int DEFAULT_ESCALATION_CHECK_INTERVAL = 30;
-
-
- /**
- * interval between readiness checks: {@value}
- */
- String MONITOR_INTERVAL = "monitor.interval.seconds";
-
- /**
- * default value: {@value}
- */
- int DEFAULT_MONITOR_INTERVAL = 30;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d44876af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
deleted file mode 100644
index 988627d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api;
-
-/**
- * Keys for entries in the <code>options</code> section
- * of a cluster description.
- */
-public interface OptionKeys extends InternalKeys {
-
- /**
- * Time in milliseconds to wait after forking any in-AM
- * process before attempting to start up the containers: {@value}
- *
- * A shorter value brings the cluster up faster, but means that if the
- * in AM process fails (due to a bad configuration), then time
- * is wasted starting containers on a cluster that isn't going to come
- * up
- */
- String APPLICATION_TYPE = "application.type";
-
- String APPLICATION_NAME = "application.name";
-
- /**
- * Prefix for site.xml options: {@value}
- */
- String SITE_XML_PREFIX = "site.";
- /**
- * Prefix for config file options: {@value}
- */
- String CONF_FILE_PREFIX = "conf.";
- /**
- * Prefix for package options: {@value}
- */
- String PKG_FILE_PREFIX = "pkg.";
- /**
- * Prefix for export options: {@value}
- */
- String EXPORT_PREFIX = "export.";
- /**
- * Type suffix for config file and package options: {@value}
- */
- String TYPE_SUFFIX = ".type";
- /**
- * Name suffix for config file and package options: {@value}
- */
- String NAME_SUFFIX = ".name";
- /**
- * Per component suffix for config file options: {@value}
- */
- String PER_COMPONENT = ".per.component";
- /**
- * Per group suffix for config file options: {@value}
- */
- String PER_GROUP = ".per.group";
-
- /**
- * Zookeeper quorum host list: {@value}
- */
- String ZOOKEEPER_QUORUM = "zookeeper.quorum";
- String ZOOKEEPER_HOSTS = "zookeeper.hosts";
- String ZOOKEEPER_PORT = "zookeeper.port";
-
- /**
- * Zookeeper path value (string): {@value}
- */
- String ZOOKEEPER_PATH = "zookeeper.path";
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org