You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/09/05 05:10:54 UTC
[30/51] [abbrv] hadoop git commit: YARN-7050. Post cleanup after
YARN-6903, removal of org.apache.slider package. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf581071/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
deleted file mode 100644
index 64fd8ae..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
+++ /dev/null
@@ -1,611 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.xml.sax.SAXException;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.StringWriter;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
-/**
- * Methods to aid in config, both in the Configuration class and
- * with other parts of setting up Slider-initated processes.
- *
- * Some of the methods take an argument of a map iterable for their sources; this allows
- * the same method
- */
-public class ConfigHelper {
- private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class);
-
- /**
- * Dump the (sorted) configuration
- * @param conf config
- * @return the sorted keyset
- */
- public static Set<String> dumpConf(Configuration conf) {
- Set<String> keys = sortedConfigKeys(conf);
- for (String key : keys) {
- log.info("{}={}", key, conf.get(key));
- }
- return keys;
- }
-
- /**
- * Take a configuration and return a sorted set
- * @param conf config
- * @return the sorted keyset
-
- */
- public static Set<String> sortedConfigKeys(Iterable<Map.Entry<String, String>> conf) {
- TreeSet<String> sorted = new TreeSet<String>();
- for (Map.Entry<String, String> entry : conf) {
- sorted.add(entry.getKey());
- }
- return sorted;
- }
-
- /**
- * Set an entire map full of values
- *
- * @param config config to patch
- * @param map map of data
- * @param origin origin data
- */
- public static void addConfigMap(Configuration config,
- Map<String, String> map,
- String origin) throws BadConfigException {
- addConfigMap(config, map.entrySet(), origin);
- }
-
- /**
- * Set an entire map full of values
- *
- * @param config config to patch
- * @param map map of data
- * @param origin origin data
- */
- public static void addConfigMap(Configuration config,
- Iterable<Map.Entry<String, String>> map,
- String origin) throws BadConfigException {
- for (Map.Entry<String, String> mapEntry : map) {
- String key = mapEntry.getKey();
- String value = mapEntry.getValue();
- if (value == null) {
- throw new BadConfigException("Null value for property " + key);
- }
- config.set(key, value, origin);
- }
- }
-
-
- /**
- * Save a config file in a destination directory on a given filesystem
- * @param systemConf system conf used for creating filesystems
- * @param confToSave config to save
- * @param confdir the directory path where the file is to go
- * @param filename the filename
- * @return the destination path where the file was saved
- * @throws IOException IO problems
- */
- public static Path saveConfig(Configuration systemConf,
- Configuration confToSave,
- Path confdir,
- String filename) throws IOException {
- FileSystem fs = FileSystem.get(confdir.toUri(), systemConf);
- Path destPath = new Path(confdir, filename);
- saveConfig(fs, destPath, confToSave);
- return destPath;
- }
-
- /**
- * Save a config
- * @param fs filesystem
- * @param destPath dest to save
- * @param confToSave config to save
- * @throws IOException IO problems
- */
- public static void saveConfig(FileSystem fs,
- Path destPath,
- Configuration confToSave) throws
- IOException {
- FSDataOutputStream fos = fs.create(destPath);
- try {
- confToSave.writeXml(fos);
- } finally {
- IOUtils.closeStream(fos);
- }
- }
-
- /**
- * Convert to an XML string
- * @param conf configuration
- * @return conf
- * @throws IOException
- */
- public static String toXml(Configuration conf) throws IOException {
- StringWriter writer = new StringWriter();
- conf.writeXml(writer);
- return writer.toString();
- }
-
- /**
- * This will load and parse a configuration to an XML document
- * @param fs filesystem
- * @param path path
- * @return an XML document
- * @throws IOException IO failure
- */
- public Document parseConfiguration(FileSystem fs,
- Path path) throws
- IOException {
-
-
- byte[] data = loadBytes(fs, path);
- //this is here to track down a parse issue
- //related to configurations
- String s = new String(data, 0, data.length, "UTF-8");
- log.debug("XML resource {} is \"{}\"", path, s);
-/* JDK7
- try (ByteArrayInputStream in = new ByteArrayInputStream(data)) {
- Document document = parseConfigXML(in);
- return document;
- } catch (ParserConfigurationException | SAXException e) {
- throw new IOException(e);
- }
-*/
- ByteArrayInputStream in= null;
- try {
- in = new ByteArrayInputStream(data);
- Document document = parseConfigXML(in);
- return document;
- } catch (ParserConfigurationException e) {
- throw new IOException(e);
- } catch (SAXException e) {
- throw new IOException(e);
- } finally {
- IOUtils.closeStream(in);
- }
- }
-
- public static byte[] loadBytes(FileSystem fs, Path path) throws IOException {
- int len = (int) fs.getLength(path);
- byte[] data = new byte[len];
- /* JDK7
- try(FSDataInputStream in = fs.open(path)) {
- in.readFully(0, data);
- }
-*/
- FSDataInputStream in = null;
- in = fs.open(path);
- try {
- in.readFully(0, data);
- } finally {
- IOUtils.closeStream(in);
- }
- return data;
- }
-
- /**
- * Load a configuration from ANY FS path. The normal Configuration
- * loader only works with file:// URIs
- * @param fs filesystem
- * @param path path
- * @return a loaded resource
- * @throws IOException
- */
- public static Configuration loadConfiguration(FileSystem fs,
- Path path) throws IOException {
- byte[] data = loadBytes(fs, path);
-
- ByteArrayInputStream in2;
-
- in2 = new ByteArrayInputStream(data);
- Configuration conf1 = new Configuration(false);
- conf1.addResource(in2);
- //now clone it while dropping all its sources
- Configuration conf2 = new Configuration(false);
- String src = path.toString();
- for (Map.Entry<String, String> entry : conf1) {
- String key = entry.getKey();
- String value = entry.getValue();
- conf2.set(key, value, src);
- }
- return conf2;
- }
-
-
- /**
- * Generate a config file in a destination directory on the local filesystem
- * @param confdir the directory path where the file is to go
- * @param filename the filename
- * @return the destination path
- */
- public static File saveConfig(Configuration generatingConf,
- File confdir,
- String filename) throws IOException {
-
-
- File destPath = new File(confdir, filename);
- OutputStream fos = new FileOutputStream(destPath);
- try {
- generatingConf.writeXml(fos);
- } finally {
- IOUtils.closeStream(fos);
- }
- return destPath;
- }
-
- /**
- * Parse an XML Hadoop configuration into an XML document. x-include
- * is supported, but as the location isn't passed in, relative
- * URIs are out.
- * @param in instream
- * @return a document
- * @throws ParserConfigurationException parser feature problems
- * @throws IOException IO problems
- * @throws SAXException XML is invalid
- */
- public static Document parseConfigXML(InputStream in) throws
- ParserConfigurationException,
- IOException,
- SAXException {
- DocumentBuilderFactory docBuilderFactory
- = DocumentBuilderFactory.newInstance();
- //ignore all comments inside the xml file
- docBuilderFactory.setIgnoringComments(true);
-
- //allow includes in the xml file
- docBuilderFactory.setNamespaceAware(true);
- docBuilderFactory.setXIncludeAware(true);
- DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
- return builder.parse(in);
- }
-
- /**
- * Load a Hadoop configuration from a local file.
- * @param file file to load
- * @return a configuration which hasn't actually had the load triggered
- * yet.
- * @throws FileNotFoundException file is not there
- * @throws IOException any other IO problem
- */
- public static Configuration loadConfFromFile(File file) throws
- IOException {
- return loadConfFromFile(file, false);
- }
-
- /**
- *
- * Load a Hadoop configuration from a local file.
- * @param file file to load
- * @param loadDefaults flag to indicate if the defaults should be loaded yet
- * @return a configuration which hasn't actually had the load triggered
- * yet.
- * @throws FileNotFoundException file is not there
- * @throws IOException any other IO problem
- */
- public static Configuration loadConfFromFile(File file,
- boolean loadDefaults) throws IOException {
- if (!file.exists()) {
- throw new FileNotFoundException("File not found :"
- + file.getAbsoluteFile());
- }
- Configuration conf = new Configuration(loadDefaults);
- try {
- conf.addResource(file.toURI().toURL());
- } catch (MalformedURLException e) {
- // should never happen...
- throw new IOException(
- "File " + file.toURI() + " doesn't have a valid URL");
- }
- return conf;
- }
-
- /**
- * Add a configuration from a file to an existing configuration
- * @param conf existing configuration
- * @param file file to load
- * @param overwrite flag to indicate new values should overwrite the predecessor
- * @return the merged configuration
- * @throws IOException
- */
- public static Configuration addConfigurationFile(Configuration conf,
- File file, boolean overwrite)
- throws IOException {
- Configuration c2 = loadConfFromFile(file, false);
- mergeConfigurations(conf, c2, file.getAbsolutePath(), overwrite);
- return conf;
- }
-
- /**
- * Add the system env variables with the given prefix (by convention, env.)
- * @param conf existing configuration
- * @param prefix prefix
- */
- public static void addEnvironmentVariables(Configuration conf, String prefix) {
- Map<String, String> env = System.getenv();
- for (Map.Entry<String, String> entry : env.entrySet()) {
- conf.set(prefix + entry.getKey(),entry.getValue(), "env");
- }
- }
-
- /**
- * looks for the config under $confdir/$templateFilename; if not there
- * loads it from /conf/templateFile.
- * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the
- * origin to help debug what's happening
- * @param systemConf system conf
- * @param confdir conf dir in FS
- * @param templateFilename filename in the confdir
- * @param fallbackResource resource to fall back on
- * @return loaded conf
- * @throws IOException IO problems
- */
- public static Configuration loadTemplateConfiguration(Configuration systemConf,
- Path confdir,
- String templateFilename,
- String fallbackResource) throws
- IOException {
- FileSystem fs = FileSystem.get(confdir.toUri(), systemConf);
-
- Path templatePath = new Path(confdir, templateFilename);
- return loadTemplateConfiguration(fs, templatePath, fallbackResource);
- }
-
- /**
- * looks for the config under $confdir/$templateFilename; if not there
- * loads it from /conf/templateFile.
- * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the
- * origin to help debug what's happening.
- * @param fs Filesystem
- * @param templatePath HDFS path for template
- * @param fallbackResource resource to fall back on, or "" for no fallback
- * @return loaded conf
- * @throws IOException IO problems
- * @throws FileNotFoundException if the path doesn't have a file and there
- * was no fallback.
- */
- public static Configuration loadTemplateConfiguration(FileSystem fs,
- Path templatePath,
- String fallbackResource)
- throws IOException {
- Configuration conf;
- String origin;
- if (fs.exists(templatePath)) {
- log.debug("Loading template configuration {}", templatePath);
- conf = loadConfiguration(fs, templatePath);
- origin = templatePath.toString();
- } else {
- if (fallbackResource.isEmpty()) {
- throw new FileNotFoundException("No config file found at " + templatePath);
- }
- log.debug("Template {} not found" +
- " -reverting to classpath resource {}", templatePath, fallbackResource);
- conf = new Configuration(false);
- conf.addResource(fallbackResource);
- origin = "Resource " + fallbackResource;
- }
- //force a get
- conf.get(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN);
- //now set the origin
- conf.set(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN, origin);
- return conf;
- }
-
-
- /**
- * For testing: dump a configuration
- * @param conf configuration
- * @return listing in key=value style
- */
- public static String dumpConfigToString(Configuration conf) {
- Set<String> sorted = sortedConfigKeys(conf);
-
- StringBuilder builder = new StringBuilder();
- for (String key : sorted) {
-
- builder.append(key)
- .append("=")
- .append(conf.get(key))
- .append("\n");
- }
- return builder.toString();
- }
-
- /**
- * Merge in one configuration above another
- * @param base base config
- * @param merge one to merge. This MUST be a non-default-load config to avoid
- * merge origin confusion
- * @param origin description of the origin for the put operation
- * @param overwrite flag to indicate new values should overwrite the predecessor
- * @return the base with the merged values
- */
- public static Configuration mergeConfigurations(Configuration base,
- Iterable<Map.Entry<String, String>> merge,
- String origin,
- boolean overwrite) {
- for (Map.Entry<String, String> entry : merge) {
- String key = entry.getKey();
- if (overwrite || base.get(key) == null) {
- base.set(key, entry.getValue(), origin);
- }
- }
- return base;
- }
-
- /**
- * Register a resource as a default resource.
- * Do not attempt to use this unless you understand that the
- * order in which default resources are loaded affects the outcome,
- * and that subclasses of Configuration often register new default
- * resources
- * @param resource the resource name
- * @return the URL or null
- */
- public static URL registerDefaultResource(String resource) {
- URL resURL = getResourceUrl(resource);
- if (resURL != null) {
- Configuration.addDefaultResource(resource);
- }
- return resURL;
- }
-
- /**
- * Load a configuration from a resource on this classpath.
- * If the resource is not found, an empty configuration is returned
- * @param resource the resource name
- * @return the loaded configuration.
- */
- public static Configuration loadFromResource(String resource) {
- Configuration conf = new Configuration(false);
- URL resURL = getResourceUrl(resource);
- if (resURL != null) {
- log.debug("loaded resources from {}", resURL);
- conf.addResource(resource);
- } else{
- log.debug("failed to find {} on the classpath", resource);
- }
- return conf;
-
- }
-
- /**
- * Get the URL to a resource, null if not on the CP
- * @param resource resource to look for
- * @return the URL or null
- */
- public static URL getResourceUrl(String resource) {
- return ConfigHelper.class.getClassLoader()
- .getResource(resource);
- }
-
- /**
- * Load a resource that must be on the classpath
- * @param resource the resource name
- * @return the loaded configuration
- * @throws FileNotFoundException if the resource is missing
- */
- public static Configuration loadMandatoryResource(String resource)
- throws FileNotFoundException {
- Configuration conf = new Configuration(false);
- URL resURL = getResourceUrl(resource);
- if (resURL != null) {
- log.debug("loaded resources from {}", resURL);
- conf.addResource(resource);
- } else {
- throw new FileNotFoundException(resource);
- }
- return conf;
- }
-
- /**
- * Propagate a property from a source to a dest config, with a best-effort
- * attempt at propagating the origin.
- * If the
- * @param dest destination
- * @param src source
- * @param key key to try to copy
- * @return true if the key was found and propagated
- */
- public static boolean propagate(Configuration dest,
- Configuration src,
- String key) {
- String val = src.get(key);
- if (val != null) {
- String[] origin = src.getPropertySources(key);
- if (origin != null && origin.length > 0) {
- dest.set(key, val, origin[0]);
- } else {
- dest.set(key, val);
- return true;
- }
- }
- return false;
- }
-
-
- /**
- * Take a configuration, return a hash map
- * @param conf conf
- * @return hash map
- */
- public static Map<String, String> buildMapFromConfiguration(Configuration conf) {
- Map<String, String> map = new HashMap<String, String>();
- return SliderUtils.mergeEntries(map, conf);
- }
-
- /**
- * This goes through the keyset of one configuration and retrieves each value
- * from a value source -a different or the same configuration. This triggers
- * the property resolution process of the value, resolving any variables against
- * in-config or inherited configurations
- * @param keysource source of keys
- * @param valuesource the source of values
- * @return a new configuration where <code>foreach key in keysource, get(key)==valuesource.get(key)</code>
- */
- public static Configuration resolveConfiguration(
- Iterable<Map.Entry<String, String>> keysource,
- Configuration valuesource) {
- Configuration result = new Configuration(false);
- for (Map.Entry<String, String> entry : keysource) {
- String key = entry.getKey();
- String value = valuesource.get(key);
- Preconditions.checkState(value != null,
- "no reference for \"%s\" in values", key);
- result.set(key, value);
- }
- return result;
- }
-
- /**
- * Register anything we consider deprecated
- */
- public static void registerDeprecatedConfigItems() {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf581071/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
deleted file mode 100644
index 43eb270..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ /dev/null
@@ -1,700 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import com.google.common.base.Preconditions;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.exceptions.ErrorStrings;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.exceptions.UnknownApplicationInstanceException;
-import org.apache.slider.core.persist.Filenames;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys.CLUSTER_DIRECTORY_PERMISSIONS;
-import static org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys.DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS;
-
-public class CoreFileSystem {
- private static final Logger
- log = LoggerFactory.getLogger(CoreFileSystem.class);
-
- private static final String UTF_8 = "UTF-8";
-
- protected final FileSystem fileSystem;
- protected final Configuration configuration;
-
- public CoreFileSystem(FileSystem fileSystem, Configuration configuration) {
- Preconditions.checkNotNull(fileSystem,
- "Cannot create a CoreFileSystem with a null FileSystem");
- Preconditions.checkNotNull(configuration,
- "Cannot create a CoreFileSystem with a null Configuration");
- this.fileSystem = fileSystem;
- this.configuration = configuration;
- }
-
- public CoreFileSystem(Configuration configuration) throws IOException {
- Preconditions.checkNotNull(configuration,
- "Cannot create a CoreFileSystem with a null Configuration");
- this.fileSystem = FileSystem.get(configuration);
- this.configuration = configuration;
- }
-
- /**
- * Get the temp path for this cluster
- * @param clustername name of the cluster
- * @return path for temp files (is not purged)
- */
- public Path getTempPathForCluster(String clustername) {
- Path clusterDir = buildClusterDirPath(clustername);
- return new Path(clusterDir, SliderKeys.TMP_DIR_PREFIX);
- }
-
- /**
- * Returns the underlying FileSystem for this object.
- *
- * @return filesystem
- */
- public FileSystem getFileSystem() {
- return fileSystem;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("CoreFileSystem{");
- sb.append("fileSystem=").append(fileSystem.getUri());
- sb.append('}');
- return sb.toString();
- }
-
- /**
- * Build up the path string for a cluster instance -no attempt to
- * create the directory is made
- *
- * @param clustername name of the cluster
- * @return the path for persistent data
- */
- public Path buildClusterDirPath(String clustername) {
- Preconditions.checkNotNull(clustername);
- Path path = getBaseApplicationPath();
- return new Path(path, SliderKeys.CLUSTER_DIRECTORY + "/" + clustername);
- }
-
- /**
- * Build up the path string for app def folder -no attempt to
- * create the directory is made
- *
- * @param clustername name of the cluster
- * @return the path for persistent data
- */
- public Path buildAppDefDirPath(String clustername) {
- Path path = buildClusterDirPath(clustername);
- return new Path(path, SliderKeys.APP_DEF_DIR);
- }
-
- /**
- * Build up the path string for addon folder -no attempt to
- * create the directory is made
- *
- * @param clustername name of the cluster
- * @return the path for persistent data
- */
- public Path buildAddonDirPath(String clustername, String addonId) {
- Preconditions.checkNotNull(addonId);
- Path path = buildClusterDirPath(clustername);
- return new Path(path, SliderKeys.ADDONS_DIR + "/" + addonId);
- }
-
- /**
- * Build up the path string for keytab install location -no attempt to
- * create the directory is made
- *
- * @return the path for keytab
- */
- public Path buildKeytabInstallationDirPath(String keytabFolder) {
- Preconditions.checkNotNull(keytabFolder);
- Path path = getBaseApplicationPath();
- return new Path(path, SliderKeys.KEYTAB_DIR + "/" + keytabFolder);
- }
-
- /**
- * Build up the path string for keytab install location -no attempt to
- * create the directory is made
- *
- * @return the path for keytab installation location
- */
- public Path buildKeytabPath(String keytabDir, String keytabName, String clusterName) {
- Path homePath = getHomeDirectory();
- Path baseKeytabDir;
- if (keytabDir != null) {
- baseKeytabDir = new Path(homePath, keytabDir);
- } else {
- baseKeytabDir = new Path(buildClusterDirPath(clusterName),
- SliderKeys.KEYTAB_DIR);
- }
- return keytabName == null ? baseKeytabDir :
- new Path(baseKeytabDir, keytabName);
- }
-
- /**
- * Build up the path string for resource install location -no attempt to
- * create the directory is made
- *
- * @return the path for resource
- */
- public Path buildResourcePath(String resourceFolder) {
- Preconditions.checkNotNull(resourceFolder);
- Path path = getBaseApplicationPath();
- return new Path(path, SliderKeys.RESOURCE_DIR + "/" + resourceFolder);
- }
-
- /**
- * Build up the path string for resource install location -no attempt to
- * create the directory is made
- *
- * @return the path for resource
- */
- public Path buildResourcePath(String dirName, String fileName) {
- Preconditions.checkNotNull(dirName);
- Preconditions.checkNotNull(fileName);
- Path path = getBaseApplicationPath();
- return new Path(path, SliderKeys.RESOURCE_DIR + "/" + dirName + "/" + fileName);
- }
-
- /**
- * Create a directory with the given permissions.
- *
- * @param dir directory
- * @param clusterPerms cluster permissions
- * @throws IOException IO problem
- * @throws BadClusterStateException any cluster state problem
- */
- public void createWithPermissions(Path dir, FsPermission clusterPerms) throws
- IOException,
- BadClusterStateException {
- if (fileSystem.isFile(dir)) {
- // HADOOP-9361 shows some filesystems don't correctly fail here
- throw new BadClusterStateException(
- "Cannot create a directory over a file %s", dir);
- }
- log.debug("mkdir {} with perms {}", dir, clusterPerms);
- //no mask whatoever
- fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000");
- fileSystem.mkdirs(dir, clusterPerms);
- //and force set it anyway just to make sure
- fileSystem.setPermission(dir, clusterPerms);
- }
-
- /**
- * Get the permissions of a path
- *
- * @param path path to check
- * @return the permissions
- * @throws IOException any IO problem (including file not found)
- */
- public FsPermission getPathPermissions(Path path) throws IOException {
- FileStatus status = fileSystem.getFileStatus(path);
- return status.getPermission();
- }
-
- public FsPermission getInstanceDirectoryPermissions() {
- String clusterDirPermsOct =
- configuration.get(CLUSTER_DIRECTORY_PERMISSIONS,
- DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS);
- return new FsPermission(clusterDirPermsOct);
- }
-
- /**
- * Verify that the cluster directory is not present
- *
- * @param clustername name of the cluster
- * @param clusterDirectory actual directory to look for
- * @throws IOException trouble with FS
- * @throws SliderException If the directory exists
- */
- public void verifyClusterDirectoryNonexistent(String clustername,
- Path clusterDirectory)
- throws IOException, SliderException {
- if (fileSystem.exists(clusterDirectory)) {
- throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS,
- ErrorStrings.PRINTF_E_INSTANCE_ALREADY_EXISTS, clustername,
- clusterDirectory);
- }
- }
- /**
- * Verify that the given directory is not present
- *
- * @param clusterDirectory actual directory to look for
- * @throws IOException trouble with FS
- * @throws SliderException If the directory exists
- */
- public void verifyDirectoryNonexistent(Path clusterDirectory) throws
- IOException,
- SliderException {
- if (fileSystem.exists(clusterDirectory)) {
-
- log.error("Dir {} exists: {}",
- clusterDirectory,
- listFSDir(clusterDirectory));
- throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS,
- ErrorStrings.PRINTF_E_INSTANCE_DIR_ALREADY_EXISTS,
- clusterDirectory);
- }
- }
-
- /**
- * Verify that a user has write access to a directory.
- * It does this by creating then deleting a temp file
- *
- * @param dirPath actual directory to look for
- * @throws FileNotFoundException file not found
- * @throws IOException trouble with FS
- * @throws BadClusterStateException if the directory is not writeable
- */
- public void verifyDirectoryWriteAccess(Path dirPath) throws IOException,
- SliderException {
- verifyPathExists(dirPath);
- Path tempFile = new Path(dirPath, "tmp-file-for-checks");
- try {
- FSDataOutputStream out ;
- out = fileSystem.create(tempFile, true);
- IOUtils.closeStream(out);
- fileSystem.delete(tempFile, false);
- } catch (IOException e) {
- log.warn("Failed to create file {}: {}", tempFile, e);
- throw new BadClusterStateException(e,
- "Unable to write to directory %s : %s", dirPath, e.toString());
- }
- }
-
- /**
- * Verify that a path exists
- * @param path path to check
- * @throws FileNotFoundException file not found
- * @throws IOException trouble with FS
- */
- public void verifyPathExists(Path path) throws IOException {
- if (!fileSystem.exists(path)) {
- throw new FileNotFoundException(path.toString());
- }
- }
-
- /**
- * Verify that a path exists
- * @param path path to check
- * @throws FileNotFoundException file not found or is not a file
- * @throws IOException trouble with FS
- */
- public void verifyFileExists(Path path) throws IOException {
- FileStatus status = fileSystem.getFileStatus(path);
-
- if (!status.isFile()) {
- throw new FileNotFoundException("Not a file: " + path.toString());
- }
- }
-
- /**
- * Given a path, check if it exists and is a file
- *
- * @param path
- * absolute path to the file to check
- * @returns true if and only if path exists and is a file, false for all other
- * reasons including if file check throws IOException
- */
- public boolean isFile(Path path) {
- boolean isFile = false;
- try {
- FileStatus status = fileSystem.getFileStatus(path);
- if (status.isFile()) {
- isFile = true;
- }
- } catch (IOException e) {
- // ignore, isFile is already set to false
- }
- return isFile;
- }
-
- /**
- * Get the base path
- *
- * @return the base path optionally configured by
- * {@link SliderXmlConfKeys#KEY_SLIDER_BASE_PATH}
- */
- public Path getBaseApplicationPath() {
- String configuredBasePath = configuration.get(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH);
- return configuredBasePath != null ? new Path(configuredBasePath) :
- new Path(getHomeDirectory(), SliderKeys.SLIDER_BASE_DIRECTORY);
- }
-
- /**
- * Get slider dependency parent dir in HDFS
- *
- * @return the parent dir path of slider.tar.gz in HDFS
- */
- public Path getDependencyPath() {
- // FIXME: 3/20/17 HDP ???????????
- String parentDir = (SliderUtils.isHdp()) ? SliderKeys.SLIDER_DEPENDENCY_HDP_PARENT_DIR
- + SliderKeys.SLIDER_DEPENDENCY_DIR
- : SliderKeys.SLIDER_DEPENDENCY_DIR;
- Path dependencyPath = new Path(String.format(parentDir,
- SliderUtils.getSliderVersion()));
- return dependencyPath;
- }
-
- /**
- * Get slider.tar.gz absolute filepath in HDFS
- *
- * @return the absolute path to slider.tar.gz in HDFS
- */
- public Path getDependencyTarGzip() {
- Path dependencyLibAmPath = getDependencyPath();
- Path dependencyLibTarGzip = new Path(
- dependencyLibAmPath.toUri().toString(),
- SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME
- + SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT);
- return dependencyLibTarGzip;
- }
-
- public Path getHomeDirectory() {
- return fileSystem.getHomeDirectory();
- }
-
- public boolean maybeAddImagePath(Map<String, LocalResource> localResources,
- Path imagePath) throws IOException {
- if (imagePath != null) {
- LocalResource resource = createAmResource(imagePath,
- LocalResourceType.ARCHIVE);
- localResources.put(SliderKeys.LOCAL_TARBALL_INSTALL_SUBDIR, resource);
- return true;
- } else {
- return false;
- }
- }
-
- public boolean maybeAddImagePath(Map<String, LocalResource> localResources,
- String imagePath) throws IOException {
-
- return imagePath != null &&
- maybeAddImagePath(localResources, new Path(imagePath));
- }
-
-
-
-
- /**
- * Create an AM resource from the
- *
- * @param destPath dest path in filesystem
- * @param resourceType resource type
- * @return the resource set up wih application-level visibility and the
- * timestamp & size set from the file stats.
- */
- public LocalResource createAmResource(Path destPath, LocalResourceType resourceType) throws IOException {
- FileStatus destStatus = fileSystem.getFileStatus(destPath);
- LocalResource amResource = Records.newRecord(LocalResource.class);
- amResource.setType(resourceType);
- // Set visibility of the resource
- // Setting to most private option
- amResource.setVisibility(LocalResourceVisibility.APPLICATION);
- // Set the resource to be copied over
- amResource.setResource(ConverterUtils.getYarnUrlFromPath(fileSystem
- .resolvePath(destStatus.getPath())));
- // Set timestamp and length of file so that the framework
- // can do basic sanity checks for the local resource
- // after it has been copied over to ensure it is the same
- // resource the client intended to use with the application
- amResource.setTimestamp(destStatus.getModificationTime());
- amResource.setSize(destStatus.getLen());
- return amResource;
- }
-
- /**
- * Register all files under a fs path as a directory to push out
- *
- * @param srcDir src dir
- * @param destRelativeDir dest dir (no trailing /)
- * @return the map of entries
- */
- public Map<String, LocalResource> submitDirectory(Path srcDir, String destRelativeDir) throws IOException {
- //now register each of the files in the directory to be
- //copied to the destination
- FileStatus[] fileset = fileSystem.listStatus(srcDir);
- Map<String, LocalResource> localResources =
- new HashMap<String, LocalResource>(fileset.length);
- for (FileStatus entry : fileset) {
-
- LocalResource resource = createAmResource(entry.getPath(),
- LocalResourceType.FILE);
- String relativePath = destRelativeDir + "/" + entry.getPath().getName();
- localResources.put(relativePath, resource);
- }
- return localResources;
- }
-
- /**
- * Submit a JAR containing a specific class, returning
- * the resource to be mapped in
- *
- * @param clazz class to look for
- * @param subdir subdirectory (expected to end in a "/")
- * @param jarName <i>At the destination</i>
- * @return the local resource ref
- * @throws IOException trouble copying to HDFS
- */
- public LocalResource submitJarWithClass(Class clazz, Path tempPath, String subdir, String jarName)
- throws IOException, SliderException {
- File localFile = SliderUtils.findContainingJarOrFail(clazz);
- return submitFile(localFile, tempPath, subdir, jarName);
- }
-
- /**
- * Submit a local file to the filesystem references by the instance's cluster
- * filesystem
- *
- * @param localFile filename
- * @param subdir subdirectory (expected to end in a "/")
- * @param destFileName destination filename
- * @return the local resource ref
- * @throws IOException trouble copying to HDFS
- */
- public LocalResource submitFile(File localFile, Path tempPath, String subdir, String destFileName)
- throws IOException {
- Path src = new Path(localFile.toString());
- Path subdirPath = new Path(tempPath, subdir);
- fileSystem.mkdirs(subdirPath);
- Path destPath = new Path(subdirPath, destFileName);
- log.debug("Copying {} (size={} bytes) to {}", localFile, localFile.length(), destPath);
-
- fileSystem.copyFromLocalFile(false, true, src, destPath);
-
- // Set the type of resource - file or archive
- // archives are untarred at destination
- // we don't need the jar file to be untarred for now
- return createAmResource(destPath, LocalResourceType.FILE);
- }
-
- /**
- * Submit the AM tar.gz resource referenced by the instance's cluster
- * filesystem. Also, update the providerResources object with the new
- * resource.
- *
- * @param providerResources
- * the provider resource map to be updated
- * @throws IOException
- * trouble copying to HDFS
- */
- public void submitTarGzipAndUpdate(
- Map<String, LocalResource> providerResources) throws IOException,
- BadClusterStateException {
- Path dependencyLibTarGzip = getDependencyTarGzip();
- LocalResource lc = createAmResource(dependencyLibTarGzip,
- LocalResourceType.ARCHIVE);
- providerResources.put(SliderKeys.SLIDER_DEPENDENCY_LOCALIZED_DIR_LINK, lc);
- }
-
- public void copyLocalFileToHdfs(File localPath,
- Path destPath, FsPermission fp)
- throws IOException {
- if (localPath == null || destPath == null) {
- throw new IOException("Either localPath or destPath is null");
- }
- fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,
- "000");
- fileSystem.mkdirs(destPath.getParent(), fp);
- log.info("Copying file {} to {}", localPath.toURI(), destPath.toUri());
-
- fileSystem.copyFromLocalFile(false, true, new Path(localPath.getPath()),
- destPath);
- // set file permissions of the destPath
- fileSystem.setPermission(destPath, fp);
- }
-
- public void copyHdfsFileToLocal(Path hdfsPath, File destFile)
- throws IOException {
- if (hdfsPath == null || destFile == null) {
- throw new IOException("Either hdfsPath or destPath is null");
- }
- log.info("Copying file {} to {}", hdfsPath.toUri(), destFile.toURI());
-
- Path destPath = new Path(destFile.getPath());
- fileSystem.copyToLocalFile(hdfsPath, destPath);
- }
-
- /**
- * list entries in a filesystem directory
- *
- * @param path directory
- * @return a listing, one to a line
- * @throws IOException
- */
- public String listFSDir(Path path) throws IOException {
- FileStatus[] stats = fileSystem.listStatus(path);
- StringBuilder builder = new StringBuilder();
- for (FileStatus stat : stats) {
- builder.append(stat.getPath().toString())
- .append("\t")
- .append(stat.getLen())
- .append("\n");
- }
- return builder.toString();
- }
-
- /**
- * List all application instances persisted for this user, giving the
- * path. The instance name is the last element in the path
- * @return a possibly empty map of application instance names to paths
- */
- public Map<String, Path> listPersistentInstances() throws IOException {
- FileSystem fs = getFileSystem();
- Path path = new Path(getBaseApplicationPath(), SliderKeys.CLUSTER_DIRECTORY);
- log.debug("Looking for all persisted application at {}", path.toString());
- if (!fs.exists(path)) {
- // special case: no instances have ever been created
- return new HashMap<String, Path>(0);
- }
- FileStatus[] statuses = fs.listStatus(path);
- Map<String, Path> instances = new HashMap<String, Path>(statuses.length);
-
- // enum the child entries
- for (FileStatus status : statuses) {
- if (status.isDirectory()) {
- // for directories, look for an internal.json underneath
- Path child = status.getPath();
- Path internalJson = new Path(child, Filenames.INTERNAL);
- if (fs.exists(internalJson)) {
- // success => this is an instance
- instances.put(child.getName(), child);
- } else {
- log.info("Malformed cluster found at {}. It does not appear to be a valid persisted instance.",
- child.toString());
- }
- }
- }
- return instances;
- }
-
- public void touch(Path path, boolean overwrite) throws IOException {
- FSDataOutputStream out = null;
- try {
- out = fileSystem.create(path, overwrite);
- } finally {
- IOUtils.closeStream(out);
- }
- }
-
- public void cat(Path path, boolean overwrite, String data) throws IOException {
- FSDataOutputStream out = null;
- try {
- out = fileSystem.create(path, overwrite);
- byte[] bytes = data.getBytes(Charset.forName("UTF-8"));
- out.write(bytes);
- } finally {
- IOUtils.closeStream(out);
- }
- }
-
- public String cat(Path path) throws IOException {
- FileStatus status = fileSystem.getFileStatus(path);
- byte[] b = new byte[(int) status.getLen()];
- FSDataInputStream in = null;
- try {
- in = fileSystem.open(path);
- int count = in.read(b);
- return new String(b, 0, count, UTF_8);
- } finally {
- IOUtils.closeStream(in);
- }
- }
-
- /**
- * Create a path that must exist in the cluster fs
- * @param uri uri to create
- * @return the path
- * @throws SliderException if the path does not exist
- */
- public Path createPathThatMustExist(String uri) throws
- SliderException, IOException {
- Preconditions.checkNotNull(uri);
- Path path = new Path(uri);
- verifyPathExists(path);
- return path;
- }
-
- /**
- * Locate an application conf json in the FS. This includes a check to verify
- * that the file is there.
- *
- * @param clustername name of the cluster
- * @return the path to the spec.
- * @throws IOException IO problems
- * @throws SliderException if the path isn't there
- */
- public Path locateInstanceDefinition(String clustername) throws IOException,
- SliderException {
- Path clusterDirectory = buildClusterDirPath(clustername);
- Path appConfPath =
- new Path(clusterDirectory, Filenames.APPCONF);
- verifyClusterSpecExists(clustername, appConfPath);
- return appConfPath;
- }
-
- /**
- * Verify that a cluster specification exists
- * @param clustername name of the cluster (For errors only)
- * @param clusterSpecPath cluster specification path
- * @throws IOException IO problems
- * @throws SliderException if the cluster specification is not present
- */
- public void verifyClusterSpecExists(String clustername, Path clusterSpecPath)
- throws IOException,
- SliderException {
- if (!fileSystem.isFile(clusterSpecPath)) {
- log.debug("Missing specification file {}", clusterSpecPath);
- throw UnknownApplicationInstanceException.unknownInstance(
- clustername + "\n (definition not found at " + clusterSpecPath);
- }
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf581071/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Duration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Duration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Duration.java
deleted file mode 100644
index e5fa424..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Duration.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import java.io.Closeable;
-
-/**
- * A duration in milliseconds. This class can be used
- * to count time, and to be polled to see if a time limit has
- * passed.
- */
-public class Duration implements Closeable {
- public long start, finish;
- public final long limit;
-
- /**
- * Create a duration instance with a limit of 0
- */
- public Duration() {
- this(0);
- }
-
- /**
- * Create a duration with a limit specified in millis
- * @param limit duration in milliseconds
- */
- public Duration(long limit) {
- this.limit = limit;
- }
-
- /**
- * Start
- * @return self
- */
- public Duration start() {
- start = now();
- return this;
- }
-
- /**
- * The close operation relays to {@link #finish()}.
- * Implementing it allows Duration instances to be automatically
- * finish()'d in Java7 try blocks for when used in measuring durations.
- */
- @Override
- public final void close() {
- finish();
- }
-
- public void finish() {
- finish = now();
- }
-
- protected long now() {
- return System.nanoTime()/1000000;
- }
-
- public long getInterval() {
- return finish - start;
- }
-
- /**
- * return true if the limit has been exceeded
- * @return true if a limit was set and the current time
- * exceeds it.
- */
- public boolean getLimitExceeded() {
- return limit >= 0 && ((now() - start) > limit);
- }
-
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder();
- builder.append("Duration");
- if (finish >= start) {
- builder.append(" finished at ").append(getInterval()).append(" millis;");
- } else {
- if (start > 0) {
- builder.append(" started but not yet finished;");
- } else {
- builder.append(" unstarted;");
- }
- }
- if (limit > 0) {
- builder.append(" limit: ").append(limit).append(" millis");
- if (getLimitExceeded()) {
- builder.append(" - exceeded");
- }
- }
- return builder.toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf581071/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
deleted file mode 100644
index 235d3da..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.common.tools;
-
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.SliderException;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * a scanner which can take an input string for a range or scan the lot.
- */
-public class PortScanner {
- private static Pattern NUMBER_RANGE = Pattern.compile("^(\\d+)\\s*-\\s*(\\d+)$");
- private static Pattern SINGLE_NUMBER = Pattern.compile("^\\d+$");
-
- private List<Integer> remainingPortsToCheck;
-
- public PortScanner() {
- }
-
- public void setPortRange(String input) throws BadConfigException {
- // first split based on commas
- Set<Integer> inputPorts= new TreeSet<Integer>();
- String[] ranges = input.split(",");
- for ( String range : ranges ) {
- if (range.trim().isEmpty()) {
- continue;
- }
- Matcher m = SINGLE_NUMBER.matcher(range.trim());
- if (m.find()) {
- inputPorts.add(Integer.parseInt(m.group()));
- continue;
- }
- m = NUMBER_RANGE.matcher(range.trim());
- if (m.find()) {
- String[] boundaryValues = m.group(0).split("-");
- int start = Integer.parseInt(boundaryValues[0].trim());
- int end = Integer.parseInt(boundaryValues[1].trim());
- if (end < start) {
- throw new BadConfigException("End of port range is before start: "
- + range + " in input: " + input);
- }
- for (int i = start; i < end + 1; i++) {
- inputPorts.add(i);
- }
- continue;
- }
- throw new BadConfigException("Bad port range: " + range + " in input: "
- + input);
- }
- if (inputPorts.size() == 0) {
- throw new BadConfigException("No ports found in range: " + input);
- }
- this.remainingPortsToCheck = new ArrayList<Integer>(inputPorts);
- }
-
- public List<Integer> getRemainingPortsToCheck() {
- return remainingPortsToCheck;
- }
-
- public int getAvailablePort() throws SliderException, IOException {
- if (remainingPortsToCheck != null) {
- return getAvailablePortViaPortArray();
- } else {
- return SliderUtils.getOpenPort();
- }
- }
-
- private int getAvailablePortViaPortArray() throws SliderException {
- boolean found = false;
- int availablePort = -1;
- Iterator<Integer> portsToCheck = this.remainingPortsToCheck.iterator();
- while (portsToCheck.hasNext() && !found) {
- int portToCheck = portsToCheck.next();
- found = SliderUtils.isPortAvailable(portToCheck);
- if (found) {
- availablePort = portToCheck;
- portsToCheck.remove();
- }
- }
-
- if (availablePort < 0) {
- throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
- "No available ports found in configured range {}",
- remainingPortsToCheck);
- }
-
- return availablePort;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf581071/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderFileSystem.java
deleted file mode 100644
index 40b07bd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderFileSystem.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-import java.io.IOException;
-
-/**
- * Extends Core Filesystem with operations to manipulate ClusterDescription
- * persistent state
- */
-public class SliderFileSystem extends CoreFileSystem {
-
- Path appDir = null;
-
- public SliderFileSystem(FileSystem fileSystem,
- Configuration configuration) {
- super(fileSystem, configuration);
- }
-
- public SliderFileSystem(Configuration configuration) throws IOException {
- super(configuration);
- }
-
- public void setAppDir(Path appDir) {
- this.appDir = appDir;
- }
-
- public Path getAppDir() {
- return this.appDir;
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org