You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2016/12/22 19:31:27 UTC
[01/51] [abbrv] hadoop git commit: YARN-5513. Move Java only tests
from slider develop to yarn-native-services. Contributed by Gour Saha [Forced
Update!]
Repository: hadoop
Updated Branches:
refs/heads/yarn-native-services 27a13ae7b -> e9716f597 (forced update)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/ContractTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/ContractTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/ContractTestUtils.java
new file mode 100644
index 0000000..7eaaefe
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/ContractTestUtils.java
@@ -0,0 +1,901 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.test;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.Assert;
+import org.junit.internal.AssumptionViolatedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.Properties;
+import java.util.UUID;
+
+/**
+ * Utilities used across test cases to make assertions about filesystems
+ * -assertions which fail with useful information.
+ * This is lifted from Hadoop common Test; that JAR isn't published, so
+ * we have to make do.
+ */
+public class ContractTestUtils extends Assert {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContractTestUtils.class);
+
+ public static final String IO_FILE_BUFFER_SIZE = "io.file.buffer.size";
+
+ // For scale testing, we can repeatedly write small chunk data to generate
+ // a large file.
+ public static final String IO_CHUNK_BUFFER_SIZE = "io.chunk.buffer.size";
+ public static final int DEFAULT_IO_CHUNK_BUFFER_SIZE = 128;
+ public static final String IO_CHUNK_MODULUS_SIZE = "io.chunk.modulus.size";
+ public static final int DEFAULT_IO_CHUNK_MODULUS_SIZE = 128;
+
+ /**
+ * Assert that a property in the property set matches the expected value
+ * @param props property set
+ * @param key property name
+ * @param expected expected value. If null, the property must not be in the set
+ */
+ public static void assertPropertyEquals(Properties props,
+ String key,
+ String expected) {
+ String val = props.getProperty(key);
+ if (expected == null) {
+ assertNull("Non null property " + key + " = " + val, val);
+ } else {
+ assertEquals("property " + key + " = " + val,
+ expected,
+ val);
+ }
+ }
+
+ /**
+ *
+ * Write a file and read it in, validating the result. Optional flags control
+ * whether file overwrite operations should be enabled, and whether the
+ * file should be deleted afterwards.
+ *
+ * If there is a mismatch between what was written and what was expected,
+ * a small range of bytes either side of the first error are logged to aid
+ * diagnosing what problem occurred -whether it was a previous file
+ * or a corrupting of the current file. This assumes that two
+ * sequential runs to the same path use datasets with different character
+ * moduli.
+ *
+ * @param fs filesystem
+ * @param path path to write to
+ * @param len length of data
+ * @param overwrite should the create option allow overwrites?
+ * @param delete should the file be deleted afterwards? -with a verification
+ * that it worked. Deletion is not attempted if an assertion has failed
+ * earlier -it is not in a <code>finally{}</code> block.
+ * @throws IOException IO problems
+ */
+ public static void writeAndRead(FileSystem fs,
+ Path path,
+ byte[] src,
+ int len,
+ int blocksize,
+ boolean overwrite,
+ boolean delete) throws IOException {
+ fs.mkdirs(path.getParent());
+
+ writeDataset(fs, path, src, len, blocksize, overwrite);
+
+ byte[] dest = readDataset(fs, path, len);
+
+ compareByteArrays(src, dest, len);
+
+ if (delete) {
+ rejectRootOperation(path);
+ boolean deleted = fs.delete(path, false);
+ assertTrue("Deleted", deleted);
+ assertPathDoesNotExist(fs, "Cleanup failed", path);
+ }
+ }
+
+ /**
+ * Write a file.
+ * Optional flags control
+ * whether file overwrite operations should be enabled
+ * @param fs filesystem
+ * @param path path to write to
+ * @param len length of data
+ * @param overwrite should the create option allow overwrites?
+ * @throws IOException IO problems
+ */
+ public static void writeDataset(FileSystem fs,
+ Path path,
+ byte[] src,
+ int len,
+ int buffersize,
+ boolean overwrite) throws IOException {
+ assertTrue(
+ "Not enough data in source array to write " + len + " bytes",
+ src.length >= len);
+ FSDataOutputStream out = fs.create(path,
+ overwrite,
+ fs.getConf()
+ .getInt(IO_FILE_BUFFER_SIZE,
+ 4096),
+ (short) 1,
+ buffersize);
+ out.write(src, 0, len);
+ out.close();
+ assertFileHasLength(fs, path, len);
+ }
+
+ /**
+ * Read the file and convert to a byte dataset.
+ * This implements readfully internally, so that it will read
+ * in the file without ever having to seek()
+ * @param fs filesystem
+ * @param path path to read from
+ * @param len length of data to read
+ * @return the bytes
+ * @throws IOException IO problems
+ */
+ public static byte[] readDataset(FileSystem fs, Path path, int len)
+ throws IOException {
+ FSDataInputStream in = fs.open(path);
+ byte[] dest = new byte[len];
+ int offset =0;
+ int nread = 0;
+ try {
+ while (nread < len) {
+ int nbytes = in.read(dest, offset + nread, len - nread);
+ if (nbytes < 0) {
+ throw new EOFException("End of file reached before reading fully.");
+ }
+ nread += nbytes;
+ }
+ } finally {
+ in.close();
+ }
+ return dest;
+ }
+
+ /**
+ * Read a file, verify its length and contents match the expected array
+ * @param fs filesystem
+ * @param path path to file
+ * @param original original dataset
+ * @throws IOException IO Problems
+ */
+ public static void verifyFileContents(FileSystem fs,
+ Path path,
+ byte[] original) throws IOException {
+ FileStatus stat = fs.getFileStatus(path);
+ String statText = stat.toString();
+ assertTrue("not a file " + statText, stat.isFile());
+ assertEquals("wrong length " + statText, original.length, stat.getLen());
+ byte[] bytes = readDataset(fs, path, original.length);
+ compareByteArrays(original,bytes,original.length);
+ }
+
+ /**
+ * Verify that the read at a specific offset in a stream
+ * matches that expected
+ * @param stm stream
+ * @param fileContents original file contents
+ * @param seekOff seek offset
+ * @param toRead number of bytes to read
+ * @throws IOException IO problems
+ */
+ public static void verifyRead(FSDataInputStream stm, byte[] fileContents,
+ int seekOff, int toRead) throws IOException {
+ byte[] out = new byte[toRead];
+ stm.seek(seekOff);
+ stm.readFully(out);
+ byte[] expected = Arrays.copyOfRange(fileContents, seekOff,
+ seekOff + toRead);
+ compareByteArrays(expected, out,toRead);
+ }
+
+ /**
+ * Assert that tthe array original[0..len] and received[] are equal.
+ * A failure triggers the logging of the bytes near where the first
+ * difference surfaces.
+ * @param original source data
+ * @param received actual
+ * @param len length of bytes to compare
+ */
+ public static void compareByteArrays(byte[] original,
+ byte[] received,
+ int len) {
+ assertEquals("Number of bytes read != number written",
+ len, received.length);
+ int errors = 0;
+ int first_error_byte = -1;
+ for (int i = 0; i < len; i++) {
+ if (original[i] != received[i]) {
+ if (errors == 0) {
+ first_error_byte = i;
+ }
+ errors++;
+ }
+ }
+
+ if (errors > 0) {
+ String message = String.format(" %d errors in file of length %d",
+ errors, len);
+ LOG.warn(message);
+ // the range either side of the first error to print
+ // this is a purely arbitrary number, to aid user debugging
+ final int overlap = 10;
+ for (int i = Math.max(0, first_error_byte - overlap);
+ i < Math.min(first_error_byte + overlap, len);
+ i++) {
+ byte actual = received[i];
+ byte expected = original[i];
+ String letter = toChar(actual);
+ String line = String.format("[%04d] %2x %s\n", i, actual, letter);
+ if (expected != actual) {
+ line = String.format("[%04d] %2x %s -expected %2x %s\n",
+ i,
+ actual,
+ letter,
+ expected,
+ toChar(expected));
+ }
+ LOG.warn(line);
+ }
+ fail(message);
+ }
+ }
+
+ /**
+ * Convert a byte to a character for printing. If the
+ * byte value is < 32 -and hence unprintable- the byte is
+ * returned as a two digit hex value
+ * @param b byte
+ * @return the printable character string
+ */
+ public static String toChar(byte b) {
+ if (b >= 0x20) {
+ return Character.toString((char) b);
+ } else {
+ return String.format("%02x", b);
+ }
+ }
+
+ /**
+ * Convert a buffer to a string, character by character
+ * @param buffer input bytes
+ * @return a string conversion
+ */
+ public static String toChar(byte[] buffer) {
+ StringBuilder builder = new StringBuilder(buffer.length);
+ for (byte b : buffer) {
+ builder.append(toChar(b));
+ }
+ return builder.toString();
+ }
+
+ public static byte[] toAsciiByteArray(String s) {
+ char[] chars = s.toCharArray();
+ int len = chars.length;
+ byte[] buffer = new byte[len];
+ for (int i = 0; i < len; i++) {
+ buffer[i] = (byte) (chars[i] & 0xff);
+ }
+ return buffer;
+ }
+
+ /**
+ * Cleanup at the end of a test run
+ * @param action action triggering the operation (for use in logging)
+ * @param fileSystem filesystem to work with. May be null
+ * @param cleanupPath path to delete as a string
+ */
+ public static void cleanup(String action,
+ FileSystem fileSystem,
+ String cleanupPath) {
+ if (fileSystem == null) {
+ return;
+ }
+ Path path = new Path(cleanupPath).makeQualified(fileSystem.getUri(),
+ fileSystem.getWorkingDirectory());
+ cleanup(action, fileSystem, path);
+ }
+
+ /**
+ * Cleanup at the end of a test run
+ * @param action action triggering the operation (for use in logging)
+ * @param fileSystem filesystem to work with. May be null
+ * @param path path to delete
+ */
+ public static void cleanup(String action, FileSystem fileSystem, Path path) {
+ noteAction(action);
+ try {
+ rm(fileSystem, path, true, false);
+ } catch (Exception e) {
+ LOG.error("Error deleting in "+ action + " - " + path + ": " + e, e);
+ }
+ }
+
+ /**
+ * Delete a directory. There's a safety check for operations against the
+ * root directory -these are intercepted and rejected with an IOException
+ * unless the allowRootDelete flag is true
+ * @param fileSystem filesystem to work with. May be null
+ * @param path path to delete
+ * @param recursive flag to enable recursive delete
+ * @param allowRootDelete can the root directory be deleted?
+ * @throws IOException on any problem.
+ */
+ public static boolean rm(FileSystem fileSystem,
+ Path path,
+ boolean recursive,
+ boolean allowRootDelete) throws
+ IOException {
+ if (fileSystem != null) {
+ rejectRootOperation(path, allowRootDelete);
+ if (fileSystem.exists(path)) {
+ return fileSystem.delete(path, recursive);
+ }
+ }
+ return false;
+
+ }
+
+ /**
+ * Block any operation on the root path. This is a safety check
+ * @param path path in the filesystem
+ * @param allowRootOperation can the root directory be manipulated?
+ * @throws IOException if the operation was rejected
+ */
+ public static void rejectRootOperation(Path path,
+ boolean allowRootOperation) throws IOException {
+ if (path.isRoot() && !allowRootOperation) {
+ throw new IOException("Root directory operation rejected: " + path);
+ }
+ }
+
+ /**
+ * Block any operation on the root path. This is a safety check
+ * @param path path in the filesystem
+ * @throws IOException if the operation was rejected
+ */
+ public static void rejectRootOperation(Path path) throws IOException {
+ rejectRootOperation(path, false);
+ }
+
+
+ public static void noteAction(String action) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("============== "+ action +" =============");
+ }
+ }
+
+ /**
+ * downgrade a failure to a message and a warning, then an
+ * exception for the Junit test runner to mark as failed
+ * @param message text message
+ * @param failure what failed
+ * @throws AssumptionViolatedException always
+ */
+ public static void downgrade(String message, Throwable failure) {
+ LOG.warn("Downgrading test " + message, failure);
+ AssumptionViolatedException ave =
+ new AssumptionViolatedException(failure, null);
+ throw ave;
+ }
+
+ /**
+ * report an overridden test as unsupported
+ * @param message message to use in the text
+ * @throws AssumptionViolatedException always
+ */
+ public static void unsupported(String message) {
+ skip(message);
+ }
+
+ /**
+ * report a test has been skipped for some reason
+ * @param message message to use in the text
+ * @throws AssumptionViolatedException always
+ */
+ public static void skip(String message) {
+ LOG.info("Skipping: {}", message);
+ throw new AssumptionViolatedException(message);
+ }
+
+ /**
+ * Fail with an exception that was received
+ * @param text text to use in the exception
+ * @param thrown a (possibly null) throwable to init the cause with
+ * @throws AssertionError with the text and throwable -always
+ */
+ public static void fail(String text, Throwable thrown) {
+ AssertionError e = new AssertionError(text);
+ e.initCause(thrown);
+ throw e;
+ }
+
+ /**
+ * Make an assertion about the length of a file
+ * @param fs filesystem
+ * @param path path of the file
+ * @param expected expected length
+ * @throws IOException on File IO problems
+ */
+ public static void assertFileHasLength(FileSystem fs, Path path,
+ int expected) throws IOException {
+ FileStatus status = fs.getFileStatus(path);
+ assertEquals(
+ "Wrong file length of file " + path + " status: " + status,
+ expected,
+ status.getLen());
+ }
+
+ /**
+ * Assert that a path refers to a directory
+ * @param fs filesystem
+ * @param path path of the directory
+ * @throws IOException on File IO problems
+ */
+ public static void assertIsDirectory(FileSystem fs,
+ Path path) throws IOException {
+ FileStatus fileStatus = fs.getFileStatus(path);
+ assertIsDirectory(fileStatus);
+ }
+
+ /**
+ * Assert that a path refers to a directory
+ * @param fileStatus stats to check
+ */
+ public static void assertIsDirectory(FileStatus fileStatus) {
+ assertTrue("Should be a directory -but isn't: " + fileStatus,
+ fileStatus.isDirectory());
+ }
+
+ /**
+ * Write the text to a file, returning the converted byte array
+ * for use in validating the round trip
+ * @param fs filesystem
+ * @param path path of file
+ * @param text text to write
+ * @param overwrite should the operation overwrite any existing file?
+ * @return the read bytes
+ * @throws IOException on IO problems
+ */
+ public static byte[] writeTextFile(FileSystem fs,
+ Path path,
+ String text,
+ boolean overwrite) throws IOException {
+ byte[] bytes = new byte[0];
+ if (text != null) {
+ bytes = toAsciiByteArray(text);
+ }
+ createFile(fs, path, overwrite, bytes);
+ return bytes;
+ }
+
+ /**
+ * Create a file
+ * @param fs filesystem
+ * @param path path to write
+ * @param overwrite overwrite flag
+ * @param data source dataset. Can be null
+ * @throws IOException on any problem
+ */
+ public static void createFile(FileSystem fs,
+ Path path,
+ boolean overwrite,
+ byte[] data) throws IOException {
+ FSDataOutputStream stream = fs.create(path, overwrite);
+ if (data != null && data.length > 0) {
+ stream.write(data);
+ }
+ stream.close();
+ }
+
+ /**
+ * Touch a file
+ * @param fs filesystem
+ * @param path path
+ * @throws IOException IO problems
+ */
+ public static void touch(FileSystem fs,
+ Path path) throws IOException {
+ createFile(fs, path, true, null);
+ }
+
+ /**
+ * Delete a file/dir and assert that delete() returned true
+ * <i>and</i> that the path no longer exists. This variant rejects
+ * all operations on root directories
+ * @param fs filesystem
+ * @param file path to delete
+ * @param recursive flag to enable recursive delete
+ * @throws IOException IO problems
+ */
+ public static void assertDeleted(FileSystem fs,
+ Path file,
+ boolean recursive) throws IOException {
+ assertDeleted(fs, file, recursive, false);
+ }
+
+ /**
+ * Delete a file/dir and assert that delete() returned true
+ * <i>and</i> that the path no longer exists. This variant rejects
+ * all operations on root directories
+ * @param fs filesystem
+ * @param file path to delete
+ * @param recursive flag to enable recursive delete
+ * @param allowRootOperations can the root dir be deleted?
+ * @throws IOException IO problems
+ */
+ public static void assertDeleted(FileSystem fs,
+ Path file,
+ boolean recursive,
+ boolean allowRootOperations) throws IOException {
+ rejectRootOperation(file, allowRootOperations);
+ assertPathExists(fs, "about to be deleted file", file);
+ boolean deleted = fs.delete(file, recursive);
+ String dir = ls(fs, file.getParent());
+ assertTrue("Delete failed on " + file + ": " + dir, deleted);
+ assertPathDoesNotExist(fs, "Deleted file", file);
+ }
+
+ /**
+ * Read in "length" bytes, convert to an ascii string
+ * @param fs filesystem
+ * @param path path to read
+ * @param length #of bytes to read.
+ * @return the bytes read and converted to a string
+ * @throws IOException IO problems
+ */
+ public static String readBytesToString(FileSystem fs,
+ Path path,
+ int length) throws IOException {
+ FSDataInputStream in = fs.open(path);
+ try {
+ byte[] buf = new byte[length];
+ in.readFully(0, buf);
+ return toChar(buf);
+ } finally {
+ in.close();
+ }
+ }
+
+ /**
+ * Take an array of filestats and convert to a string (prefixed w/ a [01] counter
+ * @param stats array of stats
+ * @param separator separator after every entry
+ * @return a stringified set
+ */
+ public static String fileStatsToString(FileStatus[] stats, String separator) {
+ StringBuilder buf = new StringBuilder(stats.length * 128);
+ for (int i = 0; i < stats.length; i++) {
+ buf.append(String.format("[%02d] %s", i, stats[i])).append(separator);
+ }
+ return buf.toString();
+ }
+
+ /**
+ * List a directory
+ * @param fileSystem FS
+ * @param path path
+ * @return a directory listing or failure message
+ * @throws IOException
+ */
+ public static String ls(FileSystem fileSystem, Path path) throws IOException {
+ if (path == null) {
+ //surfaces when someone calls getParent() on something at the top of the path
+ return "/";
+ }
+ FileStatus[] stats;
+ String pathtext = "ls " + path;
+ try {
+ stats = fileSystem.listStatus(path);
+ } catch (FileNotFoundException e) {
+ return pathtext + " -file not found";
+ } catch (IOException e) {
+ return pathtext + " -failed: " + e;
+ }
+ return dumpStats(pathtext, stats);
+ }
+
+ public static String dumpStats(String pathname, FileStatus[] stats) {
+ return pathname + fileStatsToString(stats, "\n");
+ }
+
+ /**
+ * Assert that a file exists and whose {@link FileStatus} entry
+ * declares that this is a file and not a symlink or directory.
+ * @param fileSystem filesystem to resolve path against
+ * @param filename name of the file
+ * @throws IOException IO problems during file operations
+ */
+ public static void assertIsFile(FileSystem fileSystem, Path filename) throws
+ IOException {
+ assertPathExists(fileSystem, "Expected file", filename);
+ FileStatus status = fileSystem.getFileStatus(filename);
+ assertIsFile(filename, status);
+ }
+
+ /**
+ * Assert that a file exists and whose {@link FileStatus} entry
+ * declares that this is a file and not a symlink or directory.
+ * @param filename name of the file
+ * @param status file status
+ */
+ public static void assertIsFile(Path filename, FileStatus status) {
+ String fileInfo = filename + " " + status;
+ assertFalse("File claims to be a directory " + fileInfo,
+ status.isDirectory());
+ assertFalse("File claims to be a symlink " + fileInfo,
+ status.isSymlink());
+ }
+
+ /**
+ * Create a dataset for use in the tests; all data is in the range
+ * base to (base+modulo-1) inclusive
+ * @param len length of data
+ * @param base base of the data
+ * @param modulo the modulo
+ * @return the newly generated dataset
+ */
+ public static byte[] dataset(int len, int base, int modulo) {
+ byte[] dataset = new byte[len];
+ for (int i = 0; i < len; i++) {
+ dataset[i] = (byte) (base + (i % modulo));
+ }
+ return dataset;
+ }
+
+ /**
+ * Assert that a path exists -but make no assertions as to the
+ * type of that entry
+ *
+ * @param fileSystem filesystem to examine
+ * @param message message to include in the assertion failure message
+ * @param path path in the filesystem
+ * @throws FileNotFoundException raised if the path is missing
+ * @throws IOException IO problems
+ */
+ public static void assertPathExists(FileSystem fileSystem, String message,
+ Path path) throws IOException {
+ if (!fileSystem.exists(path)) {
+ //failure, report it
+ String listing = ls(fileSystem, path.getParent());
+ throw new FileNotFoundException(message + ": not found " + path
+ + " in \"" + path.getParent() + "\" :\n" + listing);
+ }
+ }
+
+ /**
+ * Assert that a path does not exist
+ *
+ * @param fileSystem filesystem to examine
+ * @param message message to include in the assertion failure message
+ * @param path path in the filesystem
+ * @throws IOException IO problems
+ */
+ public static void assertPathDoesNotExist(FileSystem fileSystem,
+ String message,
+ Path path) throws IOException {
+ try {
+ FileStatus status = fileSystem.getFileStatus(path);
+ fail(message + ": unexpectedly found " + path + " as " + status);
+ } catch (FileNotFoundException expected) {
+ //this is expected
+
+ }
+ }
+
+ /**
+ * Assert that a FileSystem.listStatus on a dir finds the subdir/child entry
+ * @param fs filesystem
+ * @param dir directory to scan
+ * @param subdir full path to look for
+ * @throws IOException IO probles
+ */
+ public static void assertListStatusFinds(FileSystem fs,
+ Path dir,
+ Path subdir) throws IOException {
+ FileStatus[] stats = fs.listStatus(dir);
+ boolean found = false;
+ StringBuilder builder = new StringBuilder();
+ for (FileStatus stat : stats) {
+ builder.append(stat.toString()).append('\n');
+ if (stat.getPath().equals(subdir)) {
+ found = true;
+ }
+ }
+ assertTrue("Path " + subdir
+ + " not found in directory " + dir + ":" + builder,
+ found);
+ }
+
+ /**
+ * Test for the host being an OSX machine
+ * @return true if the JVM thinks that is running on OSX
+ */
+ public static boolean isOSX() {
+ return System.getProperty("os.name").contains("OS X");
+ }
+
+ /**
+ * compare content of file operations using a double byte array
+ * @param concat concatenated files
+ * @param bytes bytes
+ */
+ public static void validateFileContent(byte[] concat, byte[][] bytes) {
+ int idx = 0;
+ boolean mismatch = false;
+
+ for (byte[] bb : bytes) {
+ for (byte b : bb) {
+ if (b != concat[idx++]) {
+ mismatch = true;
+ break;
+ }
+ }
+ if (mismatch)
+ break;
+ }
+ assertFalse("File content of file is not as expected at offset " + idx,
+ mismatch);
+ }
+
+ /**
+ * Receives test data from the given input file and checks the size of the
+ * data as well as the pattern inside the received data.
+ *
+ * @param fs FileSystem
+ * @param path Input file to be checked
+ * @param expectedSize the expected size of the data to be read from the
+ * input file in bytes
+ * @param bufferLen Pattern length
+ * @param modulus Pattern modulus
+ * @throws IOException
+ * thrown if an error occurs while reading the data
+ */
+ public static void verifyReceivedData(FileSystem fs, Path path,
+ final long expectedSize,
+ final int bufferLen,
+ final int modulus) throws IOException {
+ final byte[] testBuffer = new byte[bufferLen];
+
+ long totalBytesRead = 0;
+ int nextExpectedNumber = 0;
+ final InputStream inputStream = fs.open(path);
+ try {
+ while (true) {
+ final int bytesRead = inputStream.read(testBuffer);
+ if (bytesRead < 0) {
+ break;
+ }
+
+ totalBytesRead += bytesRead;
+
+ for (int i = 0; i < bytesRead; ++i) {
+ if (testBuffer[i] != nextExpectedNumber) {
+ throw new IOException("Read number " + testBuffer[i]
+ + " but expected " + nextExpectedNumber);
+ }
+
+ ++nextExpectedNumber;
+
+ if (nextExpectedNumber == modulus) {
+ nextExpectedNumber = 0;
+ }
+ }
+ }
+
+ if (totalBytesRead != expectedSize) {
+ throw new IOException("Expected to read " + expectedSize +
+ " bytes but only received " + totalBytesRead);
+ }
+ } finally {
+ inputStream.close();
+ }
+ }
+
+ /**
+ * Generates test data of the given size according to some specific pattern
+ * and writes it to the provided output file.
+ *
+ * @param fs FileSystem
+ * @param path Test file to be generated
+ * @param size The size of the test data to be generated in bytes
+ * @param bufferLen Pattern length
+ * @param modulus Pattern modulus
+ * @throws IOException
+ * thrown if an error occurs while writing the data
+ */
+ public static long generateTestFile(FileSystem fs, Path path,
+ final long size,
+ final int bufferLen,
+ final int modulus) throws IOException {
+ final byte[] testBuffer = new byte[bufferLen];
+ for (int i = 0; i < testBuffer.length; ++i) {
+ testBuffer[i] = (byte) (i % modulus);
+ }
+
+ final OutputStream outputStream = fs.create(path, false);
+ long bytesWritten = 0;
+ try {
+ while (bytesWritten < size) {
+ final long diff = size - bytesWritten;
+ if (diff < testBuffer.length) {
+ outputStream.write(testBuffer, 0, (int) diff);
+ bytesWritten += diff;
+ } else {
+ outputStream.write(testBuffer);
+ bytesWritten += testBuffer.length;
+ }
+ }
+
+ return bytesWritten;
+ } finally {
+ outputStream.close();
+ }
+ }
+
+ /**
+ * Creates and reads a file with the given size. The test file is generated
+ * according to a specific pattern so it can be easily verified even if it's
+ * a multi-GB one.
+ * During the read phase the incoming data stream is also checked against
+ * this pattern.
+ *
+ * @param fs FileSystem
+ * @param parent Test file parent dir path
+ * @throws IOException
+ * thrown if an I/O error occurs while writing or reading the test file
+ */
+ public static void createAndVerifyFile(FileSystem fs, Path parent, final long fileSize)
+ throws IOException {
+ int testBufferSize = fs.getConf()
+ .getInt(IO_CHUNK_BUFFER_SIZE, DEFAULT_IO_CHUNK_BUFFER_SIZE);
+ int modulus = fs.getConf()
+ .getInt(IO_CHUNK_MODULUS_SIZE, DEFAULT_IO_CHUNK_MODULUS_SIZE);
+
+ final String objectName = UUID.randomUUID().toString();
+ final Path objectPath = new Path(parent, objectName);
+
+ // Write test file in a specific pattern
+ assertEquals(fileSize,
+ generateTestFile(fs, objectPath, fileSize, testBufferSize, modulus));
+ assertPathExists(fs, "not created successful", objectPath);
+
+ // Now read the same file back and verify its content
+ try {
+ verifyReceivedData(fs, objectPath, fileSize, testBufferSize, modulus);
+ } finally {
+ // Delete test file
+ fs.delete(objectPath, false);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/MiniZooKeeperCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/MiniZooKeeperCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/MiniZooKeeperCluster.java
new file mode 100644
index 0000000..d739324
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/MiniZooKeeperCluster.java
@@ -0,0 +1,395 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.zookeeper.server.NIOServerCnxnFactory;
+import org.apache.zookeeper.server.ZooKeeperServer;
+import org.apache.zookeeper.server.persistence.FileTxnLog;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.Reader;
+import java.net.BindException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+
+/**
+ * This is a version of the HBase ZK cluster cut out to be standalone
+ */
+public class MiniZooKeeperCluster {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ MiniZooKeeperCluster.class);
+
+ private static final int TICK_TIME = 2000;
+ private static final int CONNECTION_TIMEOUT = 30000;
+ public static final int MAX_CLIENT_CONNECTIONS = 1000;
+
+ private boolean started;
+
+ /** The default port. If zero, we use a random port. */
+ private int defaultClientPort = 0;
+
+ private int clientPort;
+
+ private List<NIOServerCnxnFactory> standaloneServerFactoryList;
+ private List<ZooKeeperServer> zooKeeperServers;
+ private List<Integer> clientPortList;
+
+ private int activeZKServerIndex;
+ private int tickTime = 0;
+
+ private Configuration configuration;
+
+ public MiniZooKeeperCluster() {
+ this(new Configuration());
+ }
+
+ public MiniZooKeeperCluster(Configuration configuration) {
+ this.started = false;
+ this.configuration = configuration;
+ activeZKServerIndex = -1;
+ zooKeeperServers = new ArrayList<ZooKeeperServer>();
+ clientPortList = new ArrayList<Integer>();
+ standaloneServerFactoryList = new ArrayList<NIOServerCnxnFactory>();
+ }
+
+ public void setDefaultClientPort(int clientPort) {
+ if (clientPort <= 0) {
+ throw new IllegalArgumentException("Invalid default ZK client port: "
+ + clientPort);
+ }
+ this.defaultClientPort = clientPort;
+ }
+
+ /**
+ * Selects a ZK client port. Returns the default port if specified.
+ * Otherwise, returns a random port. The random port is selected from the
+ * range between 49152 to 65535. These ports cannot be registered with IANA
+ * and are intended for dynamic allocation (see http://bit.ly/dynports).
+ */
+ private int selectClientPort() {
+ if (defaultClientPort > 0) {
+ return defaultClientPort;
+ }
+ return 0xc000 + new Random().nextInt(0x3f00);
+ }
+
+ public void setTickTime(int tickTime) {
+ this.tickTime = tickTime;
+ }
+
+ public int getBackupZooKeeperServerNum() {
+ return zooKeeperServers.size() - 1;
+ }
+
+ public int getZooKeeperServerNum() {
+ return zooKeeperServers.size();
+ }
+
+ // / XXX: From o.a.zk.t.ClientBase
+ private static void setupTestEnv() {
+ // during the tests we run with 100K prealloc in the logs.
+ // on windows systems prealloc of 64M was seen to take ~15seconds
+ // resulting in test failure (client timeout on first session).
+ // set env and directly in order to handle static init/gc issues
+ System.setProperty("zookeeper.preAllocSize", "100");
+ FileTxnLog.setPreallocSize(100 * 1024);
+ }
+
+ public int startup(File baseDir) throws IOException, InterruptedException {
+ return startup(baseDir, 1);
+ }
+
+ /**
+ * @param baseDir
+ * @param numZooKeeperServers
+ * @return ClientPort server bound to, -1 if there was a
+ * binding problem and we couldn't pick another port.
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public int startup(File baseDir, int numZooKeeperServers) throws IOException,
+ InterruptedException {
+ if (numZooKeeperServers <= 0)
+ return -1;
+
+ setupTestEnv();
+ shutdown();
+
+ int tentativePort = selectClientPort();
+
+ // running all the ZK servers
+ for (int i = 0; i < numZooKeeperServers; i++) {
+ File dir = new File(baseDir, "zookeeper_" + i).getAbsoluteFile();
+ recreateDir(dir);
+ int tickTimeToUse;
+ if (this.tickTime > 0) {
+ tickTimeToUse = this.tickTime;
+ } else {
+ tickTimeToUse = TICK_TIME;
+ }
+ ZooKeeperServer server = new ZooKeeperServer(dir, dir, tickTimeToUse);
+ NIOServerCnxnFactory standaloneServerFactory;
+ while (true) {
+ try {
+ standaloneServerFactory = new NIOServerCnxnFactory();
+ standaloneServerFactory.configure(
+ new InetSocketAddress(tentativePort),
+ MAX_CLIENT_CONNECTIONS
+ );
+ } catch (BindException e) {
+ LOG.debug("Failed binding ZK Server to client port: " +
+ tentativePort, e);
+ // We're told to use some port but it's occupied, fail
+ if (defaultClientPort > 0) return -1;
+ // This port is already in use, try to use another.
+ tentativePort = selectClientPort();
+ continue;
+ }
+ break;
+ }
+
+ // Start up this ZK server
+ standaloneServerFactory.startup(server);
+ if (!waitForServerUp(tentativePort, CONNECTION_TIMEOUT)) {
+ throw new IOException("Waiting for startup of standalone server");
+ }
+
+ // We have selected this port as a client port.
+ clientPortList.add(tentativePort);
+ standaloneServerFactoryList.add(standaloneServerFactory);
+ zooKeeperServers.add(server);
+ tentativePort++; //for the next server
+ }
+
+ // set the first one to be active ZK; Others are backups
+ activeZKServerIndex = 0;
+ started = true;
+ clientPort = clientPortList.get(activeZKServerIndex);
+ LOG.info("Started MiniZK Cluster and connect 1 ZK server " +
+ "on client port: " + clientPort);
+ return clientPort;
+ }
+
+ private void recreateDir(File dir) throws IOException {
+ if (dir.exists()) {
+ if (!FileUtil.fullyDelete(dir)) {
+ throw new IOException("Could not delete zk base directory: " + dir);
+ }
+ }
+ try {
+ dir.mkdirs();
+ } catch (SecurityException e) {
+ throw new IOException("creating dir: " + dir, e);
+ }
+ }
+
+ /**
+ * @throws IOException
+ */
+ public void shutdown() throws IOException {
+ if (!started) {
+ return;
+ }
+
+ // shut down all the zk servers
+ for (int i = 0; i < standaloneServerFactoryList.size(); i++) {
+ NIOServerCnxnFactory standaloneServerFactory =
+ standaloneServerFactoryList.get(i);
+ int clientPort = clientPortList.get(i);
+
+ standaloneServerFactory.shutdown();
+ if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
+ throw new IOException("Waiting for shutdown of standalone server");
+ }
+ }
+ for (ZooKeeperServer zkServer : zooKeeperServers) {
+ //explicitly close ZKDatabase since ZookeeperServer does not close them
+ zkServer.getZKDatabase().close();
+ }
+
+ // clear everything
+ started = false;
+ activeZKServerIndex = 0;
+ standaloneServerFactoryList.clear();
+ clientPortList.clear();
+ zooKeeperServers.clear();
+
+ LOG.info("Shutdown MiniZK cluster with all ZK servers");
+ }
+
+ /**@return clientPort return clientPort if there is another ZK backup can run
+ * when killing the current active; return -1, if there is no backups.
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public int killCurrentActiveZooKeeperServer() throws IOException,
+ InterruptedException {
+ if (!started || activeZKServerIndex < 0) {
+ return -1;
+ }
+
+ // Shutdown the current active one
+ NIOServerCnxnFactory standaloneServerFactory =
+ standaloneServerFactoryList.get(activeZKServerIndex);
+ int clientPort = clientPortList.get(activeZKServerIndex);
+
+ standaloneServerFactory.shutdown();
+ if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
+ throw new IOException("Waiting for shutdown of standalone server");
+ }
+
+ zooKeeperServers.get(activeZKServerIndex).getZKDatabase().close();
+
+ // remove the current active zk server
+ standaloneServerFactoryList.remove(activeZKServerIndex);
+ clientPortList.remove(activeZKServerIndex);
+ zooKeeperServers.remove(activeZKServerIndex);
+ LOG.info("Kill the current active ZK servers in the cluster " +
+ "on client port: " + clientPort);
+
+ if (standaloneServerFactoryList.size() == 0) {
+ // there is no backup servers;
+ return -1;
+ }
+ clientPort = clientPortList.get(activeZKServerIndex);
+ LOG.info("Activate a backup zk server in the cluster " +
+ "on client port: " + clientPort);
+ // return the next back zk server's port
+ return clientPort;
+ }
+
+ /**
+ * Kill one back up ZK servers
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public void killOneBackupZooKeeperServer() throws IOException,
+ InterruptedException {
+ if (!started || activeZKServerIndex < 0 ||
+ standaloneServerFactoryList.size() <= 1) {
+ return;
+ }
+
+ int backupZKServerIndex = activeZKServerIndex + 1;
+ // Shutdown the current active one
+ NIOServerCnxnFactory standaloneServerFactory =
+ standaloneServerFactoryList.get(backupZKServerIndex);
+ int clientPort = clientPortList.get(backupZKServerIndex);
+
+ standaloneServerFactory.shutdown();
+ if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
+ throw new IOException("Waiting for shutdown of standalone server");
+ }
+
+ zooKeeperServers.get(backupZKServerIndex).getZKDatabase().close();
+
+ // remove this backup zk server
+ standaloneServerFactoryList.remove(backupZKServerIndex);
+ clientPortList.remove(backupZKServerIndex);
+ zooKeeperServers.remove(backupZKServerIndex);
+ LOG.info("Kill one backup ZK servers in the cluster " +
+ "on client port: " + clientPort);
+ }
+
+ // XXX: From o.a.zk.t.ClientBase
+ private static boolean waitForServerDown(int port, long timeout) {
+ long start = System.currentTimeMillis();
+ while (true) {
+ try {
+ Socket sock = new Socket("localhost", port);
+ try {
+ OutputStream outstream = sock.getOutputStream();
+ outstream.write("stat".getBytes());
+ outstream.flush();
+ } finally {
+ sock.close();
+ }
+ } catch (IOException e) {
+ return true;
+ }
+
+ if (System.currentTimeMillis() > start + timeout) {
+ break;
+ }
+ try {
+ Thread.sleep(250);
+ } catch (InterruptedException e) {
+ // ignore
+ }
+ }
+ return false;
+ }
+
+ // XXX: From o.a.zk.t.ClientBase
+ private static boolean waitForServerUp(int port, long timeout) {
+ long start = System.currentTimeMillis();
+ while (true) {
+ try {
+ Socket sock = new Socket("localhost", port);
+ BufferedReader reader = null;
+ try {
+ OutputStream outstream = sock.getOutputStream();
+ outstream.write("stat".getBytes());
+ outstream.flush();
+
+ Reader isr = new InputStreamReader(sock.getInputStream());
+ reader = new BufferedReader(isr);
+ String line = reader.readLine();
+ if (line != null && line.startsWith("Zookeeper version:")) {
+ return true;
+ }
+ } finally {
+ sock.close();
+ if (reader != null) {
+ reader.close();
+ }
+ }
+ } catch (IOException e) {
+ // ignore as this is expected
+ LOG.info("server localhost:" + port + " not up " + e);
+ }
+
+ if (System.currentTimeMillis() > start + timeout) {
+ break;
+ }
+ try {
+ Thread.sleep(250);
+ } catch (InterruptedException e) {
+ // ignore
+ }
+ }
+ return false;
+ }
+
+ public int getClientPort() {
+ return clientPort;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/tools/TestUtility.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/tools/TestUtility.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/tools/TestUtility.java
new file mode 100644
index 0000000..78ce3e7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/tools/TestUtility.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.tools;
+
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
+import org.apache.commons.compress.utils.IOUtils;
+import org.junit.Assert;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+/**
+ * Various utility methods
+ * Byte comparison methods are from
+ * <code>org.apache.hadoop.fs.contract.ContractTestUtils</code>
+ */
+public class TestUtility {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestUtility.class);
+
+ public static void addDir(File dirObj, ZipArchiveOutputStream zipFile, String prefix) throws IOException {
+ for (File file : dirObj.listFiles()) {
+ if (file.isDirectory()) {
+ addDir(file, zipFile, prefix + file.getName() + File.separator);
+ } else {
+ log.info("Adding to zip - " + prefix + file.getName());
+ zipFile.putArchiveEntry(new ZipArchiveEntry(prefix + file.getName()));
+ IOUtils.copy(new FileInputStream(file), zipFile);
+ zipFile.closeArchiveEntry();
+ }
+ }
+ }
+
+ public static void zipDir(String zipFile, String dir) throws IOException {
+ File dirObj = new File(dir);
+ ZipArchiveOutputStream out = new ZipArchiveOutputStream(new FileOutputStream(zipFile));
+ log.info("Creating : {}", zipFile);
+ try {
+ addDir(dirObj, out, "");
+ } finally {
+ out.close();
+ }
+ }
+
+ public static String createAppPackage(
+ TemporaryFolder folder, String subDir, String pkgName, String srcPath) throws IOException {
+ String zipFileName;
+ File pkgPath = folder.newFolder(subDir);
+ File zipFile = new File(pkgPath, pkgName).getAbsoluteFile();
+ zipFileName = zipFile.getAbsolutePath();
+ TestUtility.zipDir(zipFileName, srcPath);
+ log.info("Created temporary zip file at {}", zipFileName);
+ return zipFileName;
+ }
+
+
+ /**
+ * Assert that tthe array original[0..len] and received[] are equal.
+ * A failure triggers the logging of the bytes near where the first
+ * difference surfaces.
+ * @param original source data
+ * @param received actual
+ * @param len length of bytes to compare
+ */
+ public static void compareByteArrays(byte[] original,
+ byte[] received,
+ int len) {
+ Assert.assertEquals("Number of bytes read != number written",
+ len, received.length);
+ int errors = 0;
+ int first_error_byte = -1;
+ for (int i = 0; i < len; i++) {
+ if (original[i] != received[i]) {
+ if (errors == 0) {
+ first_error_byte = i;
+ }
+ errors++;
+ }
+ }
+
+ if (errors > 0) {
+ String message = String.format(" %d errors in file of length %d",
+ errors, len);
+ log.warn(message);
+ // the range either side of the first error to print
+ // this is a purely arbitrary number, to aid user debugging
+ final int overlap = 10;
+ for (int i = Math.max(0, first_error_byte - overlap);
+ i < Math.min(first_error_byte + overlap, len);
+ i++) {
+ byte actual = received[i];
+ byte expected = original[i];
+ String letter = toChar(actual);
+ String line = String.format("[%04d] %2x %s\n", i, actual, letter);
+ if (expected != actual) {
+ line = String.format("[%04d] %2x %s -expected %2x %s\n",
+ i,
+ actual,
+ letter,
+ expected,
+ toChar(expected));
+ }
+ log.warn(line);
+ }
+ Assert.fail(message);
+ }
+ }
+ /**
+ * Convert a byte to a character for printing. If the
+ * byte value is < 32 -and hence unprintable- the byte is
+ * returned as a two digit hex value
+ * @param b byte
+ * @return the printable character string
+ */
+ public static String toChar(byte b) {
+ if (b >= 0x20) {
+ return Character.toString((char) b);
+ } else {
+ return String.format("%02x", b);
+ }
+ }
+
+ /**
+ * Convert a buffer to a string, character by character
+ * @param buffer input bytes
+ * @return a string conversion
+ */
+ public static String toChar(byte[] buffer) {
+ StringBuilder builder = new StringBuilder(buffer.length);
+ for (byte b : buffer) {
+ builder.append(toChar(b));
+ }
+ return builder.toString();
+ }
+
+ public static byte[] toAsciiByteArray(String s) {
+ char[] chars = s.toCharArray();
+ int len = chars.length;
+ byte[] buffer = new byte[len];
+ for (int i = 0; i < len; i++) {
+ buffer[i] = (byte) (chars[i] & 0xff);
+ }
+ return buffer;
+ }
+
+ /**
+ * Create a dataset for use in the tests; all data is in the range
+ * base to (base+modulo-1) inclusive
+ * @param len length of data
+ * @param base base of the data
+ * @param modulo the modulo
+ * @return the newly generated dataset
+ */
+ public static byte[] dataset(int len, int base, int modulo) {
+ byte[] dataset = new byte[len];
+ for (int i = 0; i < len; i++) {
+ dataset[i] = (byte) (base + (i % modulo));
+ }
+ return dataset;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.txt
new file mode 100644
index 0000000..a1d7780
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.txt
@@ -0,0 +1,16 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.xml
new file mode 100644
index 0000000..cb8eab2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/metainfo.xml
@@ -0,0 +1,98 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <application>
+ <name>STORM</name>
+ <comment>Apache Hadoop Stream processing framework</comment>
+ <version>0.9.1.2.1</version>
+ <components>
+
+ <component>
+ <name>NIMBUS</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/nimbus.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>STORM_REST_API</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/rest_api.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>SUPERVISOR</name>
+ <category>SLAVE</category>
+ <commandScript>
+ <script>scripts/supervisor.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>STORM_UI_SERVER</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/ui_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>DRPC_SERVER</name>
+ <category>MASTER</category>
+ <commandScript>
+ <script>scripts/drpc_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osType>any</osType>
+ <packages>
+ <package>
+ <type>tarball</type>
+ <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <configFiles>
+ <configFile>
+ <type>xml</type>
+ <fileName>storm-site.xml</fileName>
+ <dictionaryName>storm-site</dictionaryName>
+ </configFile>
+ </configFiles>
+ </application>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.txt
new file mode 100644
index 0000000..a1d7780
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.txt
@@ -0,0 +1,16 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.xml
new file mode 100644
index 0000000..f86e687
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/common/tools/test/someOtherFile.xml
@@ -0,0 +1,17 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo></metainfo>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml
new file mode 100644
index 0000000..fbe9299
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml
@@ -0,0 +1,180 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <application>
+ <name>STORM</name>
+ <comment>Apache Hadoop Stream processing framework</comment>
+ <version>0.9.1.2.1</version>
+ <exportedConfigs>storm-site</exportedConfigs>
+
+ <exportGroups>
+ <exportGroup>
+ <name>QuickLinks</name>
+ <exports>
+ <export>
+ <name>app.jmx</name>
+ <value>http://${STORM_REST_API_HOST}:${site.global.rest_api_port}/api/cluster/summary</value>
+ </export>
+ <export>
+ <name>app.monitor</name>
+ <value>http://${STORM_UI_SERVER_HOST}:${site.storm-site.ui.port}</value>
+ </export>
+ <export>
+ <name>app.metrics</name>
+ <value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value>
+ </export>
+ <export>
+ <name>ganglia.ui</name>
+ <value>http://${site.global.ganglia_server_host}/ganglia?c=${site.global.ganglia_server_id}</value>
+ </export>
+ <export>
+ <name>nimbus.url</name>
+ <value>http://${NIMBUS_HOST}:${site.storm-site.nimbus.thrift.port}</value>
+ </export>
+ </exports>
+ </exportGroup>
+ </exportGroups>
+
+ <commandOrders>
+ <commandOrder>
+ <command>NIMBUS-START</command>
+ <requires>SUPERVISOR-INSTALLED,STORM_UI_SERVER-INSTALLED,DRPC_SERVER-INSTALLED,STORM_REST_API-INSTALLED
+ </requires>
+ </commandOrder>
+ <commandOrder>
+ <command>SUPERVISOR-START</command>
+ <requires>NIMBUS-STARTED</requires>
+ </commandOrder>
+ <commandOrder>
+ <command>DRPC_SERVER-START</command>
+ <requires>NIMBUS-STARTED</requires>
+ </commandOrder>
+ <commandOrder>
+ <command>STORM_REST_API-START</command>
+ <requires>NIMBUS-STARTED,DRPC_SERVER-STARTED,STORM_UI_SERVER-STARTED</requires>
+ </commandOrder>
+ <commandOrder>
+ <command>STORM_UI_SERVER-START</command>
+ <requires>NIMBUS-STARTED</requires>
+ </commandOrder>
+ </commandOrders>
+
+ <components>
+
+ <component>
+ <name>NIMBUS</name>
+ <category>MASTER</category>
+ <autoStartOnFailure>true</autoStartOnFailure>
+ <appExports>QuickLinks-nimbus.url,QuickLinks-ganglia.ui,QuickLinks-app.metrics</appExports>
+ <commandScript>
+ <script>scripts/nimbus.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>STORM_REST_API</name>
+ <category>MASTER</category>
+ <autoStartOnFailure>true</autoStartOnFailure>
+ <appExports>QuickLinks-app.jmx</appExports>
+ <commandScript>
+ <script>scripts/rest_api.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>SUPERVISOR</name>
+ <category>SLAVE</category>
+ <autoStartOnFailure>true</autoStartOnFailure>
+ <componentExports>
+ <componentExport>
+ <name>log_viewer_port</name>
+ <value>${THIS_HOST}:${site.storm-site.logviewer.port}</value>
+ </componentExport>
+ </componentExports>
+ <commandScript>
+ <script>scripts/supervisor.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>STORM_UI_SERVER</name>
+ <category>MASTER</category>
+ <publishConfig>true</publishConfig>
+ <appExports>QuickLinks-app.monitor</appExports>
+ <autoStartOnFailure>true</autoStartOnFailure>
+ <commandScript>
+ <script>scripts/ui_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>DRPC_SERVER</name>
+ <category>MASTER</category>
+ <autoStartOnFailure>true</autoStartOnFailure>
+ <commandScript>
+ <script>scripts/drpc_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+
+ <component>
+ <name>ANOTHER_COMPONENT</name>
+ <category>MASTER</category>
+ <commands>
+ <command>
+ <exec>start command</exec>
+ </command>
+ <command>
+ <exec>stop command</exec>
+ <name>STOP</name>
+ </command>
+ </commands>
+ </component>
+ </components>
+
+ <osSpecifics>
+ <osSpecific>
+ <osType>any</osType>
+ <packages>
+ <package>
+ <type>tarball</type>
+ <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <packages>
+ <package>
+ <type>tarball</type>
+ <name>test-tarball-name.tgz</name>
+ </package>
+ </packages>
+ </application>
+</metainfo>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/51] [abbrv] hadoop git commit: YARN-5770. Performance improvement
of native-services REST API service. Contributed by Gour Saha
Posted by ji...@apache.org.
YARN-5770. Performance improvement of native-services REST API service. Contributed by Gour Saha
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/31848c7e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/31848c7e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/31848c7e
Branch: refs/heads/yarn-native-services
Commit: 31848c7e615ef1c269ac7e9ea2c3f7e5cfacfe99
Parents: b544efb
Author: Billie Rinaldi <bi...@apache.org>
Authored: Wed Oct 26 08:34:39 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../api/impl/ApplicationApiService.java | 144 +++++++++----------
.../yarn/services/resource/Application.java | 7 +-
.../yarn/services/resource/Container.java | 4 +-
.../services/webapp/ApplicationApiWebApp.java | 12 +-
.../org/apache/slider/client/SliderClient.java | 25 ++--
.../apache/slider/client/SliderClientAPI.java | 11 ++
6 files changed, 107 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31848c7e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index 73df4a1..cf43ac2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -20,8 +20,7 @@ package org.apache.hadoop.yarn.services.api.impl;
import static org.apache.hadoop.yarn.services.utils.RestApiConstants.*;
import static org.apache.hadoop.yarn.services.utils.RestApiErrorMessages.*;
-import java.io.File;
-import java.io.FileReader;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.UndeclaredThrowableException;
import java.security.PrivilegedExceptionAction;
@@ -36,7 +35,6 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.regex.Pattern;
-import javax.inject.Singleton;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.GET;
@@ -52,6 +50,7 @@ import javax.ws.rs.core.Response.Status;
import org.apache.commons.lang.SerializationUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -78,7 +77,6 @@ import org.apache.slider.common.params.ActionFlexArgs;
import org.apache.slider.common.params.ActionFreezeArgs;
import org.apache.slider.common.params.ActionListArgs;
import org.apache.slider.common.params.ActionRegistryArgs;
-import org.apache.slider.common.params.ActionStatusArgs;
import org.apache.slider.common.params.ActionThawArgs;
import org.apache.slider.common.params.ComponentArgsDelegate;
import org.apache.slider.common.tools.SliderUtils;
@@ -98,6 +96,7 @@ import com.google.gson.JsonElement;
import com.google.gson.JsonNull;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
+import com.google.inject.Singleton;
@Singleton
@Path(APPLICATIONS_API_RESOURCE_PATH)
@@ -109,6 +108,11 @@ public class ApplicationApiService implements ApplicationApi {
private static org.apache.hadoop.conf.Configuration SLIDER_CONFIG;
private static UserGroupInformation SLIDER_USER;
private static SliderClient SLIDER_CLIENT;
+ private static Response SLIDER_VERSION;
+ private static final JsonParser JSON_PARSER = new JsonParser();
+ private static final JsonObject EMPTY_JSON_OBJECT = new JsonObject();
+ private static final ActionListArgs ACTION_LIST_ARGS = new ActionListArgs();
+ private static final ActionFreezeArgs ACTION_FREEZE_ARGS = new ActionFreezeArgs();
static {
init();
@@ -119,24 +123,27 @@ public class ApplicationApiService implements ApplicationApi {
SLIDER_CONFIG = getSliderClientConfiguration();
SLIDER_USER = getSliderUser();
SLIDER_CLIENT = createSliderClient();
+ SLIDER_VERSION = initSliderVersion();
}
@GET
- @Path("/slider-version")
+ @Path("/versions/slider-version")
@Consumes({ MediaType.APPLICATION_JSON })
@Produces({ MediaType.APPLICATION_JSON })
public Response getSliderVersion() {
logger.info("GET: getSliderVersion");
+ return SLIDER_VERSION;
+ }
+ private static Response initSliderVersion() {
Map<String, Object> metadata = new HashMap<>();
BuildHelper.addBuildMetadata(metadata, "org.apache.hadoop.yarn.services");
String sliderVersion = metadata.toString();
logger.info("Slider version = {}", sliderVersion);
String hadoopVersion = SliderVersionInfo.getHadoopVersionString();
logger.info("Hadoop version = {}", hadoopVersion);
- return Response.ok(
- "{ \"slider_version\": \"" + sliderVersion
- + "\", \"hadoop_version\": \"" + hadoopVersion + "\"}").build();
+ return Response.ok("{ \"slider_version\": \"" + sliderVersion
+ + "\", \"hadoop_version\": \"" + hadoopVersion + "\"}").build();
}
@POST
@@ -196,7 +203,8 @@ public class ApplicationApiService implements ApplicationApi {
}
// If the application has no components do top-level checks
- if (application.getComponents() == null) {
+ if (application.getComponents() == null
+ || application.getComponents().size() == 0) {
// artifact
if (application.getArtifact() == null) {
throw new IllegalArgumentException(ERROR_ARTIFACT_INVALID);
@@ -222,6 +230,9 @@ public class ApplicationApiService implements ApplicationApi {
if (application.getNumberOfContainers() == null) {
throw new IllegalArgumentException(ERROR_CONTAINERS_COUNT_INVALID);
}
+
+ // Since it is a simple app with no components, create a default component
+ application.setComponents(getDefaultComponentAsList(application));
} else {
// If the application has components, then run checks for each component.
// Let global values take effect if component level values are not
@@ -274,11 +285,6 @@ public class ApplicationApiService implements ApplicationApi {
}
}
- // If it is a simple app with no components, then create a default component
- if (application.getComponents() == null) {
- application.setComponents(getDefaultComponentAsList(application));
- }
-
// Application lifetime if not specified, is set to unlimited lifetime
if (application.getLifetime() == null) {
application.setLifetime(DEFAULT_UNLIMITED_LIFETIME);
@@ -853,15 +859,12 @@ public class ApplicationApiService implements ApplicationApi {
// TODO: add status
app.setState(ApplicationState.ACCEPTED);
JsonObject appStatus = null;
- JsonObject appRegistryDocker = null;
JsonObject appRegistryQuicklinks = null;
try {
appStatus = getSliderApplicationStatus(appName);
- appRegistryDocker = getSliderApplicationRegistry(appName, "docker");
appRegistryQuicklinks = getSliderApplicationRegistry(appName,
"quicklinks");
- return populateAppData(app, appStatus, appRegistryDocker,
- appRegistryQuicklinks);
+ return populateAppData(app, appStatus, appRegistryQuicklinks);
} catch (BadClusterStateException | NotFoundException e) {
logger.error(
"Get application failed, application not in running state yet", e);
@@ -881,7 +884,7 @@ public class ApplicationApiService implements ApplicationApi {
}
private Response populateAppData(Application app, JsonObject appStatus,
- JsonObject appRegistryDocker, JsonObject appRegistryQuicklinks) {
+ JsonObject appRegistryQuicklinks) {
String appName = jsonGetAsString(appStatus, "name");
Long totalNumberOfRunningContainers = 0L;
Long totalExpectedNumberOfRunningContainers = 0L;
@@ -944,7 +947,7 @@ public class ApplicationApiService implements ApplicationApi {
JsonObject applicationStatistics = jsonGetAsObject(appStatus, "statistics");
if (applicationRoles == null) {
// initialize to empty object to avoid too many null checks
- applicationRoles = new JsonObject();
+ applicationRoles = EMPTY_JSON_OBJECT;
}
if (applicationStatus != null) {
JsonObject applicationLive = jsonGetAsObject(applicationStatus, "live");
@@ -954,8 +957,9 @@ public class ApplicationApiService implements ApplicationApi {
continue;
}
componentNames.add(entry.getKey());
- JsonObject componentRole = applicationRoles.get(entry.getKey()) == null ? new JsonObject()
- : applicationRoles.get(entry.getKey()).getAsJsonObject();
+ JsonObject componentRole = applicationRoles
+ .get(entry.getKey()) == null ? EMPTY_JSON_OBJECT
+ : applicationRoles.get(entry.getKey()).getAsJsonObject();
JsonObject liveContainers = entry.getValue().getAsJsonObject();
if (liveContainers != null) {
for (Map.Entry<String, JsonElement> liveContainerEntry : liveContainers
@@ -1052,67 +1056,57 @@ public class ApplicationApiService implements ApplicationApi {
private JsonObject getSliderApplicationStatus(final String appName)
throws IOException, YarnException, InterruptedException {
- final File appStatusOutputFile = File.createTempFile("status_", ".json");
- final ActionStatusArgs statusArgs = new ActionStatusArgs();
- statusArgs.output = appStatusOutputFile.getAbsolutePath();
- return invokeSliderClientRunnable(new SliderClientContextRunnable<JsonObject>() {
- @Override
- public JsonObject run(SliderClient sliderClient) throws YarnException,
- IOException, InterruptedException {
- sliderClient.actionStatus(appName, statusArgs);
- JsonParser parser = new JsonParser();
- FileReader reader = null;
- JsonElement statusElement = null;
- try {
- reader = new FileReader(appStatusOutputFile);
- statusElement = parser.parse(reader);
- } finally {
- if (reader != null) {
- reader.close();
+ return invokeSliderClientRunnable(
+ new SliderClientContextRunnable<JsonObject>() {
+ @Override
+ public JsonObject run(SliderClient sliderClient)
+ throws YarnException, IOException, InterruptedException {
+ String status = null;
+ try {
+ status = sliderClient.actionStatus(appName);
+ } catch (Exception e) {
+ logger.error("Exception calling slider.actionStatus", e);
+ return EMPTY_JSON_OBJECT;
+ }
+ JsonElement statusElement = JSON_PARSER.parse(status);
+ return (statusElement == null || statusElement instanceof JsonNull)
+ ? EMPTY_JSON_OBJECT : (JsonObject) statusElement;
}
- appStatusOutputFile.delete();
- }
- return (statusElement == null || statusElement instanceof JsonNull) ?
- new JsonObject() : (JsonObject) statusElement;
- }
- });
+ });
}
private JsonObject getSliderApplicationRegistry(final String appName,
- final String registryName) throws IOException, YarnException,
- InterruptedException {
- final File appRegistryOutputFile = File
- .createTempFile("registry_", ".json");
+ final String registryName)
+ throws IOException, YarnException, InterruptedException {
final ActionRegistryArgs registryArgs = new ActionRegistryArgs();
- registryArgs.out = appRegistryOutputFile;
registryArgs.name = appName;
registryArgs.getConf = registryName;
registryArgs.format = ConfigFormat.JSON.toString();
- return invokeSliderClientRunnable(new SliderClientContextRunnable<JsonObject>() {
- @Override
- public JsonObject run(SliderClient sliderClient) throws YarnException,
- IOException, InterruptedException {
- sliderClient.actionRegistry(registryArgs);
- JsonParser parser = new JsonParser();
- FileReader reader = null;
- JsonElement registryElement = null;
- try {
- reader = new FileReader(appRegistryOutputFile);
- registryElement = parser.parse(reader);
- } catch (Throwable t) {
- logger.error("Error reading file {}", appRegistryOutputFile);
- } finally {
- if (reader != null) {
- reader.close();
+ return invokeSliderClientRunnable(
+ new SliderClientContextRunnable<JsonObject>() {
+ @Override
+ public JsonObject run(SliderClient sliderClient)
+ throws YarnException, IOException, InterruptedException {
+ String registry = null;
+ try {
+ registry = sliderClient.actionRegistryGetConfig(registryArgs)
+ .asJson();
+ } catch (FileNotFoundException | PathNotFoundException e) {
+ // ignore and return empty object
+ return EMPTY_JSON_OBJECT;
+ } catch (Exception e) {
+ logger.error("Exception calling slider.actionRegistryGetConfig",
+ e);
+ return EMPTY_JSON_OBJECT;
+ }
+ JsonElement registryElement = JSON_PARSER.parse(registry);
+ return (registryElement == null
+ || registryElement instanceof JsonNull) ? EMPTY_JSON_OBJECT
+ : (JsonObject) registryElement;
}
- appRegistryOutputFile.delete();
- }
- return (registryElement == null || registryElement instanceof JsonNull) ?
- new JsonObject() : (JsonObject) registryElement;
- }
- });
+ });
}
private Integer getSliderList(final String appName)
@@ -1130,8 +1124,7 @@ public class ApplicationApiService implements ApplicationApi {
if (liveOnly) {
status = sliderClient.actionList(appName);
} else {
- ActionListArgs listArgs = new ActionListArgs();
- status = sliderClient.actionList(appName, listArgs);
+ status = sliderClient.actionList(appName, ACTION_LIST_ARGS);
}
return status;
}
@@ -1228,8 +1221,7 @@ public class ApplicationApiService implements ApplicationApi {
@Override
public Response run(SliderClient sliderClient) throws YarnException,
IOException, InterruptedException {
- ActionFreezeArgs freezeArgs = new ActionFreezeArgs();
- int returnCode = sliderClient.actionFreeze(appName, freezeArgs);
+ int returnCode = sliderClient.actionFreeze(appName, ACTION_FREEZE_ARGS);
if (returnCode == 0) {
logger.info("Successfully stopped application {}", appName);
return Response.status(Status.NO_CONTENT).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31848c7e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
index ed65ad2..beeffba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
@@ -44,7 +44,8 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder;
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@XmlRootElement
@JsonInclude(JsonInclude.Include.NON_NULL)
-@JsonPropertyOrder({ " name, state, resource, numberOfContainers, lifetime, containers " })
+@JsonPropertyOrder({ "name", "state", "resource", "number_of_containers",
+ "lifetime", "containers" })
public class Application extends BaseResource {
private static final long serialVersionUID = -4491694636566094885L;
@@ -174,8 +175,8 @@ public class Application extends BaseResource {
@ApiModelProperty(example = "null", value = "The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.")
@JsonProperty("launch_time")
- public String getLaunchTime() {
- return launchTime == null ? null : launchTime.toString();
+ public Date getLaunchTime() {
+ return launchTime == null ? null : (Date) launchTime.clone();
}
@XmlElement(name = "launch_time")
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31848c7e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java
index a4efdf3..f11c7b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java
@@ -79,8 +79,8 @@ public class Container extends BaseResource {
@ApiModelProperty(example = "null", value = "The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.")
@JsonProperty("launch_time")
- public String getLaunchTime() {
- return launchTime == null ? null : launchTime.toString();
+ public Date getLaunchTime() {
+ return launchTime == null ? null : (Date) launchTime.clone();
}
@XmlElement(name = "launch_time")
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31848c7e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java
index b1b6d7c..52a9de6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java
@@ -27,6 +27,7 @@ import java.util.Arrays;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.services.api.impl.ApplicationApiService;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
import org.mortbay.jetty.webapp.Configuration;
@@ -95,13 +96,10 @@ public class ApplicationApiWebApp extends AbstractService {
.setName("services-rest-api")
.addEndpoint(URI.create("http://" + webHost + ":" + webPort)).build();
- String apiPackages = "org.apache.hadoop.yarn.services.api" + SEP
- + "org.apache.hadoop.yarn.services.api.impl" + SEP
- + "org.apache.hadoop.yarn.services.resource" + SEP
- + "org.apache.hadoop.yarn.services.utils" + SEP
- + "org.apache.hadoop.yarn.services.webapp" + SEP
- + GenericExceptionHandler.class.getPackage().getName() + SEP
- + YarnJacksonJaxbJsonProvider.class.getPackage().getName();
+ String apiPackages =
+ ApplicationApiService.class.getPackage().getName() + SEP
+ + GenericExceptionHandler.class.getPackage().getName() + SEP
+ + YarnJacksonJaxbJsonProvider.class.getPackage().getName();
applicationApiServer.addJerseyResourcePackage(apiPackages, CONTEXT_ROOT
+ "/*");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31848c7e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 94e51e5..4b546cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -3137,16 +3137,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
@Override
@VisibleForTesting
- public int actionStatus(String clustername, ActionStatusArgs statusArgs) throws
- YarnException,
- IOException {
- verifyBindingsDefined();
- validateClusterName(clustername);
+ public int actionStatus(String clustername, ActionStatusArgs statusArgs)
+ throws YarnException, IOException {
+ ClusterDescription status = verifyAndGetClusterDescription(clustername);
String outfile = statusArgs.getOutput();
- ClusterDescription status = getClusterDescription(clustername);
- String text = status.toJsonString();
if (outfile == null) {
- log.info(text);
+ log.info(status.toJsonString());
} else {
status.save(new File(outfile).getAbsoluteFile());
}
@@ -3154,6 +3150,19 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
@Override
+ public String actionStatus(String clustername)
+ throws YarnException, IOException {
+ return verifyAndGetClusterDescription(clustername).toJsonString();
+ }
+
+ private ClusterDescription verifyAndGetClusterDescription(String clustername)
+ throws YarnException, IOException {
+ verifyBindingsDefined();
+ validateClusterName(clustername);
+ return getClusterDescription(clustername);
+ }
+
+ @Override
public int actionVersion() {
SliderVersionInfo.loadAndPrintVersionInfo(log);
return EXIT_SUCCESS;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31848c7e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
index 30f6ba9..c6cc2d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
@@ -280,6 +280,17 @@ public interface SliderClientAPI extends Service {
throws YarnException, IOException;
/**
+ * Status operation which returns the status object as a string instead of
+ * printing it to the console or file.
+ *
+ * @param clustername cluster name
+ * @return cluster status details
+ * @throws YarnException
+ * @throws IOException
+ */
+ String actionStatus(String clustername) throws YarnException, IOException;
+
+ /**
* Version Details
* @return exit code
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[51/51] [abbrv] hadoop git commit: YARN-5828. Native services client
errors out when config formats are uppercase. Contributed by Billie Rinaldi
Posted by ji...@apache.org.
YARN-5828. Native services client errors out when config formats are uppercase. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/af1025ca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/af1025ca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/af1025ca
Branch: refs/heads/yarn-native-services
Commit: af1025ca3323b3dd9341918680bf877b5d8b13e9
Parents: a55eb5d
Author: Gour Saha <go...@apache.org>
Authored: Thu Nov 3 18:15:44 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../org/apache/slider/core/registry/docstore/ConfigFormat.java | 4 +++-
.../src/main/java/org/apache/slider/providers/ProviderUtils.java | 2 +-
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af1025ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
index ddab606..723b975 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
@@ -18,6 +18,8 @@
package org.apache.slider.core.registry.docstore;
+import java.util.Locale;
+
public enum ConfigFormat {
JSON("json"),
@@ -51,7 +53,7 @@ public enum ConfigFormat {
*/
public static ConfigFormat resolve(String type) {
for (ConfigFormat format: values()) {
- if (format.getSuffix().equals(type)) {
+ if (format.getSuffix().equals(type.toLowerCase(Locale.ENGLISH))) {
return format;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af1025ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index c5e6782..39986c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -611,7 +611,7 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
}
ConfigFormat configFormat = ConfigFormat.resolve(configFileType);
if (configFormat == null) {
- throw new BadConfigException("Config format " + configFormat +
+ throw new BadConfigException("Config format " + configFileType +
" doesn't exist");
}
localizeConfigFile(launcher, roleName, roleGroup, configEntry.getKey(),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/51] [abbrv] hadoop git commit: YARN-5961. Generate native
services protobuf classes during build. Contributed by Billie Rinaldi
Posted by ji...@apache.org.
YARN-5961. Generate native services protobuf classes during build. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a62a4905
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a62a4905
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a62a4905
Branch: refs/heads/yarn-native-services
Commit: a62a490558e9356cc4dfd6a8671d5b20a9675328
Parents: c1459aa
Author: Jian He <ji...@apache.org>
Authored: Sat Dec 3 13:27:04 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../hadoop-yarn-slider-core/pom.xml | 65 +-
.../org/apache/slider/api/proto/Messages.java | 34473 -----------------
.../slider/api/proto/SliderClusterAPI.java | 2293 --
3 files changed, 27 insertions(+), 36804 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62a4905/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index 10cf6b1..7453d12 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -37,6 +37,33 @@
<plugins>
<plugin>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-maven-plugins</artifactId>
+ <executions>
+ <execution>
+ <id>compile-protoc</id>
+ <goals>
+ <goal>protoc</goal>
+ </goals>
+ <configuration>
+ <protocVersion>${protobuf.version}</protocVersion>
+ <protocCommand>${protoc.path}</protocCommand>
+ <imports>
+ <param>${basedir}/src/main/proto</param>
+ </imports>
+ <source>
+ <directory>${basedir}/src/main/proto</directory>
+ <includes>
+ <include>SliderClusterMessages.proto</include>
+ <include>SliderClusterProtocol.proto</include>
+ </includes>
+ </source>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
@@ -85,9 +112,6 @@
<excludes>
<exclude>**/*.json</exclude>
<exclude>src/main/resources/webapps/slideram/.keep</exclude>
- <!-- protobuf generated classes -->
- <exclude>src/main/java/org/apache/slider/api/proto/Messages.java</exclude>
- <exclude>src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java</exclude>
</excludes>
</configuration>
</plugin>
@@ -362,41 +386,6 @@
</plugins>
</build>
</profile>
- <profile>
- <id>compile-protobuf</id>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-maven-plugins</artifactId>
- <executions>
- <execution>
- <id>compile-protoc</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>protoc</goal>
- </goals>
- <configuration>
- <protocVersion>${protobuf.version}</protocVersion>
- <protocCommand>protoc</protocCommand>
- <imports>
- <param>${basedir}/src/main/proto</param>
- </imports>
- <source>
- <directory>${basedir}/src/main/proto</directory>
- <includes>
- <include>SliderClusterMessages.proto</include>
- <include>SliderClusterProtocol.proto</include>
- </includes>
- </source>
- <output>${basedir}/src/main/java</output>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
</profiles>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/51] [abbrv] hadoop git commit: YARN-5961. Generate native
services protobuf classes during build. Contributed by Billie Rinaldi
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62a4905/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
deleted file mode 100644
index 081b7fa..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
+++ /dev/null
@@ -1,2293 +0,0 @@
-// Generated by the protocol buffer compiler. DO NOT EDIT!
-// source: SliderClusterProtocol.proto
-
-package org.apache.slider.api.proto;
-
-public final class SliderClusterAPI {
- private SliderClusterAPI() {}
- public static void registerAllExtensions(
- com.google.protobuf.ExtensionRegistry registry) {
- }
- /**
- * Protobuf service {@code org.apache.slider.api.SliderClusterProtocolPB}
- *
- * <pre>
- **
- * Protocol used from between Slider Client and AM
- * </pre>
- */
- public static abstract class SliderClusterProtocolPB
- implements com.google.protobuf.Service {
- protected SliderClusterProtocolPB() {}
-
- public interface Interface {
- /**
- * <code>rpc stopCluster(.org.apache.slider.api.StopClusterRequestProto) returns (.org.apache.slider.api.StopClusterResponseProto);</code>
- */
- public abstract void stopCluster(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.StopClusterRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.StopClusterResponseProto> done);
-
- /**
- * <code>rpc upgradeContainers(.org.apache.slider.api.UpgradeContainersRequestProto) returns (.org.apache.slider.api.UpgradeContainersResponseProto);</code>
- *
- * <pre>
- **
- * Upgrade containers
- * </pre>
- */
- public abstract void upgradeContainers(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.UpgradeContainersRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.UpgradeContainersResponseProto> done);
-
- /**
- * <code>rpc flexCluster(.org.apache.slider.api.FlexClusterRequestProto) returns (.org.apache.slider.api.FlexClusterResponseProto);</code>
- *
- * <pre>
- **
- * Flex the cluster.
- * </pre>
- */
- public abstract void flexCluster(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.FlexClusterRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.FlexClusterResponseProto> done);
-
- /**
- * <code>rpc getJSONClusterStatus(.org.apache.slider.api.GetJSONClusterStatusRequestProto) returns (.org.apache.slider.api.GetJSONClusterStatusResponseProto);</code>
- *
- * <pre>
- **
- * Get the current cluster status
- * </pre>
- */
- public abstract void getJSONClusterStatus(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetJSONClusterStatusRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetJSONClusterStatusResponseProto> done);
-
- /**
- * <code>rpc getInstanceDefinition(.org.apache.slider.api.GetInstanceDefinitionRequestProto) returns (.org.apache.slider.api.GetInstanceDefinitionResponseProto);</code>
- *
- * <pre>
- **
- * Get the instance definition
- * </pre>
- */
- public abstract void getInstanceDefinition(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetInstanceDefinitionRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetInstanceDefinitionResponseProto> done);
-
- /**
- * <code>rpc listNodeUUIDsByRole(.org.apache.slider.api.ListNodeUUIDsByRoleRequestProto) returns (.org.apache.slider.api.ListNodeUUIDsByRoleResponseProto);</code>
- *
- * <pre>
- **
- * List all running nodes in a role
- * </pre>
- */
- public abstract void listNodeUUIDsByRole(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleResponseProto> done);
-
- /**
- * <code>rpc getNode(.org.apache.slider.api.GetNodeRequestProto) returns (.org.apache.slider.api.GetNodeResponseProto);</code>
- *
- * <pre>
- **
- * Get the details on a node
- * </pre>
- */
- public abstract void getNode(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetNodeRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetNodeResponseProto> done);
-
- /**
- * <code>rpc getClusterNodes(.org.apache.slider.api.GetClusterNodesRequestProto) returns (.org.apache.slider.api.GetClusterNodesResponseProto);</code>
- *
- * <pre>
- **
- * Get the
- * details on a list of nodes.
- * Unknown nodes are not returned
- * <i>Important: the order of the results are undefined</i>
- * </pre>
- */
- public abstract void getClusterNodes(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetClusterNodesRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetClusterNodesResponseProto> done);
-
- /**
- * <code>rpc echo(.org.apache.slider.api.EchoRequestProto) returns (.org.apache.slider.api.EchoResponseProto);</code>
- *
- * <pre>
- **
- * echo some text
- * </pre>
- */
- public abstract void echo(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EchoRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.EchoResponseProto> done);
-
- /**
- * <code>rpc killContainer(.org.apache.slider.api.KillContainerRequestProto) returns (.org.apache.slider.api.KillContainerResponseProto);</code>
- *
- * <pre>
- **
- * kill a container
- * </pre>
- */
- public abstract void killContainer(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.KillContainerRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.KillContainerResponseProto> done);
-
- /**
- * <code>rpc amSuicide(.org.apache.slider.api.AMSuicideRequestProto) returns (.org.apache.slider.api.AMSuicideResponseProto);</code>
- *
- * <pre>
- **
- * kill the AM
- * </pre>
- */
- public abstract void amSuicide(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.AMSuicideRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.AMSuicideResponseProto> done);
-
- /**
- * <code>rpc getLivenessInformation(.org.apache.slider.api.GetApplicationLivenessRequestProto) returns (.org.apache.slider.api.ApplicationLivenessInformationProto);</code>
- */
- public abstract void getLivenessInformation(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetApplicationLivenessRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.ApplicationLivenessInformationProto> done);
-
- /**
- * <code>rpc getLiveContainers(.org.apache.slider.api.GetLiveContainersRequestProto) returns (.org.apache.slider.api.GetLiveContainersResponseProto);</code>
- */
- public abstract void getLiveContainers(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveContainersRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetLiveContainersResponseProto> done);
-
- /**
- * <code>rpc getLiveContainer(.org.apache.slider.api.GetLiveContainerRequestProto) returns (.org.apache.slider.api.ContainerInformationProto);</code>
- */
- public abstract void getLiveContainer(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveContainerRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.ContainerInformationProto> done);
-
- /**
- * <code>rpc getLiveComponents(.org.apache.slider.api.GetLiveComponentsRequestProto) returns (.org.apache.slider.api.GetLiveComponentsResponseProto);</code>
- */
- public abstract void getLiveComponents(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveComponentsRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetLiveComponentsResponseProto> done);
-
- /**
- * <code>rpc getLiveComponent(.org.apache.slider.api.GetLiveComponentRequestProto) returns (.org.apache.slider.api.ComponentInformationProto);</code>
- */
- public abstract void getLiveComponent(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveComponentRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.ComponentInformationProto> done);
-
- /**
- * <code>rpc getLiveNodes(.org.apache.slider.api.GetLiveNodesRequestProto) returns (.org.apache.slider.api.GetLiveNodesResponseProto);</code>
- */
- public abstract void getLiveNodes(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto> done);
-
- /**
- * <code>rpc getLiveNode(.org.apache.slider.api.GetLiveNodeRequestProto) returns (.org.apache.slider.api.NodeInformationProto);</code>
- */
- public abstract void getLiveNode(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.NodeInformationProto> done);
-
- /**
- * <code>rpc getModelDesired(.org.apache.slider.api.EmptyPayloadProto) returns (.org.apache.slider.api.WrappedJsonProto);</code>
- *
- * <pre>
- * AggregateConf getModelDesired()
- * </pre>
- */
- public abstract void getModelDesired(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done);
-
- /**
- * <code>rpc getModelDesiredAppconf(.org.apache.slider.api.EmptyPayloadProto) returns (.org.apache.slider.api.WrappedJsonProto);</code>
- *
- * <pre>
- * ConfTree getModelDesiredAppconf
- * </pre>
- */
- public abstract void getModelDesiredAppconf(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done);
-
- /**
- * <code>rpc getModelDesiredResources(.org.apache.slider.api.EmptyPayloadProto) returns (.org.apache.slider.api.WrappedJsonProto);</code>
- *
- * <pre>
- * ConfTree getModelDesiredResources
- * </pre>
- */
- public abstract void getModelDesiredResources(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done);
-
- /**
- * <code>rpc getModelResolved(.org.apache.slider.api.EmptyPayloadProto) returns (.org.apache.slider.api.WrappedJsonProto);</code>
- *
- * <pre>
- * AggregateConf getModelResolved()
- * </pre>
- */
- public abstract void getModelResolved(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done);
-
- /**
- * <code>rpc getModelResolvedAppconf(.org.apache.slider.api.EmptyPayloadProto) returns (.org.apache.slider.api.WrappedJsonProto);</code>
- *
- * <pre>
- * ConfTree getModelResolvedAppconf
- * </pre>
- */
- public abstract void getModelResolvedAppconf(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done);
-
- /**
- * <code>rpc getModelResolvedResources(.org.apache.slider.api.EmptyPayloadProto) returns (.org.apache.slider.api.WrappedJsonProto);</code>
- *
- * <pre>
- * ConfTree getModelResolvedResources
- * </pre>
- */
- public abstract void getModelResolvedResources(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done);
-
- /**
- * <code>rpc getLiveResources(.org.apache.slider.api.EmptyPayloadProto) returns (.org.apache.slider.api.WrappedJsonProto);</code>
- *
- * <pre>
- * ConfTree getLiveResources
- * </pre>
- */
- public abstract void getLiveResources(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done);
-
- /**
- * <code>rpc getClientCertificateStore(.org.apache.slider.api.GetCertificateStoreRequestProto) returns (.org.apache.slider.api.GetCertificateStoreResponseProto);</code>
- */
- public abstract void getClientCertificateStore(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetCertificateStoreRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetCertificateStoreResponseProto> done);
-
- }
-
- public static com.google.protobuf.Service newReflectiveService(
- final Interface impl) {
- return new SliderClusterProtocolPB() {
- @java.lang.Override
- public void stopCluster(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.StopClusterRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.StopClusterResponseProto> done) {
- impl.stopCluster(controller, request, done);
- }
-
- @java.lang.Override
- public void upgradeContainers(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.UpgradeContainersRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.UpgradeContainersResponseProto> done) {
- impl.upgradeContainers(controller, request, done);
- }
-
- @java.lang.Override
- public void flexCluster(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.FlexClusterRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.FlexClusterResponseProto> done) {
- impl.flexCluster(controller, request, done);
- }
-
- @java.lang.Override
- public void getJSONClusterStatus(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetJSONClusterStatusRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetJSONClusterStatusResponseProto> done) {
- impl.getJSONClusterStatus(controller, request, done);
- }
-
- @java.lang.Override
- public void getInstanceDefinition(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetInstanceDefinitionRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetInstanceDefinitionResponseProto> done) {
- impl.getInstanceDefinition(controller, request, done);
- }
-
- @java.lang.Override
- public void listNodeUUIDsByRole(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleResponseProto> done) {
- impl.listNodeUUIDsByRole(controller, request, done);
- }
-
- @java.lang.Override
- public void getNode(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetNodeRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetNodeResponseProto> done) {
- impl.getNode(controller, request, done);
- }
-
- @java.lang.Override
- public void getClusterNodes(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetClusterNodesRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetClusterNodesResponseProto> done) {
- impl.getClusterNodes(controller, request, done);
- }
-
- @java.lang.Override
- public void echo(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EchoRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.EchoResponseProto> done) {
- impl.echo(controller, request, done);
- }
-
- @java.lang.Override
- public void killContainer(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.KillContainerRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.KillContainerResponseProto> done) {
- impl.killContainer(controller, request, done);
- }
-
- @java.lang.Override
- public void amSuicide(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.AMSuicideRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.AMSuicideResponseProto> done) {
- impl.amSuicide(controller, request, done);
- }
-
- @java.lang.Override
- public void getLivenessInformation(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetApplicationLivenessRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.ApplicationLivenessInformationProto> done) {
- impl.getLivenessInformation(controller, request, done);
- }
-
- @java.lang.Override
- public void getLiveContainers(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveContainersRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetLiveContainersResponseProto> done) {
- impl.getLiveContainers(controller, request, done);
- }
-
- @java.lang.Override
- public void getLiveContainer(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveContainerRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.ContainerInformationProto> done) {
- impl.getLiveContainer(controller, request, done);
- }
-
- @java.lang.Override
- public void getLiveComponents(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveComponentsRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetLiveComponentsResponseProto> done) {
- impl.getLiveComponents(controller, request, done);
- }
-
- @java.lang.Override
- public void getLiveComponent(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveComponentRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.ComponentInformationProto> done) {
- impl.getLiveComponent(controller, request, done);
- }
-
- @java.lang.Override
- public void getLiveNodes(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto> done) {
- impl.getLiveNodes(controller, request, done);
- }
-
- @java.lang.Override
- public void getLiveNode(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.NodeInformationProto> done) {
- impl.getLiveNode(controller, request, done);
- }
-
- @java.lang.Override
- public void getModelDesired(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
- impl.getModelDesired(controller, request, done);
- }
-
- @java.lang.Override
- public void getModelDesiredAppconf(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
- impl.getModelDesiredAppconf(controller, request, done);
- }
-
- @java.lang.Override
- public void getModelDesiredResources(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
- impl.getModelDesiredResources(controller, request, done);
- }
-
- @java.lang.Override
- public void getModelResolved(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
- impl.getModelResolved(controller, request, done);
- }
-
- @java.lang.Override
- public void getModelResolvedAppconf(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
- impl.getModelResolvedAppconf(controller, request, done);
- }
-
- @java.lang.Override
- public void getModelResolvedResources(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
- impl.getModelResolvedResources(controller, request, done);
- }
-
- @java.lang.Override
- public void getLiveResources(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
- impl.getLiveResources(controller, request, done);
- }
-
- @java.lang.Override
- public void getClientCertificateStore(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetCertificateStoreRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetCertificateStoreResponseProto> done) {
- impl.getClientCertificateStore(controller, request, done);
- }
-
- };
- }
-
- public static com.google.protobuf.BlockingService
- newReflectiveBlockingService(final BlockingInterface impl) {
- return new com.google.protobuf.BlockingService() {
- public final com.google.protobuf.Descriptors.ServiceDescriptor
- getDescriptorForType() {
- return getDescriptor();
- }
-
- public final com.google.protobuf.Message callBlockingMethod(
- com.google.protobuf.Descriptors.MethodDescriptor method,
- com.google.protobuf.RpcController controller,
- com.google.protobuf.Message request)
- throws com.google.protobuf.ServiceException {
- if (method.getService() != getDescriptor()) {
- throw new java.lang.IllegalArgumentException(
- "Service.callBlockingMethod() given method descriptor for " +
- "wrong service type.");
- }
- switch(method.getIndex()) {
- case 0:
- return impl.stopCluster(controller, (org.apache.slider.api.proto.Messages.StopClusterRequestProto)request);
- case 1:
- return impl.upgradeContainers(controller, (org.apache.slider.api.proto.Messages.UpgradeContainersRequestProto)request);
- case 2:
- return impl.flexCluster(controller, (org.apache.slider.api.proto.Messages.FlexClusterRequestProto)request);
- case 3:
- return impl.getJSONClusterStatus(controller, (org.apache.slider.api.proto.Messages.GetJSONClusterStatusRequestProto)request);
- case 4:
- return impl.getInstanceDefinition(controller, (org.apache.slider.api.proto.Messages.GetInstanceDefinitionRequestProto)request);
- case 5:
- return impl.listNodeUUIDsByRole(controller, (org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleRequestProto)request);
- case 6:
- return impl.getNode(controller, (org.apache.slider.api.proto.Messages.GetNodeRequestProto)request);
- case 7:
- return impl.getClusterNodes(controller, (org.apache.slider.api.proto.Messages.GetClusterNodesRequestProto)request);
- case 8:
- return impl.echo(controller, (org.apache.slider.api.proto.Messages.EchoRequestProto)request);
- case 9:
- return impl.killContainer(controller, (org.apache.slider.api.proto.Messages.KillContainerRequestProto)request);
- case 10:
- return impl.amSuicide(controller, (org.apache.slider.api.proto.Messages.AMSuicideRequestProto)request);
- case 11:
- return impl.getLivenessInformation(controller, (org.apache.slider.api.proto.Messages.GetApplicationLivenessRequestProto)request);
- case 12:
- return impl.getLiveContainers(controller, (org.apache.slider.api.proto.Messages.GetLiveContainersRequestProto)request);
- case 13:
- return impl.getLiveContainer(controller, (org.apache.slider.api.proto.Messages.GetLiveContainerRequestProto)request);
- case 14:
- return impl.getLiveComponents(controller, (org.apache.slider.api.proto.Messages.GetLiveComponentsRequestProto)request);
- case 15:
- return impl.getLiveComponent(controller, (org.apache.slider.api.proto.Messages.GetLiveComponentRequestProto)request);
- case 16:
- return impl.getLiveNodes(controller, (org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto)request);
- case 17:
- return impl.getLiveNode(controller, (org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto)request);
- case 18:
- return impl.getModelDesired(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request);
- case 19:
- return impl.getModelDesiredAppconf(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request);
- case 20:
- return impl.getModelDesiredResources(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request);
- case 21:
- return impl.getModelResolved(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request);
- case 22:
- return impl.getModelResolvedAppconf(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request);
- case 23:
- return impl.getModelResolvedResources(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request);
- case 24:
- return impl.getLiveResources(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request);
- case 25:
- return impl.getClientCertificateStore(controller, (org.apache.slider.api.proto.Messages.GetCertificateStoreRequestProto)request);
- default:
- throw new java.lang.AssertionError("Can't get here.");
- }
- }
-
- public final com.google.protobuf.Message
- getRequestPrototype(
- com.google.protobuf.Descriptors.MethodDescriptor method) {
- if (method.getService() != getDescriptor()) {
- throw new java.lang.IllegalArgumentException(
- "Service.getRequestPrototype() given method " +
- "descriptor for wrong service type.");
- }
- switch(method.getIndex()) {
- case 0:
- return org.apache.slider.api.proto.Messages.StopClusterRequestProto.getDefaultInstance();
- case 1:
- return org.apache.slider.api.proto.Messages.UpgradeContainersRequestProto.getDefaultInstance();
- case 2:
- return org.apache.slider.api.proto.Messages.FlexClusterRequestProto.getDefaultInstance();
- case 3:
- return org.apache.slider.api.proto.Messages.GetJSONClusterStatusRequestProto.getDefaultInstance();
- case 4:
- return org.apache.slider.api.proto.Messages.GetInstanceDefinitionRequestProto.getDefaultInstance();
- case 5:
- return org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleRequestProto.getDefaultInstance();
- case 6:
- return org.apache.slider.api.proto.Messages.GetNodeRequestProto.getDefaultInstance();
- case 7:
- return org.apache.slider.api.proto.Messages.GetClusterNodesRequestProto.getDefaultInstance();
- case 8:
- return org.apache.slider.api.proto.Messages.EchoRequestProto.getDefaultInstance();
- case 9:
- return org.apache.slider.api.proto.Messages.KillContainerRequestProto.getDefaultInstance();
- case 10:
- return org.apache.slider.api.proto.Messages.AMSuicideRequestProto.getDefaultInstance();
- case 11:
- return org.apache.slider.api.proto.Messages.GetApplicationLivenessRequestProto.getDefaultInstance();
- case 12:
- return org.apache.slider.api.proto.Messages.GetLiveContainersRequestProto.getDefaultInstance();
- case 13:
- return org.apache.slider.api.proto.Messages.GetLiveContainerRequestProto.getDefaultInstance();
- case 14:
- return org.apache.slider.api.proto.Messages.GetLiveComponentsRequestProto.getDefaultInstance();
- case 15:
- return org.apache.slider.api.proto.Messages.GetLiveComponentRequestProto.getDefaultInstance();
- case 16:
- return org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto.getDefaultInstance();
- case 17:
- return org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto.getDefaultInstance();
- case 18:
- return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
- case 19:
- return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
- case 20:
- return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
- case 21:
- return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
- case 22:
- return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
- case 23:
- return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
- case 24:
- return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
- case 25:
- return org.apache.slider.api.proto.Messages.GetCertificateStoreRequestProto.getDefaultInstance();
- default:
- throw new java.lang.AssertionError("Can't get here.");
- }
- }
-
- public final com.google.protobuf.Message
- getResponsePrototype(
- com.google.protobuf.Descriptors.MethodDescriptor method) {
- if (method.getService() != getDescriptor()) {
- throw new java.lang.IllegalArgumentException(
- "Service.getResponsePrototype() given method " +
- "descriptor for wrong service type.");
- }
- switch(method.getIndex()) {
- case 0:
- return org.apache.slider.api.proto.Messages.StopClusterResponseProto.getDefaultInstance();
- case 1:
- return org.apache.slider.api.proto.Messages.UpgradeContainersResponseProto.getDefaultInstance();
- case 2:
- return org.apache.slider.api.proto.Messages.FlexClusterResponseProto.getDefaultInstance();
- case 3:
- return org.apache.slider.api.proto.Messages.GetJSONClusterStatusResponseProto.getDefaultInstance();
- case 4:
- return org.apache.slider.api.proto.Messages.GetInstanceDefinitionResponseProto.getDefaultInstance();
- case 5:
- return org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleResponseProto.getDefaultInstance();
- case 6:
- return org.apache.slider.api.proto.Messages.GetNodeResponseProto.getDefaultInstance();
- case 7:
- return org.apache.slider.api.proto.Messages.GetClusterNodesResponseProto.getDefaultInstance();
- case 8:
- return org.apache.slider.api.proto.Messages.EchoResponseProto.getDefaultInstance();
- case 9:
- return org.apache.slider.api.proto.Messages.KillContainerResponseProto.getDefaultInstance();
- case 10:
- return org.apache.slider.api.proto.Messages.AMSuicideResponseProto.getDefaultInstance();
- case 11:
- return org.apache.slider.api.proto.Messages.ApplicationLivenessInformationProto.getDefaultInstance();
- case 12:
- return org.apache.slider.api.proto.Messages.GetLiveContainersResponseProto.getDefaultInstance();
- case 13:
- return org.apache.slider.api.proto.Messages.ContainerInformationProto.getDefaultInstance();
- case 14:
- return org.apache.slider.api.proto.Messages.GetLiveComponentsResponseProto.getDefaultInstance();
- case 15:
- return org.apache.slider.api.proto.Messages.ComponentInformationProto.getDefaultInstance();
- case 16:
- return org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto.getDefaultInstance();
- case 17:
- return org.apache.slider.api.proto.Messages.NodeInformationProto.getDefaultInstance();
- case 18:
- return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
- case 19:
- return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
- case 20:
- return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
- case 21:
- return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
- case 22:
- return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
- case 23:
- return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
- case 24:
- return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
- case 25:
- return org.apache.slider.api.proto.Messages.GetCertificateStoreResponseProto.getDefaultInstance();
- default:
- throw new java.lang.AssertionError("Can't get here.");
- }
- }
-
- };
- }
-
- /**
- * <code>rpc stopCluster(.org.apache.slider.api.StopClusterRequestProto) returns (.org.apache.slider.api.StopClusterResponseProto);</code>
- */
- public abstract void stopCluster(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.StopClusterRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.StopClusterResponseProto> done);
-
- /**
- * <code>rpc upgradeContainers(.org.apache.slider.api.UpgradeContainersRequestProto) returns (.org.apache.slider.api.UpgradeContainersResponseProto);</code>
- *
- * <pre>
- **
- * Upgrade containers
- * </pre>
- */
- public abstract void upgradeContainers(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.UpgradeContainersRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.UpgradeContainersResponseProto> done);
-
- /**
- * <code>rpc flexCluster(.org.apache.slider.api.FlexClusterRequestProto) returns (.org.apache.slider.api.FlexClusterResponseProto);</code>
- *
- * <pre>
- **
- * Flex the cluster.
- * </pre>
- */
- public abstract void flexCluster(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.FlexClusterRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.FlexClusterResponseProto> done);
-
- /**
- * <code>rpc getJSONClusterStatus(.org.apache.slider.api.GetJSONClusterStatusRequestProto) returns (.org.apache.slider.api.GetJSONClusterStatusResponseProto);</code>
- *
- * <pre>
- **
- * Get the current cluster status
- * </pre>
- */
- public abstract void getJSONClusterStatus(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetJSONClusterStatusRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetJSONClusterStatusResponseProto> done);
-
- /**
- * <code>rpc getInstanceDefinition(.org.apache.slider.api.GetInstanceDefinitionRequestProto) returns (.org.apache.slider.api.GetInstanceDefinitionResponseProto);</code>
- *
- * <pre>
- **
- * Get the instance definition
- * </pre>
- */
- public abstract void getInstanceDefinition(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetInstanceDefinitionRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetInstanceDefinitionResponseProto> done);
-
- /**
- * <code>rpc listNodeUUIDsByRole(.org.apache.slider.api.ListNodeUUIDsByRoleRequestProto) returns (.org.apache.slider.api.ListNodeUUIDsByRoleResponseProto);</code>
- *
- * <pre>
- **
- * List all running nodes in a role
- * </pre>
- */
- public abstract void listNodeUUIDsByRole(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleResponseProto> done);
-
- /**
- * <code>rpc getNode(.org.apache.slider.api.GetNodeRequestProto) returns (.org.apache.slider.api.GetNodeResponseProto);</code>
- *
- * <pre>
- **
- * Get the details on a node
- * </pre>
- */
- public abstract void getNode(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetNodeRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetNodeResponseProto> done);
-
- /**
- * <code>rpc getClusterNodes(.org.apache.slider.api.GetClusterNodesRequestProto) returns (.org.apache.slider.api.GetClusterNodesResponseProto);</code>
- *
- * <pre>
- **
- * Get the
- * details on a list of nodes.
- * Unknown nodes are not returned
- * <i>Important: the order of the results are undefined</i>
- * </pre>
- */
- public abstract void getClusterNodes(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetClusterNodesRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetClusterNodesResponseProto> done);
-
- /**
- * <code>rpc echo(.org.apache.slider.api.EchoRequestProto) returns (.org.apache.slider.api.EchoResponseProto);</code>
- *
- * <pre>
- **
- * echo some text
- * </pre>
- */
- public abstract void echo(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EchoRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.EchoResponseProto> done);
-
- /**
- * <code>rpc killContainer(.org.apache.slider.api.KillContainerRequestProto) returns (.org.apache.slider.api.KillContainerResponseProto);</code>
- *
- * <pre>
- **
- * kill a container
- * </pre>
- */
- public abstract void killContainer(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.KillContainerRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.KillContainerResponseProto> done);
-
- /**
- * <code>rpc amSuicide(.org.apache.slider.api.AMSuicideRequestProto) returns (.org.apache.slider.api.AMSuicideResponseProto);</code>
- *
- * <pre>
- **
- * kill the AM
- * </pre>
- */
- public abstract void amSuicide(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.AMSuicideRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.AMSuicideResponseProto> done);
-
- /**
- * <code>rpc getLivenessInformation(.org.apache.slider.api.GetApplicationLivenessRequestProto) returns (.org.apache.slider.api.ApplicationLivenessInformationProto);</code>
- */
- public abstract void getLivenessInformation(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetApplicationLivenessRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.ApplicationLivenessInformationProto> done);
-
- /**
- * <code>rpc getLiveContainers(.org.apache.slider.api.GetLiveContainersRequestProto) returns (.org.apache.slider.api.GetLiveContainersResponseProto);</code>
- */
- public abstract void getLiveContainers(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveContainersRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetLiveContainersResponseProto> done);
-
- /**
- * <code>rpc getLiveContainer(.org.apache.slider.api.GetLiveContainerRequestProto) returns (.org.apache.slider.api.ContainerInformationProto);</code>
- */
- public abstract void getLiveContainer(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveContainerRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.ContainerInformationProto> done);
-
- /**
- * <code>rpc getLiveComponents(.org.apache.slider.api.GetLiveComponentsRequestProto) returns (.org.apache.slider.api.GetLiveComponentsResponseProto);</code>
- */
- public abstract void getLiveComponents(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveComponentsRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetLiveComponentsResponseProto> done);
-
- /**
- * <code>rpc getLiveComponent(.org.apache.slider.api.GetLiveComponentRequestProto) returns (.org.apache.slider.api.ComponentInformationProto);</code>
- */
- public abstract void getLiveComponent(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveComponentRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.ComponentInformationProto> done);
-
- /**
- * <code>rpc getLiveNodes(.org.apache.slider.api.GetLiveNodesRequestProto) returns (.org.apache.slider.api.GetLiveNodesResponseProto);</code>
- */
- public abstract void getLiveNodes(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto> done);
-
- /**
- * <code>rpc getLiveNode(.org.apache.slider.api.GetLiveNodeRequestProto) returns (.org.apache.slider.api.NodeInformationProto);</code>
- */
- public abstract void getLiveNode(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.NodeInformationProto> done);
-
- /**
- * <code>rpc getModelDesired(.org.apache.slider.api.EmptyPayloadProto) returns (.org.apache.slider.api.WrappedJsonProto);</code>
- *
- * <pre>
- * AggregateConf getModelDesired()
- * </pre>
- */
- public abstract void getModelDesired(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done);
-
- /**
- * <code>rpc getModelDesiredAppconf(.org.apache.slider.api.EmptyPayloadProto) returns (.org.apache.slider.api.WrappedJsonProto);</code>
- *
- * <pre>
- * ConfTree getModelDesiredAppconf
- * </pre>
- */
- public abstract void getModelDesiredAppconf(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done);
-
- /**
- * <code>rpc getModelDesiredResources(.org.apache.slider.api.EmptyPayloadProto) returns (.org.apache.slider.api.WrappedJsonProto);</code>
- *
- * <pre>
- * ConfTree getModelDesiredResources
- * </pre>
- */
- public abstract void getModelDesiredResources(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done);
-
- /**
- * <code>rpc getModelResolved(.org.apache.slider.api.EmptyPayloadProto) returns (.org.apache.slider.api.WrappedJsonProto);</code>
- *
- * <pre>
- * AggregateConf getModelResolved()
- * </pre>
- */
- public abstract void getModelResolved(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done);
-
- /**
- * <code>rpc getModelResolvedAppconf(.org.apache.slider.api.EmptyPayloadProto) returns (.org.apache.slider.api.WrappedJsonProto);</code>
- *
- * <pre>
- * ConfTree getModelResolvedAppconf
- * </pre>
- */
- public abstract void getModelResolvedAppconf(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done);
-
- /**
- * <code>rpc getModelResolvedResources(.org.apache.slider.api.EmptyPayloadProto) returns (.org.apache.slider.api.WrappedJsonProto);</code>
- *
- * <pre>
- * ConfTree getModelResolvedResources
- * </pre>
- */
- public abstract void getModelResolvedResources(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done);
-
- /**
- * <code>rpc getLiveResources(.org.apache.slider.api.EmptyPayloadProto) returns (.org.apache.slider.api.WrappedJsonProto);</code>
- *
- * <pre>
- * ConfTree getLiveResources
- * </pre>
- */
- public abstract void getLiveResources(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done);
-
- /**
- * <code>rpc getClientCertificateStore(.org.apache.slider.api.GetCertificateStoreRequestProto) returns (.org.apache.slider.api.GetCertificateStoreResponseProto);</code>
- */
- public abstract void getClientCertificateStore(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetCertificateStoreRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetCertificateStoreResponseProto> done);
-
- public static final
- com.google.protobuf.Descriptors.ServiceDescriptor
- getDescriptor() {
- return org.apache.slider.api.proto.SliderClusterAPI.getDescriptor().getServices().get(0);
- }
- public final com.google.protobuf.Descriptors.ServiceDescriptor
- getDescriptorForType() {
- return getDescriptor();
- }
-
- public final void callMethod(
- com.google.protobuf.Descriptors.MethodDescriptor method,
- com.google.protobuf.RpcController controller,
- com.google.protobuf.Message request,
- com.google.protobuf.RpcCallback<
- com.google.protobuf.Message> done) {
- if (method.getService() != getDescriptor()) {
- throw new java.lang.IllegalArgumentException(
- "Service.callMethod() given method descriptor for wrong " +
- "service type.");
- }
- switch(method.getIndex()) {
- case 0:
- this.stopCluster(controller, (org.apache.slider.api.proto.Messages.StopClusterRequestProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.StopClusterResponseProto>specializeCallback(
- done));
- return;
- case 1:
- this.upgradeContainers(controller, (org.apache.slider.api.proto.Messages.UpgradeContainersRequestProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.UpgradeContainersResponseProto>specializeCallback(
- done));
- return;
- case 2:
- this.flexCluster(controller, (org.apache.slider.api.proto.Messages.FlexClusterRequestProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.FlexClusterResponseProto>specializeCallback(
- done));
- return;
- case 3:
- this.getJSONClusterStatus(controller, (org.apache.slider.api.proto.Messages.GetJSONClusterStatusRequestProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.GetJSONClusterStatusResponseProto>specializeCallback(
- done));
- return;
- case 4:
- this.getInstanceDefinition(controller, (org.apache.slider.api.proto.Messages.GetInstanceDefinitionRequestProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.GetInstanceDefinitionResponseProto>specializeCallback(
- done));
- return;
- case 5:
- this.listNodeUUIDsByRole(controller, (org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleRequestProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleResponseProto>specializeCallback(
- done));
- return;
- case 6:
- this.getNode(controller, (org.apache.slider.api.proto.Messages.GetNodeRequestProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.GetNodeResponseProto>specializeCallback(
- done));
- return;
- case 7:
- this.getClusterNodes(controller, (org.apache.slider.api.proto.Messages.GetClusterNodesRequestProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.GetClusterNodesResponseProto>specializeCallback(
- done));
- return;
- case 8:
- this.echo(controller, (org.apache.slider.api.proto.Messages.EchoRequestProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.EchoResponseProto>specializeCallback(
- done));
- return;
- case 9:
- this.killContainer(controller, (org.apache.slider.api.proto.Messages.KillContainerRequestProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.KillContainerResponseProto>specializeCallback(
- done));
- return;
- case 10:
- this.amSuicide(controller, (org.apache.slider.api.proto.Messages.AMSuicideRequestProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.AMSuicideResponseProto>specializeCallback(
- done));
- return;
- case 11:
- this.getLivenessInformation(controller, (org.apache.slider.api.proto.Messages.GetApplicationLivenessRequestProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.ApplicationLivenessInformationProto>specializeCallback(
- done));
- return;
- case 12:
- this.getLiveContainers(controller, (org.apache.slider.api.proto.Messages.GetLiveContainersRequestProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.GetLiveContainersResponseProto>specializeCallback(
- done));
- return;
- case 13:
- this.getLiveContainer(controller, (org.apache.slider.api.proto.Messages.GetLiveContainerRequestProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.ContainerInformationProto>specializeCallback(
- done));
- return;
- case 14:
- this.getLiveComponents(controller, (org.apache.slider.api.proto.Messages.GetLiveComponentsRequestProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.GetLiveComponentsResponseProto>specializeCallback(
- done));
- return;
- case 15:
- this.getLiveComponent(controller, (org.apache.slider.api.proto.Messages.GetLiveComponentRequestProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.ComponentInformationProto>specializeCallback(
- done));
- return;
- case 16:
- this.getLiveNodes(controller, (org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto>specializeCallback(
- done));
- return;
- case 17:
- this.getLiveNode(controller, (org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.NodeInformationProto>specializeCallback(
- done));
- return;
- case 18:
- this.getModelDesired(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.WrappedJsonProto>specializeCallback(
- done));
- return;
- case 19:
- this.getModelDesiredAppconf(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.WrappedJsonProto>specializeCallback(
- done));
- return;
- case 20:
- this.getModelDesiredResources(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.WrappedJsonProto>specializeCallback(
- done));
- return;
- case 21:
- this.getModelResolved(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.WrappedJsonProto>specializeCallback(
- done));
- return;
- case 22:
- this.getModelResolvedAppconf(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.WrappedJsonProto>specializeCallback(
- done));
- return;
- case 23:
- this.getModelResolvedResources(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.WrappedJsonProto>specializeCallback(
- done));
- return;
- case 24:
- this.getLiveResources(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.WrappedJsonProto>specializeCallback(
- done));
- return;
- case 25:
- this.getClientCertificateStore(controller, (org.apache.slider.api.proto.Messages.GetCertificateStoreRequestProto)request,
- com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.GetCertificateStoreResponseProto>specializeCallback(
- done));
- return;
- default:
- throw new java.lang.AssertionError("Can't get here.");
- }
- }
-
- public final com.google.protobuf.Message
- getRequestPrototype(
- com.google.protobuf.Descriptors.MethodDescriptor method) {
- if (method.getService() != getDescriptor()) {
- throw new java.lang.IllegalArgumentException(
- "Service.getRequestPrototype() given method " +
- "descriptor for wrong service type.");
- }
- switch(method.getIndex()) {
- case 0:
- return org.apache.slider.api.proto.Messages.StopClusterRequestProto.getDefaultInstance();
- case 1:
- return org.apache.slider.api.proto.Messages.UpgradeContainersRequestProto.getDefaultInstance();
- case 2:
- return org.apache.slider.api.proto.Messages.FlexClusterRequestProto.getDefaultInstance();
- case 3:
- return org.apache.slider.api.proto.Messages.GetJSONClusterStatusRequestProto.getDefaultInstance();
- case 4:
- return org.apache.slider.api.proto.Messages.GetInstanceDefinitionRequestProto.getDefaultInstance();
- case 5:
- return org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleRequestProto.getDefaultInstance();
- case 6:
- return org.apache.slider.api.proto.Messages.GetNodeRequestProto.getDefaultInstance();
- case 7:
- return org.apache.slider.api.proto.Messages.GetClusterNodesRequestProto.getDefaultInstance();
- case 8:
- return org.apache.slider.api.proto.Messages.EchoRequestProto.getDefaultInstance();
- case 9:
- return org.apache.slider.api.proto.Messages.KillContainerRequestProto.getDefaultInstance();
- case 10:
- return org.apache.slider.api.proto.Messages.AMSuicideRequestProto.getDefaultInstance();
- case 11:
- return org.apache.slider.api.proto.Messages.GetApplicationLivenessRequestProto.getDefaultInstance();
- case 12:
- return org.apache.slider.api.proto.Messages.GetLiveContainersRequestProto.getDefaultInstance();
- case 13:
- return org.apache.slider.api.proto.Messages.GetLiveContainerRequestProto.getDefaultInstance();
- case 14:
- return org.apache.slider.api.proto.Messages.GetLiveComponentsRequestProto.getDefaultInstance();
- case 15:
- return org.apache.slider.api.proto.Messages.GetLiveComponentRequestProto.getDefaultInstance();
- case 16:
- return org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto.getDefaultInstance();
- case 17:
- return org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto.getDefaultInstance();
- case 18:
- return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
- case 19:
- return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
- case 20:
- return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
- case 21:
- return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
- case 22:
- return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
- case 23:
- return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
- case 24:
- return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
- case 25:
- return org.apache.slider.api.proto.Messages.GetCertificateStoreRequestProto.getDefaultInstance();
- default:
- throw new java.lang.AssertionError("Can't get here.");
- }
- }
-
- public final com.google.protobuf.Message
- getResponsePrototype(
- com.google.protobuf.Descriptors.MethodDescriptor method) {
- if (method.getService() != getDescriptor()) {
- throw new java.lang.IllegalArgumentException(
- "Service.getResponsePrototype() given method " +
- "descriptor for wrong service type.");
- }
- switch(method.getIndex()) {
- case 0:
- return org.apache.slider.api.proto.Messages.StopClusterResponseProto.getDefaultInstance();
- case 1:
- return org.apache.slider.api.proto.Messages.UpgradeContainersResponseProto.getDefaultInstance();
- case 2:
- return org.apache.slider.api.proto.Messages.FlexClusterResponseProto.getDefaultInstance();
- case 3:
- return org.apache.slider.api.proto.Messages.GetJSONClusterStatusResponseProto.getDefaultInstance();
- case 4:
- return org.apache.slider.api.proto.Messages.GetInstanceDefinitionResponseProto.getDefaultInstance();
- case 5:
- return org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleResponseProto.getDefaultInstance();
- case 6:
- return org.apache.slider.api.proto.Messages.GetNodeResponseProto.getDefaultInstance();
- case 7:
- return org.apache.slider.api.proto.Messages.GetClusterNodesResponseProto.getDefaultInstance();
- case 8:
- return org.apache.slider.api.proto.Messages.EchoResponseProto.getDefaultInstance();
- case 9:
- return org.apache.slider.api.proto.Messages.KillContainerResponseProto.getDefaultInstance();
- case 10:
- return org.apache.slider.api.proto.Messages.AMSuicideResponseProto.getDefaultInstance();
- case 11:
- return org.apache.slider.api.proto.Messages.ApplicationLivenessInformationProto.getDefaultInstance();
- case 12:
- return org.apache.slider.api.proto.Messages.GetLiveContainersResponseProto.getDefaultInstance();
- case 13:
- return org.apache.slider.api.proto.Messages.ContainerInformationProto.getDefaultInstance();
- case 14:
- return org.apache.slider.api.proto.Messages.GetLiveComponentsResponseProto.getDefaultInstance();
- case 15:
- return org.apache.slider.api.proto.Messages.ComponentInformationProto.getDefaultInstance();
- case 16:
- return org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto.getDefaultInstance();
- case 17:
- return org.apache.slider.api.proto.Messages.NodeInformationProto.getDefaultInstance();
- case 18:
- return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
- case 19:
- return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
- case 20:
- return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
- case 21:
- return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
- case 22:
- return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
- case 23:
- return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
- case 24:
- return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
- case 25:
- return org.apache.slider.api.proto.Messages.GetCertificateStoreResponseProto.getDefaultInstance();
- default:
- throw new java.lang.AssertionError("Can't get here.");
- }
- }
-
- public static Stub newStub(
- com.google.protobuf.RpcChannel channel) {
- return new Stub(channel);
- }
-
- public static final class Stub extends org.apache.slider.api.proto.SliderClusterAPI.SliderClusterProtocolPB implements Interface {
- private Stub(com.google.protobuf.RpcChannel channel) {
- this.channel = channel;
- }
-
- private final com.google.protobuf.RpcChannel channel;
-
- public com.google.protobuf.RpcChannel getChannel() {
- return channel;
- }
-
- public void stopCluster(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.StopClusterRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.StopClusterResponseProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(0),
- controller,
- request,
- org.apache.slider.api.proto.Messages.StopClusterResponseProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.StopClusterResponseProto.class,
- org.apache.slider.api.proto.Messages.StopClusterResponseProto.getDefaultInstance()));
- }
-
- public void upgradeContainers(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.UpgradeContainersRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.UpgradeContainersResponseProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(1),
- controller,
- request,
- org.apache.slider.api.proto.Messages.UpgradeContainersResponseProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.UpgradeContainersResponseProto.class,
- org.apache.slider.api.proto.Messages.UpgradeContainersResponseProto.getDefaultInstance()));
- }
-
- public void flexCluster(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.FlexClusterRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.FlexClusterResponseProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(2),
- controller,
- request,
- org.apache.slider.api.proto.Messages.FlexClusterResponseProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.FlexClusterResponseProto.class,
- org.apache.slider.api.proto.Messages.FlexClusterResponseProto.getDefaultInstance()));
- }
-
- public void getJSONClusterStatus(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetJSONClusterStatusRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetJSONClusterStatusResponseProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(3),
- controller,
- request,
- org.apache.slider.api.proto.Messages.GetJSONClusterStatusResponseProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.GetJSONClusterStatusResponseProto.class,
- org.apache.slider.api.proto.Messages.GetJSONClusterStatusResponseProto.getDefaultInstance()));
- }
-
- public void getInstanceDefinition(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetInstanceDefinitionRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetInstanceDefinitionResponseProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(4),
- controller,
- request,
- org.apache.slider.api.proto.Messages.GetInstanceDefinitionResponseProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.GetInstanceDefinitionResponseProto.class,
- org.apache.slider.api.proto.Messages.GetInstanceDefinitionResponseProto.getDefaultInstance()));
- }
-
- public void listNodeUUIDsByRole(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleResponseProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(5),
- controller,
- request,
- org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleResponseProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleResponseProto.class,
- org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleResponseProto.getDefaultInstance()));
- }
-
- public void getNode(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetNodeRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetNodeResponseProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(6),
- controller,
- request,
- org.apache.slider.api.proto.Messages.GetNodeResponseProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.GetNodeResponseProto.class,
- org.apache.slider.api.proto.Messages.GetNodeResponseProto.getDefaultInstance()));
- }
-
- public void getClusterNodes(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetClusterNodesRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetClusterNodesResponseProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(7),
- controller,
- request,
- org.apache.slider.api.proto.Messages.GetClusterNodesResponseProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.GetClusterNodesResponseProto.class,
- org.apache.slider.api.proto.Messages.GetClusterNodesResponseProto.getDefaultInstance()));
- }
-
- public void echo(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EchoRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.EchoResponseProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(8),
- controller,
- request,
- org.apache.slider.api.proto.Messages.EchoResponseProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.EchoResponseProto.class,
- org.apache.slider.api.proto.Messages.EchoResponseProto.getDefaultInstance()));
- }
-
- public void killContainer(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.KillContainerRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.KillContainerResponseProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(9),
- controller,
- request,
- org.apache.slider.api.proto.Messages.KillContainerResponseProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.KillContainerResponseProto.class,
- org.apache.slider.api.proto.Messages.KillContainerResponseProto.getDefaultInstance()));
- }
-
- public void amSuicide(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.AMSuicideRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.AMSuicideResponseProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(10),
- controller,
- request,
- org.apache.slider.api.proto.Messages.AMSuicideResponseProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.AMSuicideResponseProto.class,
- org.apache.slider.api.proto.Messages.AMSuicideResponseProto.getDefaultInstance()));
- }
-
- public void getLivenessInformation(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetApplicationLivenessRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.ApplicationLivenessInformationProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(11),
- controller,
- request,
- org.apache.slider.api.proto.Messages.ApplicationLivenessInformationProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.ApplicationLivenessInformationProto.class,
- org.apache.slider.api.proto.Messages.ApplicationLivenessInformationProto.getDefaultInstance()));
- }
-
- public void getLiveContainers(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveContainersRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetLiveContainersResponseProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(12),
- controller,
- request,
- org.apache.slider.api.proto.Messages.GetLiveContainersResponseProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.GetLiveContainersResponseProto.class,
- org.apache.slider.api.proto.Messages.GetLiveContainersResponseProto.getDefaultInstance()));
- }
-
- public void getLiveContainer(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveContainerRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.ContainerInformationProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(13),
- controller,
- request,
- org.apache.slider.api.proto.Messages.ContainerInformationProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.ContainerInformationProto.class,
- org.apache.slider.api.proto.Messages.ContainerInformationProto.getDefaultInstance()));
- }
-
- public void getLiveComponents(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveComponentsRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetLiveComponentsResponseProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(14),
- controller,
- request,
- org.apache.slider.api.proto.Messages.GetLiveComponentsResponseProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.GetLiveComponentsResponseProto.class,
- org.apache.slider.api.proto.Messages.GetLiveComponentsResponseProto.getDefaultInstance()));
- }
-
- public void getLiveComponent(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveComponentRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.ComponentInformationProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(15),
- controller,
- request,
- org.apache.slider.api.proto.Messages.ComponentInformationProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.ComponentInformationProto.class,
- org.apache.slider.api.proto.Messages.ComponentInformationProto.getDefaultInstance()));
- }
-
- public void getLiveNodes(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(16),
- controller,
- request,
- org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto.class,
- org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto.getDefaultInstance()));
- }
-
- public void getLiveNode(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.NodeInformationProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(17),
- controller,
- request,
- org.apache.slider.api.proto.Messages.NodeInformationProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.NodeInformationProto.class,
- org.apache.slider.api.proto.Messages.NodeInformationProto.getDefaultInstance()));
- }
-
- public void getModelDesired(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(18),
- controller,
- request,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.class,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance()));
- }
-
- public void getModelDesiredAppconf(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(19),
- controller,
- request,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.class,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance()));
- }
-
- public void getModelDesiredResources(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(20),
- controller,
- request,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.class,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance()));
- }
-
- public void getModelResolved(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(21),
- controller,
- request,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.class,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance()));
- }
-
- public void getModelResolvedAppconf(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(22),
- controller,
- request,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.class,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance()));
- }
-
- public void getModelResolvedResources(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(23),
- controller,
- request,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.class,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance()));
- }
-
- public void getLiveResources(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(24),
- controller,
- request,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.class,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance()));
- }
-
- public void getClientCertificateStore(
- com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.GetCertificateStoreRequestProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetCertificateStoreResponseProto> done) {
- channel.callMethod(
- getDescriptor().getMethods().get(25),
- controller,
- request,
- org.apache.slider.api.proto.Messages.GetCertificateStoreResponseProto.getDefaultInstance(),
- com.google.protobuf.RpcUtil.generalizeCallback(
- done,
- org.apache.slider.api.proto.Messages.GetCertificateStoreResponseProto.class,
- org.apache.slider.api.proto.Messages.GetCertificateStoreResponseProto.getDefaultInstance()));
- }
- }
-
- public static BlockingInterface newBlockingStub(
- com.google.protobuf.BlockingRpcChannel channel) {
- return new BlockingStub(channel);
- }
-
- public interface BlockingInterface {
- public org.apache.slider.api.proto.Messages.StopClusterResponseProto stopCluster(
- com.google.protobuf.RpcController controller,
-
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/51] [abbrv] hadoop git commit: YARN-6014. Followup fix for slider
core module findbugs. Contributed by Jian He
Posted by ji...@apache.org.
YARN-6014. Followup fix for slider core module findbugs. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47a16db3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47a16db3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47a16db3
Branch: refs/heads/yarn-native-services
Commit: 47a16db3d2086e367f840655a86e246db92f66ee
Parents: fa46df9
Author: Billie Rinaldi <bi...@apache.org>
Authored: Tue Dec 20 10:59:58 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../dev-support/findbugs-exclude.xml | 17 +++++++++++++++++
.../web/rest/application/actions/StopResponse.java | 2 +-
2 files changed, 18 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47a16db3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
index d253762..e5cde4e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
@@ -71,6 +71,23 @@
<Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
</Match>
<Match>
+ <Class name="org.apache.slider.server.appmaster.web.rest.registry.PathEntryResource" />
+ <Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.server.appmaster.web.rest.application.actions.StopResponse" />
+ <Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.server.appmaster.state.ContainerAllocationResults" />
+ <Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.core.persist.JsonSerDeser" />
+ <Method name="save" />
+ <Bug pattern="OBL_UNSATISFIED_OBLIGATION" />
+ </Match>
+ <Match>
<Class name="org.apache.slider.server.services.workflow.ForkedProcessService" />
<Bug pattern="JLM_JSR166_UTILCONCURRENT_MONITORENTER" />
</Match>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47a16db3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/StopResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/StopResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/StopResponse.java
index 9af6a22..d591f57 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/StopResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/StopResponse.java
@@ -24,6 +24,6 @@ import org.codehaus.jackson.map.annotate.JsonSerialize;
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class StopResponse {
- String verb;
+ public String verb;
public String text;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/51] [abbrv] hadoop git commit: YARN-5909. Remove agent related
code in slider AM. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
index a0fe310..bd4d2bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
@@ -23,7 +23,6 @@ import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.state.RoleStatus;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
import org.apache.slider.server.services.security.CertificateManager;
import org.slf4j.Logger;
@@ -97,11 +96,6 @@ public class WebAppApiImpl implements WebAppApi {
}
@Override
- public AgentRestOperations getAgentRestOperations() {
- return provider.getAgentRestOperations();
- }
-
- @Override
public RegistryOperations getRegistryOperations() {
return registryOperations;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
index 03bf703..aed87d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
@@ -19,7 +19,6 @@ package org.apache.slider.server.appmaster.web.rest;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentResource;
import org.apache.slider.server.appmaster.web.rest.application.ApplicationResource;
import org.apache.slider.server.appmaster.web.rest.management.ManagementResource;
import org.apache.slider.server.appmaster.web.rest.publisher.PublisherResource;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentCommandType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentCommandType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentCommandType.java
deleted file mode 100644
index 17cd8f2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentCommandType.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-public enum AgentCommandType {
- EXECUTION_COMMAND,
- STATUS_COMMAND,
- REGISTRATION_COMMAND
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentEnv.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentEnv.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentEnv.java
deleted file mode 100644
index 781ae00..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentEnv.java
+++ /dev/null
@@ -1,376 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import com.google.gson.annotations.SerializedName;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class AgentEnv {
-
- /**
- * Various directories, configurable in <code>ambari-agent.ini</code>
- */
- private Directory[] stackFoldersAndFiles = new Directory[0];
-
- /**
- * Directories that match name <code>/etc/alternatives/*conf</code>
- */
- private Alternative[] alternatives = new Alternative[0];
-
- /**
- * List of existing users
- */
- private ExistingUser[] existingUsers = new ExistingUser[0];
-
- /**
- * List of repos
- */
- private String[] existingRepos = new String[0];
-
- /**
- * List of packages
- */
- private PackageDetail[] installedPackages = new PackageDetail[0];
-
- /**
- * The host health report
- */
- private HostHealth hostHealth = new HostHealth();
-
- private Integer umask;
-
- private Boolean iptablesIsRunning;
-
- public Integer getUmask() {
- return umask;
- }
-
- public void setUmask(Integer umask) {
- this.umask = umask;
- }
-
- public Directory[] getStackFoldersAndFiles() {
- return stackFoldersAndFiles;
- }
-
- public void setStackFoldersAndFiles(Directory[] dirs) {
- stackFoldersAndFiles = dirs;
- }
-
- public void setExistingUsers(ExistingUser[] users) {
- existingUsers = users;
- }
-
- public ExistingUser[] getExistingUsers() {
- return existingUsers;
- }
-
- public void setAlternatives(Alternative[] dirs) {
- alternatives = dirs;
- }
-
- public Alternative[] getAlternatives() {
- return alternatives;
- }
-
- public void setExistingRepos(String[] repos) {
- existingRepos = repos;
- }
-
- public String[] getExistingRepos() {
- return existingRepos;
- }
-
- public void setInstalledPackages(PackageDetail[] packages) {
- installedPackages = packages;
- }
-
- public PackageDetail[] getInstalledPackages() {
- return installedPackages;
- }
-
- public void setHostHealth(HostHealth healthReport) {
- hostHealth = healthReport;
- }
-
- public HostHealth getHostHealth() {
- return hostHealth;
- }
-
- public Boolean getIptablesIsRunning() {
- return iptablesIsRunning;
- }
-
- public void setIptablesIsRunning(Boolean iptablesIsRunning) {
- this.iptablesIsRunning = iptablesIsRunning;
- }
-
- public static class HostHealth {
- /**
- * Java processes running on the system. Default empty array.
- */
- @SerializedName("activeJavaProcs")
- private JavaProc[] activeJavaProcs = new JavaProc[0];
-
- /**
- * The current time when agent send the host check report
- */
- @SerializedName("agentTimeStampAtReporting")
- private long agentTimeStampAtReporting = 0;
-
- /**
- * The current time when host check report was received
- */
- @SerializedName("serverTimeStampAtReporting")
- private long serverTimeStampAtReporting = 0;
-
- /**
- * Live services running on the agent
- */
- @SerializedName("liveServices")
- private LiveService[] liveServices = new LiveService[0];
-
- public void setAgentTimeStampAtReporting(long currentTime) {
- agentTimeStampAtReporting = currentTime;
- }
-
- public long getAgentTimeStampAtReporting() {
- return agentTimeStampAtReporting;
- }
-
- public void setServerTimeStampAtReporting(long currentTime) {
- serverTimeStampAtReporting = currentTime;
- }
-
- public long getServerTimeStampAtReporting() {
- return serverTimeStampAtReporting;
- }
-
- public void setActiveJavaProcs(JavaProc[] procs) {
- activeJavaProcs = procs;
- }
-
- public JavaProc[] getActiveJavaProcs() {
- return activeJavaProcs;
- }
-
- public void setLiveServices(LiveService[] services) {
- liveServices = services;
- }
-
- public LiveService[] getLiveServices() {
- return liveServices;
- }
- }
-
- public static class PackageDetail {
- @SerializedName("name")
- private String pkgName;
- @SerializedName("version")
- private String pkgVersion;
- @SerializedName("repoName")
- private String pkgRepoName;
-
- public void setName(String name) {
- pkgName = name;
- }
-
- public String getName() {
- return pkgName;
- }
-
- public void setVersion(String version) {
- pkgVersion = version;
- }
-
- public String getVersion() {
- return pkgVersion;
- }
-
- public void setRepoName(String repoName) {
- pkgRepoName = repoName;
- }
-
- public String getRepoName() {
- return pkgRepoName;
- }
- }
-
- /**
- * Represents information about a directory of interest.
- */
- public static class Directory {
- @SerializedName("name")
- private String dirName;
- @SerializedName("type")
- private String dirType;
-
- public void setName(String name) {
- dirName = name;
- }
-
- public String getName() {
- return dirName;
- }
-
- public void setType(String type) {
- dirType = type;
- }
-
- public String getType() {
- return dirType;
- }
- }
-
- /**
- * Represents information about running java processes.
- */
- public static class JavaProc {
- @SerializedName("user")
- private String user;
- @SerializedName("pid")
- private int pid = 0;
- @SerializedName("hadoop")
- private boolean is_hadoop = false;
- @SerializedName("command")
- private String command;
-
- public void setUser(String user) {
- this.user = user;
- }
-
- public String getUser() {
- return user;
- }
-
- public void setPid(int pid) {
- this.pid = pid;
- }
-
- public int getPid() {
- return pid;
- }
-
- public void setHadoop(boolean hadoop) {
- is_hadoop = hadoop;
- }
-
- public boolean isHadoop() {
- return is_hadoop;
- }
-
- public void setCommand(String cmd) {
- command = cmd;
- }
-
- public String getCommand() {
- return command;
- }
- }
-
- public static class Alternative {
- @SerializedName("name")
- private String altName;
- @SerializedName("target")
- private String altTarget;
-
- public void setName(String name) {
- altName = name;
- }
-
- public String getName() {
- return altName;
- }
-
- public void setTarget(String target) {
- altTarget = target;
- }
-
- public String getTarget() {
- return altTarget;
- }
- }
-
- public static class LiveService {
- @SerializedName("name")
- private String svcName;
- @SerializedName("status")
- private String svcStatus;
- @SerializedName("desc")
- private String svcDesc;
-
- public void setName(String name) {
- svcName = name;
- }
-
- public String getName() {
- return svcName;
- }
-
- public void setStatus(String status) {
- svcStatus = status;
- }
-
- public String getStatus() {
- return svcStatus;
- }
-
- public void setDesc(String desc) {
- svcDesc = desc;
- }
-
- public String getDesc() {
- return svcDesc;
- }
- }
-
- public static class ExistingUser {
- @SerializedName("name")
- private String name;
- @SerializedName("homeDir")
- private String homeDir;
- @SerializedName("status")
- private String status;
-
- public void setUserName(String userName) {
- name = userName;
- }
-
- public String getUserName() {
- return name;
- }
-
- public void setUserHomeDir(String userHomeDir) {
- homeDir = userHomeDir;
- }
-
- public String getUserHomeDir() {
- return homeDir;
- }
-
- public void setUserStatus(String userStatus) {
- status = userStatus;
- }
-
- public String getUserStatus() {
- return status;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java
deleted file mode 100644
index 20ef068..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.services.security.SignCertResponse;
-import org.apache.slider.server.services.security.SignMessage;
-import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-/**
- *
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class AgentResource extends AbstractSliderResource {
-
- private String agent_name;
-
- public AgentResource(WebAppApi slider) {
- super(slider);
- }
-
- private void init(HttpServletResponse res) {
- res.setContentType(null);
- }
-
- @GET
- @Path("/agent/register")
- public Response endpointAgentRegister() {
- Response response = Response.status(200).entity("/agent/register").build();
- return response;
- }
-
- @GET
- @Path("/agent")
- public Response endpointAgent() {
- Response response = Response.status(200).entity("/agent").build();
- return response;
- }
- @GET
- @Path("/")
- public Response endpointRoot() {
- Response response = Response.status(200).entity("/").build();
- return response;
- }
-
- @POST
- @Path("/{agent_name: [a-zA-Z][a-zA-Z0-9_-]*}/register")
- @Consumes({MediaType.APPLICATION_JSON})
- @Produces({MediaType.APPLICATION_JSON})
- public RegistrationResponse register(Register registration,
- @Context HttpServletResponse res,
- @PathParam("agent_name") String agent_name) {
- init(res);
- this.agent_name = agent_name;
- AgentRestOperations ops = slider.getAgentRestOperations();
- return ops.handleRegistration(registration);
-
- }
-
- @POST
- @Path("/{agent_name: [a-zA-Z][a-zA-Z0-9_-]*}/heartbeat")
- @Consumes(MediaType.APPLICATION_JSON)
- @Produces({MediaType.APPLICATION_JSON})
- public HeartBeatResponse heartbeat(HeartBeat message,
- @Context HttpServletResponse res,
- @PathParam("agent_name") String agent_name) {
- init(res);
- AgentRestOperations ops = slider.getAgentRestOperations();
- return ops.handleHeartBeat(message);
- }
-
- @GET
- @Path("/cert/ca")
- @Produces({MediaType.TEXT_PLAIN})
- public String downloadSrvrCrt() {
- return slider.getCertificateManager().getServerCert();
- }
-
- @Path("/certs/{hostName}")
- @POST
- @Consumes(MediaType.APPLICATION_JSON)
- @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
- public SignCertResponse signAgentCrt(@PathParam("hostName") String hostname,
- SignMessage message, @Context HttpServletRequest req) {
- return slider.getCertificateManager().signAgentCrt(hostname,
- message.getCsr(),
- message.getPassphrase());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentRestOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentRestOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentRestOperations.java
deleted file mode 100644
index 2891be8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentRestOperations.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-/**
- *
- */
-public interface AgentRestOperations {
-
- RegistrationResponse handleRegistration(Register registration);
-
- HeartBeatResponse handleHeartBeat(HeartBeat heartBeat);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
deleted file mode 100644
index 3a3b0c0..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
+++ /dev/null
@@ -1,258 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.core.ResourceConfig;
-import com.sun.jersey.spi.container.WebApplication;
-import com.sun.jersey.spi.container.servlet.ServletContainer;
-import com.sun.jersey.spi.container.servlet.WebConfig;
-import com.sun.jersey.spi.inject.SingletonTypeInjectableProvider;
-import org.apache.slider.core.conf.MapOperations;
-import org.apache.slider.providers.agent.AgentKeys;
-import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.services.security.SecurityUtils;
-import org.mortbay.jetty.Connector;
-import org.mortbay.jetty.Server;
-import org.mortbay.jetty.security.SslSelectChannelConnector;
-import org.mortbay.jetty.servlet.Context;
-import org.mortbay.jetty.servlet.ServletHolder;
-import org.mortbay.thread.QueuedThreadPool;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.ws.rs.ext.Provider;
-import java.io.Closeable;
-import java.io.File;
-import java.io.IOException;
-import java.net.BindException;
-import java.util.Set;
-
-/**
- *
- */
-public class AgentWebApp implements Closeable {
- protected static final Logger LOG = LoggerFactory.getLogger(AgentWebApp.class);
- private int port;
- private int securedPort;
- private static Server agentServer;
- public static final String BASE_PATH = "slideragent";
-
- public static class Builder {
- final String name;
- final String wsName;
- final WebAppApi application;
- int port;
- int securedPort;
- MapOperations configsMap;
-
- public Builder(String name, String wsName, WebAppApi application) {
- this.name = name;
- this.wsName = wsName;
- this.application = application;
- }
-
- public Builder withComponentConfig(MapOperations appMasterConfig) {
- this.configsMap = appMasterConfig;
- return this;
- }
-
- public Builder withPort (int port) {
- this.port = port;
- return this;
- }
-
- public Builder withSecuredPort (int securedPort) {
- this.securedPort = securedPort;
- return this;
- }
-
- public AgentWebApp start() throws IOException {
- if (configsMap == null) {
- throw new IllegalStateException("No SSL Configuration Available");
- }
-
- agentServer = new Server();
- agentServer.setThreadPool(
- new QueuedThreadPool(
- configsMap.getOptionInt("agent.threadpool.size.max", 25)));
- agentServer.setStopAtShutdown(true);
- agentServer.setGracefulShutdown(1000);
-
- SslSelectChannelConnector ssl1WayConnector = createSSLConnector(false, port);
- SslSelectChannelConnector ssl2WayConnector =
- createSSLConnector(Boolean.valueOf(
- configsMap.getOption(AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED,
- "false")), securedPort);
- agentServer.setConnectors(new Connector[]{ssl1WayConnector,
- ssl2WayConnector});
-
- ServletHolder agent = new ServletHolder(new AgentServletContainer());
- Context agentRoot = new Context(agentServer, "/", Context.SESSIONS);
-
- agent.setInitParameter("com.sun.jersey.config.property.resourceConfigClass",
- "com.sun.jersey.api.core.PackagesResourceConfig");
- agent.setInitParameter("com.sun.jersey.config.property.packages",
- "org.apache.slider.server.appmaster.web.rest.agent");
- agent.setInitParameter("com.sun.jersey.api.json.POJOMappingFeature",
- "true");
-// agent.setInitParameter("com.sun.jersey.spi.container.ContainerRequestFilters", "com.sun.jersey.api.container.filter.LoggingFilter");
-// agent.setInitParameter("com.sun.jersey.spi.container.ContainerResponseFilters", "com.sun.jersey.api.container.filter.LoggingFilter");
-// agent.setInitParameter("com.sun.jersey.config.feature.Trace", "true");
- agentRoot.addServlet(agent, "/*");
-
- try {
- openListeners();
- agentServer.start();
- } catch (IOException e) {
- LOG.error("Unable to start agent server", e);
- throw e;
- } catch (Exception e) {
- LOG.error("Unable to start agent server", e);
- throw new IOException("Unable to start agent server: " + e, e);
- }
-
- AgentWebApp webApp = new AgentWebApp();
- webApp.setPort(getConnectorPort(agentServer, 0));
- webApp.setSecuredPort(getConnectorPort(agentServer, 1));
- return webApp;
-
- }
-
- private void openListeners() throws Exception {
- // from HttpServer2.openListeners()
- for (Connector listener : agentServer.getConnectors()) {
- if (listener.getLocalPort() != -1) {
- // This listener is either started externally or has been bound
- continue;
- }
- int port = listener.getPort();
- while (true) {
- // jetty has a bug where you can't reopen a listener that previously
- // failed to open w/o issuing a close first, even if the port is changed
- try {
- listener.close();
- listener.open();
- LOG.info("Jetty bound to port " + listener.getLocalPort());
- break;
- } catch (BindException ex) {
- if (port == 0) {
- BindException be = new BindException("Port in use: "
- + listener.getHost() + ":" + listener.getPort());
- be.initCause(ex);
- throw be;
- }
- }
- // try the next port number
- listener.setPort(++port);
- Thread.sleep(100);
- }
- }
- }
-
- private SslSelectChannelConnector createSSLConnector(boolean needClientAuth, int port) {
- SslSelectChannelConnector sslConnector = new
- SslSelectChannelConnector();
-
- String keystore = SecurityUtils.getSecurityDir() +
- File.separator + "keystore.p12";
- String srvrCrtPass = SecurityUtils.getKeystorePass();
- sslConnector.setKeystore(keystore);
- sslConnector.setTruststore(keystore);
- sslConnector.setPassword(srvrCrtPass);
- sslConnector.setKeyPassword(srvrCrtPass);
- sslConnector.setTrustPassword(srvrCrtPass);
- sslConnector.setKeystoreType("PKCS12");
- sslConnector.setTruststoreType("PKCS12");
- sslConnector.setNeedClientAuth(needClientAuth);
-
- sslConnector.setPort(port);
- sslConnector.setAcceptors(2);
- return sslConnector;
- }
-
- @Provider
- public class WebAppApiProvider extends
- SingletonTypeInjectableProvider<javax.ws.rs.core.Context, WebAppApi> {
-
- public WebAppApiProvider () {
- super(WebAppApi.class, application);
- }
- }
-
- public class AgentServletContainer extends ServletContainer {
- public AgentServletContainer() {
- super();
- }
-
- @Override
- protected void configure(WebConfig wc,
- ResourceConfig rc,
- WebApplication wa) {
- super.configure(wc, rc, wa);
- Set<Object> singletons = rc.getSingletons();
- singletons.add(new WebAppApiProvider());
- }
- }
-
- private int getConnectorPort(Server webServer, int index) {
- Preconditions.checkArgument(index >= 0);
- if (index > webServer.getConnectors().length)
- throw new IllegalStateException("Illegal connect index requested");
-
- Connector c = webServer.getConnectors()[index];
- if (c.getLocalPort() == -1) {
- // The connector is not bounded
- throw new IllegalStateException("The connector is not bound to a port");
- }
-
- return c.getLocalPort();
- }
- }
-
- public static Builder $for(String name, WebAppApi app, String wsPrefix) {
- return new Builder(name, wsPrefix, app);
- }
-
- public int getPort() {
- return port;
- }
-
- public void setPort(int port) {
- this.port = port;
- }
-
- public void setSecuredPort(int securedPort) {
- this.securedPort = securedPort;
- }
-
- public int getSecuredPort() {
- return securedPort;
- }
-
- public void close() throws IOException{
- //need to stop server and reset injector
- try {
- agentServer.stop();
- } catch (IOException e) {
- throw e;
- } catch (Exception e) {
- throw new IOException(e.toString(), e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebServices.java
deleted file mode 100644
index 684ce6f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebServices.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.appmaster.web.rest.RestPaths;
-
-import javax.ws.rs.Path;
-import javax.ws.rs.core.Context;
-
-/** The available agent REST services exposed by a slider AM. */
-@Path(RestPaths.SLIDER_AGENT_CONTEXT_ROOT)
-public class AgentWebServices {
- /** AM/WebApp info object */
- @Context
- private WebAppApi slider;
-
- public AgentWebServices() {
- }
-
- @Path(RestPaths.SLIDER_SUBPATH_AGENTS)
- public AgentResource getAgentResource () {
- return new AgentResource(slider);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/CommandReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/CommandReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/CommandReport.java
deleted file mode 100644
index a37e490..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/CommandReport.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.Map;
-
-/**
- *
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class CommandReport {
-
- int exitCode;
- private String role;
- private String actionId;
- private String stdout;
- private String stderr;
- private String structuredOut;
- private String status;
- private String clusterName;
- private String serviceName;
- private long taskId;
- private String roleCommand;
- private Map<String, String> folders;
- private Map<String, String> allocatedPorts;
- private Map<String, Map<String, String>> configurationTags;
-
- @JsonProperty("taskId")
- public long getTaskId() {
- return taskId;
- }
-
- @JsonProperty("taskId")
- public void setTaskId(long taskId) {
- this.taskId = taskId;
- }
-
- @JsonProperty("clusterName")
- public String getClusterName() {
- return this.clusterName;
- }
-
- @JsonProperty("clusterName")
- public void setClusterName(String clusterName) {
- this.clusterName = clusterName;
- }
-
- @JsonProperty("actionId")
- public String getActionId() {
- return this.actionId;
- }
-
- @JsonProperty("actionId")
- public void setActionId(String actionId) {
- this.actionId = actionId;
- }
-
- @JsonProperty("stderr")
- public String getStdErr() {
- return this.stderr;
- }
-
- @JsonProperty("stderr")
- public void setStdErr(String stderr) {
- this.stderr = stderr;
- }
-
- @JsonProperty("exitcode")
- public int getExitCode() {
- return this.exitCode;
- }
-
- @JsonProperty("exitcode")
- public void setExitCode(int exitCode) {
- this.exitCode = exitCode;
- }
-
- @JsonProperty("stdout")
- public String getStdOut() {
- return this.stdout;
- }
-
- @JsonProperty("stdout")
- public void setStdOut(String stdout) {
- this.stdout = stdout;
- }
-
- @JsonProperty("structuredOut")
- public String getStructuredOut() {
- return this.structuredOut;
- }
-
- @JsonProperty("structuredOut")
- public void setStructuredOut(String structuredOut) {
- this.structuredOut = structuredOut;
- }
-
- @JsonProperty("roleCommand")
- public String getRoleCommand() {
- return this.roleCommand;
- }
-
- @JsonProperty("roleCommand")
- public void setRoleCommand(String roleCommand) {
- this.roleCommand = roleCommand;
- }
-
- @JsonProperty("role")
- public String getRole() {
- return role;
- }
-
- @JsonProperty("role")
- public void setRole(String role) {
- this.role = role;
- }
-
- @JsonProperty("status")
- public String getStatus() {
- return status;
- }
-
- @JsonProperty("status")
- public void setStatus(String status) {
- this.status = status;
- }
-
- @JsonProperty("serviceName")
- public String getServiceName() {
- return serviceName;
- }
-
- @JsonProperty("serviceName")
- public void setServiceName(String serviceName) {
- this.serviceName = serviceName;
- }
-
- /** @return the config tags that match this command, or <code>null</code> if none are present */
- @JsonProperty("configurationTags")
- public Map<String, Map<String, String>> getConfigurationTags() {
- return configurationTags;
- }
-
- /** @param tags the config tags that match this command */
- @JsonProperty("configurationTags")
- public void setConfigurationTags(Map<String, Map<String, String>> tags) {
- configurationTags = tags;
- }
-
- /** @return the allocated ports, or <code>null</code> if none are present */
- @JsonProperty("allocatedPorts")
- public Map<String, String> getAllocatedPorts() {
- return allocatedPorts;
- }
-
- /** @param ports allocated ports */
- @JsonProperty("allocatedPorts")
- public void setAllocatedPorts(Map<String, String> ports) {
- this.allocatedPorts = ports;
- }
-
- /** @return the folders, or <code>null</code> if none are present */
- @JsonProperty("folders")
- public Map<String, String> getFolders() {
- return folders;
- }
-
- /** @param folders allocated ports */
- @JsonProperty("folders")
- public void setFolders(Map<String, String> folders) {
- this.folders = folders;
- }
-
- @Override
- public String toString() {
- return "CommandReport{" +
- "role='" + role + '\'' +
- ", actionId='" + actionId + '\'' +
- ", status='" + status + '\'' +
- ", exitCode=" + exitCode +
- ", clusterName='" + clusterName + '\'' +
- ", serviceName='" + serviceName + '\'' +
- ", taskId=" + taskId +
- ", roleCommand=" + roleCommand +
- ", configurationTags=" + configurationTags +
- '}';
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java
deleted file mode 100644
index acdc234..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.Map;
-
-/**
- *
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class ComponentStatus {
- String componentName;
- String msg;
- String status;
- String serviceName;
- String clusterName;
- String roleCommand;
- String ip;
- String hostname;
- @JsonProperty("configurations")
- private Map<String, Map<String, String>> configurations;
-
- public String getRoleCommand() {
- return roleCommand;
- }
-
- public void setRoleCommand(String roleCommand) {
- this.roleCommand = roleCommand;
- }
-
- public String getComponentName() {
- return this.componentName;
- }
-
- public void setComponentName(String componentName) {
- this.componentName = componentName;
- }
-
- public String getMessage() {
- return this.msg;
- }
-
- public void setMessage(String msg) {
- this.msg = msg;
- }
-
- public String getStatus() {
- return this.status;
- }
-
- public void setStatus(String status) {
- this.status = status;
- }
-
- public String getServiceName() {
- return serviceName;
- }
-
- public void setServiceName(String serviceName) {
- this.serviceName = serviceName;
- }
-
- public String getClusterName() {
- return clusterName;
- }
-
- public void setClusterName(String clusterName) {
- this.clusterName = clusterName;
- }
-
- /** @return the config tags that match this command, or <code>null</code> if none are present */
- public Map<String, Map<String, String>> getConfigs() {
- return configurations;
- }
-
- /** @param configs the config tags that match this status */
- public void setConfigs(Map<String, Map<String, String>> configs) {
- this.configurations = configs;
- }
-
- @Override
- public String toString() {
- return "ComponentStatus{" +
- "componentName='" + componentName + '\'' +
- ", msg='" + msg + '\'' +
- ", status='" + status + '\'' +
- ", serviceName='" + serviceName + '\'' +
- ", clusterName='" + clusterName + '\'' +
- ", roleCommand='" + roleCommand + '\'' +
- ", ip='" + ip + '\'' +
- ", hostname='" + hostname + '\'' +
- '}';
- }
-
- public String getIp() {
- return ip;
- }
-
- public void setIp(String ip) {
- this.ip = ip;
- }
-
- public String getHostname() {
- return hostname;
- }
-
- public void setHostname(String hostname) {
- this.hostname = hostname;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/DiskInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/DiskInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/DiskInfo.java
deleted file mode 100644
index 27c4d54..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/DiskInfo.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-/**
- *
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class DiskInfo {
- String available;
- String mountpoint;
- String device;
- String used;
- String percent;
- String size;
- String type;
-
- /**
- * DiskInfo object that tracks information about a disk.
- * @param mountpoint
- * @param available
- * @param used
- * @param percent
- * @param size
- */
- public DiskInfo(String device, String mountpoint, String available,
- String used, String percent, String size, String type) {
- this.device = device;
- this.mountpoint = mountpoint;
- this.available = available;
- this.used = used;
- this.percent = percent;
- this.size = size;
- this.type = type;
- }
-
- /**
- * Needed for Serialization
- */
- public DiskInfo() {}
-
- @JsonProperty("available")
- public void setAvailable(String available) {
- this.available = available;
- }
-
- @JsonProperty("available")
- public String getAvailable() {
- return this.available;
- }
-
- @JsonProperty("mountpoint")
- public String getMountPoint() {
- return this.mountpoint;
- }
-
- @JsonProperty("mountpoint")
- public void setMountPoint(String mountpoint) {
- this.mountpoint = mountpoint;
- }
-
- @JsonProperty("type")
- public String getType() {
- return this.type;
- }
-
- @JsonProperty("type")
- public void setType(String type) {
- this.type = type;
- }
-
- @JsonProperty("used")
- public String getUsed() {
- return this.used;
- }
-
- @JsonProperty("used")
- public void setUsed(String used) {
- this.used = used;
- }
-
- @JsonProperty("percent")
- public String getPercent() {
- return this.percent;
- }
-
- @JsonProperty("percent")
- public void setPercent(String percent) {
- this.percent = percent;
- }
-
- @JsonProperty("size")
- public String getSize() {
- return this.size;
- }
-
- @JsonProperty("size")
- public void setSize(String size) {
- this.size = size;
- }
-
- @Override
- public String toString() {
- return "available=" + this.available + ",mountpoint=" + this.mountpoint
- + ",used=" + this.used + ",percent=" + this.percent + ",size=" +
- this.size + ",device=" + this.device +
- ",type=" + this.type;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
deleted file mode 100644
index d3864b8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
+++ /dev/null
@@ -1,310 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.apache.slider.providers.agent.application.metadata.Component;
-import org.apache.slider.providers.agent.application.metadata.DockerContainer;
-import org.apache.slider.providers.agent.application.metadata.DockerContainerInputFile;
-import org.apache.slider.providers.agent.application.metadata.DockerContainerMount;
-import org.apache.slider.providers.agent.application.metadata.DockerContainerPort;
-import org.apache.slider.providers.agent.application.metadata.Metainfo;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- *
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class ExecutionCommand {
- protected static final Logger log =
- LoggerFactory.getLogger(ExecutionCommand.class);
- private AgentCommandType commandType = AgentCommandType.EXECUTION_COMMAND;
- private String clusterName;
- private long taskId;
- private String commandId;
- //TODO Remove hostname from being set in the command
- private String hostname;
- private String role;
- private Map<String, String> hostLevelParams = new HashMap<String, String>();
- private Map<String, String> roleParams = null;
- private String roleCommand;
- private Map<String, Map<String, String>> configurations;
- private Map<String, Map<String, String>> componentConfigurations;
- private Map<String, String> commandParams;
- private String serviceName;
- private String componentName;
- private String componentType;
- private List<DockerContainer> containers = new ArrayList<>();
- private String pkg;
- private boolean yarnDockerMode = false;
-
- public ExecutionCommand(AgentCommandType commandType) {
- this.commandType = commandType;
- }
-
- @JsonProperty("commandType")
- public AgentCommandType getCommandType() {
- return commandType;
- }
-
- @JsonProperty("commandType")
- public void setCommandType(AgentCommandType commandType) {
- this.commandType = commandType;
- }
-
- @JsonProperty("commandId")
- public String getCommandId() {
- return this.commandId;
- }
-
- @JsonProperty("commandId")
- public void setCommandId(String commandId) {
- this.commandId = commandId;
- }
-
- @JsonProperty("taskId")
- public long getTaskId() {
- return taskId;
- }
-
- @JsonProperty("taskId")
- public void setTaskId(long taskId) {
- this.taskId = taskId;
- }
-
- @JsonProperty("role")
- public String getRole() {
- return role;
- }
-
- @JsonProperty("role")
- public void setRole(String role) {
- this.role = role;
- }
-
- @JsonProperty("roleParams")
- public Map<String, String> getRoleParams() {
- return roleParams;
- }
-
- @JsonProperty("roleParams")
- public void setRoleParams(Map<String, String> roleParams) {
- this.roleParams = roleParams;
- }
-
- @JsonProperty("roleCommand")
- public String getRoleCommand() {
- return roleCommand;
- }
-
- @JsonProperty("roleCommand")
- public void setRoleCommand(String cmd) {
- this.roleCommand = cmd;
- }
-
- @JsonProperty("clusterName")
- public String getClusterName() {
- return clusterName;
- }
-
- @JsonProperty("clusterName")
- public void setClusterName(String clusterName) {
- this.clusterName = clusterName;
- }
-
- @JsonProperty("componentType")
- public String getComponentType() {
- return componentType;
- }
-
- @JsonProperty("componentType")
- public void setComponentType(String componentType) {
- this.componentType = componentType;
- }
-
- @JsonProperty("hostname")
- public String getHostname() {
- return hostname;
- }
-
- @JsonProperty("hostname")
- public void setHostname(String hostname) {
- this.hostname = hostname;
- }
-
- @JsonProperty("hostLevelParams")
- public Map<String, String> getHostLevelParams() {
- return hostLevelParams;
- }
-
- @JsonProperty("hostLevelParams")
- public void setHostLevelParams(Map<String, String> params) {
- this.hostLevelParams = params;
- }
-
- @JsonProperty("configurations")
- public Map<String, Map<String, String>> getConfigurations() {
- return configurations;
- }
-
- @JsonProperty("configurations")
- public void setConfigurations(Map<String, Map<String, String>> configurations) {
- this.configurations = configurations;
- }
-
- @JsonProperty("commandParams")
- public Map<String, String> getCommandParams() {
- return commandParams;
- }
-
- @JsonProperty("commandParams")
- public void setCommandParams(Map<String, String> commandParams) {
- this.commandParams = commandParams;
- }
-
- @JsonProperty("serviceName")
- public String getServiceName() {
- return serviceName;
- }
-
- @JsonProperty("serviceName")
- public void setServiceName(String serviceName) {
- this.serviceName = serviceName;
- }
-
- @JsonProperty("componentName")
- public String getComponentName() {
- return componentName;
- }
-
- @JsonProperty("componentName")
- public void setComponentName(String componentName) {
- this.componentName = componentName;
- }
-
- @JsonProperty("package")
- public String getPkg() {
- return pkg;
- }
-
- @JsonProperty("package")
- public void setPkg(String pkg) {
- this.pkg = pkg;
- }
-
- @JsonProperty("componentConfig")
- public Map<String, Map<String, String>> getComponentConfigurations() {
- return componentConfigurations;
- }
-
- @JsonProperty("componentConfig")
- public void setComponentConfigurations(
- Map<String, Map<String, String>> componentConfigurations) {
- this.componentConfigurations = componentConfigurations;
- }
-
- @JsonProperty("containers")
- public List<DockerContainer> getContainers() {
- return containers;
- }
-
- @JsonProperty("yarnDockerMode")
- public boolean isYarnDockerMode() {
- return yarnDockerMode ;
- }
-
- @JsonProperty("yarnDockerMode")
- public void setYarnDockerMode(boolean yarnDockerMode) {
- this.yarnDockerMode = yarnDockerMode;
- }
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder();
- builder.append("ExecutionCommand [commandType=").append(commandType)
- .append(", clusterName=").append(clusterName).append(", taskId=")
- .append(taskId).append(", commandId=").append(commandId)
- .append(", hostname=").append(hostname).append(", role=").append(role)
- .append(", hostLevelParams=").append(hostLevelParams)
- .append(", roleParams=").append(roleParams).append(", roleCommand=")
- .append(roleCommand).append(", configurations=").append(configurations)
- .append(", commandParams=").append(commandParams)
- .append(", serviceName=").append(serviceName)
- .append(", componentName=").append(componentName)
- .append(", componentType=").append(componentType)
- .append(", yarnDockerMode=").append(yarnDockerMode).append(", pkg=")
- .append(pkg).append("]");
- return builder.toString();
- }
-
- public void addContainerDetails(String componentGroup, Metainfo metaInfo) {
- Component component = metaInfo.getApplicationComponent(componentGroup);
- this.setComponentType(component.getType());
- log.info("Adding container details for {}", componentGroup, " from ",
- metaInfo.toString());
- for (DockerContainer metaContainer : component.getDockerContainers()) {
- DockerContainer container = new DockerContainer();
- container.setImage(metaContainer.getImage());
- container.setNetwork(metaContainer.getNetwork());
- container.setUseNetworkScript(metaContainer.getUseNetworkScript());
- container.setName(metaContainer.getName());
- container.setOptions(metaContainer.getOptions());
- container.setAdditionalParam(metaContainer.getAdditionalParam());
- container.setCommandPath(metaContainer.getAdditionalParam());
- container.setStatusCommand(metaContainer.getStatusCommand());
- container.setStartCommand(metaContainer.getStartCommand());
- if (metaContainer.getMounts().size() > 0) {
- for (DockerContainerMount metaContMount : metaContainer.getMounts()) {
- DockerContainerMount contMnt = new DockerContainerMount();
- contMnt.setContainerMount(metaContMount.getContainerMount());
- contMnt.setHostMount(metaContMount.getHostMount());
- container.getMounts().add(contMnt);
- }
- }
- if (metaContainer.getPorts().size() > 0) {
- for (DockerContainerPort metaCntPort : metaContainer.getPorts()) {
- DockerContainerPort cntPort = new DockerContainerPort();
- cntPort.setContainerPort(metaCntPort.getContainerPort());
- cntPort.setHostPort(metaCntPort.getHostPort());
- container.getPorts().add(cntPort);
- }
- }
- if (metaContainer.getInputFiles().size() > 0) {
- for (DockerContainerInputFile metaInpFile : metaContainer
- .getInputFiles()) {
- DockerContainerInputFile inpFile = new DockerContainerInputFile();
- inpFile.setContainerMount(metaInpFile.getContainerMount());
- inpFile.setFileLocalPath(metaInpFile.getFileLocalPath());
- container.getInputFiles().add(inpFile);
- }
- }
- if (metaContainer.getConfigFiles() != null) {
- container.setConfigFiles(metaContainer.getConfigFiles());
- }
- log.info("Docker container meta info ready: " + container.toString());
- this.getContainers().add(container);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java
deleted file mode 100644
index d17c465..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- *
- *
- * Data model for agent heartbeat for server (ambari or app master).
- *
- */
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class HeartBeat {
- private long responseId = -1;
- private long timestamp;
- private String hostname;
- List<CommandReport> reports = new ArrayList<CommandReport>();
- List<ComponentStatus> componentStatus = new ArrayList<ComponentStatus>();
- private List<DiskInfo> mounts = new ArrayList<DiskInfo>();
- HostStatus nodeStatus;
- private AgentEnv agentEnv = null;
- private String fqdn;
- private String pkg;
-
- public long getResponseId() {
- return responseId;
- }
-
- public void setResponseId(long responseId) {
- this.responseId=responseId;
- }
-
- public long getTimestamp() {
- return timestamp;
- }
-
- public void setTimestamp(long timestamp) {
- this.timestamp = timestamp;
- }
-
- public String getHostname() {
- return hostname;
- }
-
- public void setHostname(String hostname) {
- this.hostname = hostname;
- }
-
- public String getFqdn() {
- return fqdn;
- }
-
- public void setFqdn(String fqdn) {
- this.fqdn = fqdn;
- }
-
- @JsonProperty("reports")
- public List<CommandReport> getReports() {
- return this.reports;
- }
-
- @JsonProperty("reports")
- public void setReports(List<CommandReport> reports) {
- this.reports = reports;
- }
-
- public HostStatus getNodeStatus() {
- return nodeStatus;
- }
-
- public void setNodeStatus(HostStatus nodeStatus) {
- this.nodeStatus = nodeStatus;
- }
-
- public AgentEnv getAgentEnv() {
- return agentEnv;
- }
-
- public void setAgentEnv(AgentEnv env) {
- agentEnv = env;
- }
-
- @JsonProperty("componentStatus")
- public List<ComponentStatus> getComponentStatus() {
- return componentStatus;
- }
-
- @JsonProperty("componentStatus")
- public void setComponentStatus(List<ComponentStatus> componentStatus) {
- this.componentStatus = componentStatus;
- }
-
- @JsonProperty("mounts")
- public List<DiskInfo> getMounts() {
- return this.mounts;
- }
-
- @JsonProperty("mounts")
- public void setMounts(List<DiskInfo> mounts) {
- this.mounts = mounts;
- }
-
- @JsonProperty("package")
- public String getPackage() {
- return pkg;
- }
-
- @JsonProperty("package")
- public void setPackage(String pkg) {
- this.pkg = pkg;
- }
-
- @Override
- public String toString() {
- return "HeartBeat{" +
- "responseId=" + responseId +
- ", timestamp=" + timestamp +
- ", hostname='" + hostname + '\'' +
- ", reports=" + reports +
- ", componentStatus=" + componentStatus +
- ", package=" + pkg +
- ", nodeStatus=" + nodeStatus +
- '}';
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java
deleted file mode 100644
index b500d67..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- *
- * Controller to Agent response data model.
- *
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class HeartBeatResponse {
-
- private long responseId;
-
- List<ExecutionCommand> executionCommands = new ArrayList<ExecutionCommand>();
- List<StatusCommand> statusCommands = new ArrayList<StatusCommand>();
-
- RegistrationCommand registrationCommand;
-
- boolean yarnDockerMode = false;
- boolean restartAgent = false;
- boolean restartEnabled = true;
- boolean hasMappedComponents = false;
- boolean terminateAgent = false;
-
- @JsonProperty("responseId")
- public long getResponseId() {
- return responseId;
- }
-
- @JsonProperty("responseId")
- public void setResponseId(long responseId) {
- this.responseId=responseId;
- }
-
- @JsonProperty("executionCommands")
- public List<ExecutionCommand> getExecutionCommands() {
- return executionCommands;
- }
-
- @JsonProperty("executionCommands")
- public void setExecutionCommands(List<ExecutionCommand> executionCommands) {
- this.executionCommands = executionCommands;
- }
-
- @JsonProperty("statusCommands")
- public List<StatusCommand> getStatusCommands() {
- return statusCommands;
- }
-
- @JsonProperty("statusCommands")
- public void setStatusCommands(List<StatusCommand> statusCommands) {
- this.statusCommands = statusCommands;
- }
-
- @JsonProperty("registrationCommand")
- public RegistrationCommand getRegistrationCommand() {
- return registrationCommand;
- }
-
- @JsonProperty("registrationCommand")
- public void setRegistrationCommand(RegistrationCommand registrationCommand) {
- this.registrationCommand = registrationCommand;
- }
-
- @JsonProperty("restartAgent")
- public boolean isRestartAgent() {
- return restartAgent;
- }
-
- @JsonProperty("restartAgent")
- public void setRestartAgent(boolean restartAgent) {
- this.restartAgent = restartAgent;
- }
-
- @JsonProperty("restartEnabled")
- public boolean getRstartEnabled() {
- return restartEnabled;
- }
-
- @JsonProperty("restartEnabled")
- public void setRestartEnabled(boolean restartEnabled) {
- this.restartEnabled = restartEnabled;
- }
-
- @JsonProperty("hasMappedComponents")
- public boolean hasMappedComponents() {
- return hasMappedComponents;
- }
-
- @JsonProperty("hasMappedComponents")
- public void setHasMappedComponents(boolean hasMappedComponents) {
- this.hasMappedComponents = hasMappedComponents;
- }
-
- @JsonProperty("terminateAgent")
- public boolean isTerminateAgent() {
- return terminateAgent;
- }
-
- @JsonProperty("terminateAgent")
- public void setTerminateAgent(boolean terminateAgent) {
- this.terminateAgent = terminateAgent;
- }
-
- public void addExecutionCommand(ExecutionCommand execCmd) {
- executionCommands.add(execCmd);
- }
-
- public void addStatusCommand(StatusCommand statCmd) {
- statusCommands.add(statCmd);
- }
-
- @Override
- public String toString() {
- return "HeartBeatResponse{" +
- "responseId=" + responseId +
- ", executionCommands=" + executionCommands +
- ", statusCommands=" + statusCommands +
- ", registrationCommand=" + registrationCommand +
- ", restartAgent=" + restartAgent +
- ", terminateAgent=" + terminateAgent +
- '}';
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostInfo.java
deleted file mode 100644
index bef7b07..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostInfo.java
+++ /dev/null
@@ -1,398 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.ArrayList;
-import java.util.List;
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class HostInfo {
- private String architecture;
- private String domain;
- private String fqdn;
- private String hardwareisa;
- private String hardwaremodel;
- private String hostname;
- private String id;
- private String interfaces;
- private String ipaddress;
- private String kernel;
- private String kernelmajversion;
- private String kernelrelease;
- private String kernelversion;
- private String macaddress;
- private long memoryfree;
- private long memorysize;
- private List<DiskInfo> mounts = new ArrayList<DiskInfo>();
- private long memorytotal;
- private String netmask;
- private String operatingsystem;
- private String operatingsystemrelease;
- private String osfamily;
- private int physicalprocessorcount;
- private int processorcount;
- private boolean selinux;
- private String swapfree;
- private String swapsize;
- private String timezone;
- private String uptime;
- private long uptime_days;
- private long uptime_hours;
-
-
- @JsonProperty("architecture")
- public String getArchitecture() {
- return this.architecture;
- }
-
- @JsonProperty("architecture")
- public void setArchitecture(String architecture) {
- this.architecture = architecture;
- }
-
- @JsonProperty("domain")
- public String getDomain() {
- return this.domain;
- }
-
- @JsonProperty("domain")
- public void setDomain(String domain) {
- this.domain = domain;
- }
-
- @JsonProperty("fqdn")
- public String getFQDN() {
- return this.fqdn;
- }
-
- @JsonProperty("fqdn")
- public void setFQDN(String fqdn) {
- this.fqdn = fqdn;
- }
-
- @JsonProperty("hardwareisa")
- public String getHardwareIsa() {
- return hardwareisa;
- }
-
- @JsonProperty("hardwareisa")
- public void setHardwareIsa(String hardwareisa) {
- this.hardwareisa = hardwareisa;
- }
-
- @JsonProperty("hardwaremodel")
- public String getHardwareModel() {
- return this.hardwaremodel;
- }
-
- @JsonProperty("hardwaremodel")
- public void setHardwareModel(String hardwaremodel) {
- this.hardwaremodel = hardwaremodel;
- }
-
- @JsonProperty("hostname")
- public String getHostName() {
- return this.hostname;
- }
-
- @JsonProperty("hostname")
- public void setHostName(String hostname) {
- this.hostname = hostname;
- }
-
- @JsonProperty("id")
- public String getAgentUserId() {
- return id;
- }
-
- @JsonProperty("id")
- public void setAgentUserId(String id) {
- this.id = id;
- }
-
- @JsonProperty("interfaces")
- public String getInterfaces() {
- return this.interfaces;
- }
-
- @JsonProperty("interfaces")
- public void setInterfaces(String interfaces) {
- this.interfaces = interfaces;
- }
-
- @JsonProperty("ipaddress")
- public String getIPAddress() {
- return this.ipaddress;
- }
-
- @JsonProperty("ipaddress")
- public void setIPAddress(String ipaddress) {
- this.ipaddress = ipaddress;
- }
-
- @JsonProperty("kernel")
- public String getKernel() {
- return this.kernel;
- }
-
- @JsonProperty("kernel")
- public void setKernel(String kernel) {
- this.kernel = kernel;
- }
-
- @JsonProperty("kernelmajversion")
- public String getKernelMajVersion() {
- return this.kernelmajversion;
- }
-
- @JsonProperty("kernelmajversion")
- public void setKernelMajVersion(String kernelmajversion) {
- this.kernelmajversion = kernelmajversion;
- }
-
- @JsonProperty("kernelrelease")
- public String getKernelRelease() {
- return this.kernelrelease;
- }
-
- @JsonProperty("kernelrelease")
- public void setKernelRelease(String kernelrelease) {
- this.kernelrelease = kernelrelease;
- }
-
- @JsonProperty("kernelversion")
- public String getKernelVersion() {
- return this.kernelversion;
- }
-
- @JsonProperty("kernelversion")
- public void setKernelVersion(String kernelversion) {
- this.kernelversion = kernelversion;
- }
-
- @JsonProperty("macaddress")
- public String getMacAddress() {
- return this.macaddress;
- }
-
- @JsonProperty("macaddress")
- public void setMacAddress(String macaddress) {
- this.macaddress = macaddress;
- }
-
- @JsonProperty("memoryfree")
- public long getFreeMemory() {
- return this.memoryfree;
- }
-
- @JsonProperty("memoryfree")
- public void setFreeMemory(long memoryfree) {
- this.memoryfree = memoryfree;
- }
-
- @JsonProperty("memorysize")
- public long getMemorySize() {
- return this.memorysize;
- }
-
- @JsonProperty("memorysize")
- public void setMemorySize(long memorysize) {
- this.memorysize = memorysize;
- }
-
- @JsonProperty("mounts")
- public List<DiskInfo> getMounts() {
- return this.mounts;
- }
-
- @JsonProperty("mounts")
- public void setMounts(List<DiskInfo> mounts) {
- this.mounts = mounts;
- }
-
- @JsonProperty("memorytotal")
- public long getMemoryTotal() {
- return this.memorytotal;
- }
-
- @JsonProperty("memorytotal")
- public void setMemoryTotal(long memorytotal) {
- this.memorytotal = memorytotal;
- }
-
- @JsonProperty("netmask")
- public String getNetMask() {
- return this.netmask;
- }
-
- @JsonProperty("netmask")
- public void setNetMask(String netmask) {
- this.netmask = netmask;
- }
-
- @JsonProperty("operatingsystem")
- public String getOS() {
- return this.operatingsystem;
- }
-
- @JsonProperty("operatingsystem")
- public void setOS(String operatingsystem) {
- this.operatingsystem = operatingsystem;
- }
-
- @JsonProperty("operatingsystemrelease")
- public String getOSRelease() {
- return this.operatingsystemrelease;
- }
-
- @JsonProperty("operatingsystemrelease")
- public void setOSRelease(String operatingsystemrelease) {
- this.operatingsystemrelease = operatingsystemrelease;
- }
-
- @JsonProperty("osfamily")
- public String getOSFamily() {
- return this.osfamily;
- }
-
- @JsonProperty("osfamily")
- public void setOSFamily(String osfamily) {
- this.osfamily = osfamily;
- }
-
- @JsonProperty("physicalprocessorcount")
- public int getPhysicalProcessorCount() {
- return this.physicalprocessorcount;
- }
-
- @JsonProperty("physicalprocessorcount")
- public void setPhysicalProcessorCount(int physicalprocessorcount) {
- this.physicalprocessorcount = physicalprocessorcount;
- }
-
- @JsonProperty("processorcount")
- public int getProcessorCount() {
- return this.processorcount;
- }
-
- @JsonProperty("processorcount")
- public void setProcessorCount(int processorcount) {
- this.processorcount = processorcount;
- }
-
- @JsonProperty("selinux")
- public boolean getSeLinux() {
- return selinux;
- }
-
- @JsonProperty("selinux")
- public void setSeLinux(boolean selinux) {
- this.selinux = selinux;
- }
-
- @JsonProperty("swapfree")
- public String getSwapFree() {
- return this.swapfree;
- }
-
- @JsonProperty("swapfree")
- public void setSwapFree(String swapfree) {
- this.swapfree = swapfree;
- }
-
- @JsonProperty("swapsize")
- public String getSwapSize() {
- return swapsize;
- }
-
- @JsonProperty("swapsize")
- public void setSwapSize(String swapsize) {
- this.swapsize = swapsize;
- }
-
- @JsonProperty("timezone")
- public String getTimeZone() {
- return this.timezone;
- }
-
- @JsonProperty("timezone")
- public void setTimeZone(String timezone) {
- this.timezone = timezone;
- }
-
- @JsonProperty("uptime")
- public String getUptime() {
- return this.uptime;
- }
-
- @JsonProperty("uptime")
- public void setUpTime(String uptime) {
- this.uptime = uptime;
- }
-
- @JsonProperty("uptime_hours")
- public long getUptimeHours() {
- return this.uptime_hours;
- }
-
- @JsonProperty("uptime_hours")
- public void setUpTimeHours(long uptime_hours) {
- this.uptime_hours = uptime_hours;
- }
-
- @JsonProperty("uptime_days")
- public long getUpTimeDays() {
- return this.uptime_days;
- }
-
- @JsonProperty("uptime_days")
- public void setUpTimeDays(long uptime_days) {
- this.uptime_days = uptime_days;
- }
-
- private String getDiskString() {
- if (mounts == null) {
- return null;
- }
- StringBuilder ret = new StringBuilder();
- for (DiskInfo diskInfo : mounts) {
- ret.append("(").append(diskInfo.toString()).append(")");
- }
- return ret.toString();
- }
-
- public String toString() {
- return "[" +
- "hostname=" + this.hostname + "," +
- "fqdn=" + this.fqdn + "," +
- "domain=" + this.domain + "," +
- "architecture=" + this.architecture + "," +
- "processorcount=" + this.processorcount + "," +
- "physicalprocessorcount=" + this.physicalprocessorcount + "," +
- "osname=" + this.operatingsystem + "," +
- "osversion=" + this.operatingsystemrelease + "," +
- "osfamily=" + this.osfamily + "," +
- "memory=" + this.memorytotal + "," +
- "uptime_hours=" + this.uptime_hours + "," +
- "mounts=" + getDiskString() + "]\n";
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostStatus.java
deleted file mode 100644
index c584149..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HostStatus.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-/**
- *
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class HostStatus {
- public HostStatus(Status status, String cause) {
- super();
- this.status = status;
- this.cause = cause;
- }
- public HostStatus() {
- super();
- }
-
- public enum Status {
- HEALTHY,
- UNHEALTHY
- }
- Status status;
- String cause;
- public Status getStatus() {
- return status;
- }
- public void setStatus(Status status) {
- this.status = status;
- }
- public String getCause() {
- return cause;
- }
- public void setCause(String cause) {
- this.cause = cause;
- }
-
- @Override
- public String toString() {
- return "HostStatus{" +
- "status=" + status +
- ", cause='" + cause + '\'' +
- '}';
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/51] [abbrv] hadoop git commit: YARN-5941. Slider handles
"per.component" for multiple components incorrectly. Contributed by Billie
Rinaldi
Posted by ji...@apache.org.
YARN-5941. Slider handles "per.component" for multiple components incorrectly. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/076ae6b1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/076ae6b1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/076ae6b1
Branch: refs/heads/yarn-native-services
Commit: 076ae6b1f30c0792142399ba6bd3cb2fcdd63d45
Parents: a31823a
Author: Gour Saha <go...@apache.org>
Authored: Wed Nov 30 14:00:22 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/slider/api/OptionKeys.java | 14 ++++++++++++++
.../org/apache/slider/common/SliderKeys.java | 2 --
.../apache/slider/providers/ProviderUtils.java | 20 ++++++++++++++++----
3 files changed, 30 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/076ae6b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
index 434b1d9..988627d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
@@ -53,8 +53,22 @@ public interface OptionKeys extends InternalKeys {
* Prefix for export options: {@value}
*/
String EXPORT_PREFIX = "export.";
+ /**
+ * Type suffix for config file and package options: {@value}
+ */
String TYPE_SUFFIX = ".type";
+ /**
+ * Name suffix for config file and package options: {@value}
+ */
String NAME_SUFFIX = ".name";
+ /**
+ * Per component suffix for config file options: {@value}
+ */
+ String PER_COMPONENT = ".per.component";
+ /**
+ * Per group suffix for config file options: {@value}
+ */
+ String PER_GROUP = ".per.group";
/**
* Zookeeper quorum host list: {@value}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/076ae6b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 3d25d33..adf40ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -304,8 +304,6 @@ public interface SliderKeys extends SliderXmlConfKeys {
String APP_RESOURCES = "application.resources";
String APP_RESOURCES_DIR = "app/resources";
- String PER_COMPONENT = "per.component";
- String PER_GROUP = "per.group";
String APP_PACKAGES_DIR = "app/packages";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/076ae6b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index 39986c1..bc237f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -614,8 +614,16 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
throw new BadConfigException("Config format " + configFileType +
" doesn't exist");
}
+ boolean perComponent = appConf.getComponentOptBool(roleGroup,
+ OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys
+ .PER_COMPONENT, false);
+ boolean perGroup = appConf.getComponentOptBool(roleGroup,
+ OptionKeys.CONF_FILE_PREFIX + configEntry.getKey() + OptionKeys
+ .PER_GROUP, false);
+
localizeConfigFile(launcher, roleName, roleGroup, configEntry.getKey(),
- configFormat, configFileName, configs, env, fileSystem, clusterName);
+ configFormat, configFileName, configs, env, fileSystem,
+ clusterName, perComponent, perGroup);
}
}
@@ -631,6 +639,8 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
* @param env environment variables
* @param fileSystem file system
* @param clusterName app name
+ * @param perComponent true if file should be created per unique component
+ * @param perGroup true if file should be created per component group
* @throws IOException file cannot be uploaded
*/
public void localizeConfigFile(ContainerLauncher launcher,
@@ -639,7 +649,9 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
Map<String, Map<String, String>> configs,
MapOperations env,
SliderFileSystem fileSystem,
- String clusterName)
+ String clusterName,
+ boolean perComponent,
+ boolean perGroup)
throws IOException {
if (launcher == null) {
return;
@@ -655,9 +667,9 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
}
String folder = null;
- if ("true".equals(config.get(PER_COMPONENT))) {
+ if (perComponent) {
folder = roleName;
- } else if ("true".equals(config.get(PER_GROUP))) {
+ } else if (perGroup) {
folder = roleGroup;
}
if (folder != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/51] [abbrv] hadoop git commit: YARN-5680. Add 2 new fields in
Slider status output - image-name and is-privileged-container. Contributed by
Billie Rinaldi
Posted by ji...@apache.org.
YARN-5680. Add 2 new fields in Slider status output - image-name and is-privileged-container. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b544efb1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b544efb1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b544efb1
Branch: refs/heads/yarn-native-services
Commit: b544efb1f2008e938623e44d00a222b481f8117a
Parents: c89c04a
Author: Gour Saha <go...@apache.org>
Authored: Tue Oct 25 20:00:27 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../org/apache/slider/providers/docker/DockerKeys.java | 1 +
.../slider/providers/docker/DockerProviderService.java | 2 +-
.../apache/slider/server/appmaster/state/AppState.java | 12 ++++++++++++
3 files changed, 14 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b544efb1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
index 40b73a2..0e1d288 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
@@ -26,6 +26,7 @@ public interface DockerKeys {
String DOCKER_START_COMMAND = DOCKER_PREFIX + "startCommand";
String DEFAULT_DOCKER_NETWORK = "bridge";
+ Boolean DEFAULT_DOCKER_USE_PRIVILEGED = false;
String OUT_FILE = "stdout.txt";
String ERR_FILE = "stderr.txt";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b544efb1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
index af36620..cc319ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
@@ -131,7 +131,7 @@ public class DockerProviderService extends AbstractProviderService implements
launcher.setDockerNetwork(appConf.getComponentOpt(roleGroup, DOCKER_NETWORK,
DEFAULT_DOCKER_NETWORK));
launcher.setRunPrivilegedContainer(appConf.getComponentOptBool(roleGroup,
- DOCKER_USE_PRIVILEGED, false));
+ DOCKER_USE_PRIVILEGED, DEFAULT_DOCKER_USE_PRIVILEGED));
// Set the environment
Map<String, String> standardTokens = providerUtils.getStandardTokenMap(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b544efb1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 49e7b78..6db375d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -91,6 +91,9 @@ import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.slider.api.ResourceKeys.*;
import static org.apache.slider.api.RoleKeys.*;
import static org.apache.slider.api.StateValues.*;
+import static org.apache.slider.providers.docker.DockerKeys.DEFAULT_DOCKER_USE_PRIVILEGED;
+import static org.apache.slider.providers.docker.DockerKeys.DOCKER_IMAGE;
+import static org.apache.slider.providers.docker.DockerKeys.DOCKER_USE_PRIVILEGED;
/**
* The model of all the ongoing state of a Slider AM.
@@ -1823,6 +1826,15 @@ public class AppState {
if (SliderUtils.isSet(prefix)) {
cd.setRoleOpt(rolename, ROLE_PREFIX, SliderUtils.trimPrefix(prefix));
}
+ String dockerImage = instanceDefinition.getAppConfOperations()
+ .getComponentOpt(role.getGroup(), DOCKER_IMAGE, null);
+ if (SliderUtils.isSet(dockerImage)) {
+ cd.setRoleOpt(rolename, DOCKER_IMAGE, dockerImage);
+ Boolean dockerUsePrivileged = instanceDefinition.getAppConfOperations()
+ .getComponentOptBool(role.getGroup(), DOCKER_USE_PRIVILEGED,
+ DEFAULT_DOCKER_USE_PRIVILEGED);
+ cd.setRoleOpt(rolename, DOCKER_USE_PRIVILEGED, dockerUsePrivileged);
+ }
List<String> instances = instanceMap.get(rolename);
int nodeCount = instances != null ? instances.size(): 0;
cd.setRoleOpt(rolename, COMPONENT_INSTANCES,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/51] [abbrv] hadoop git commit: YARN-5996. Native services AM
kills app on AMRMClientAsync onError call. Contributed by Billie Rinaldi
Posted by ji...@apache.org.
YARN-5996. Native services AM kills app on AMRMClientAsync onError call. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0cb7071d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0cb7071d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0cb7071d
Branch: refs/heads/yarn-native-services
Commit: 0cb7071d3031a1b673556b54a1f918a33a692e96
Parents: b39605f
Author: Gour Saha <go...@apache.org>
Authored: Fri Dec 16 12:09:29 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../org/apache/slider/server/appmaster/SliderAppMaster.java | 7 +------
1 file changed, 1 insertion(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0cb7071d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 74dbc88..611a3e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -1937,12 +1937,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
@Override //AMRMClientAsync
public void onError(Throwable e) {
- //callback says it's time to finish
- LOG_YARN.error("AMRMClientAsync.onError() received {}", e, e);
- signalAMComplete(new ActionStopSlider("stop",
- EXIT_EXCEPTION_THROWN,
- FinalApplicationStatus.FAILED,
- "AMRMClientAsync.onError() received " + e));
+ LOG_YARN.info("Ignoring AMRMClientAsync.onError() received {}", e);
}
/* =================================================================== */
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/51] [abbrv] hadoop git commit: YARN-5218. Initial core change for
DNS for YARN. Contributed by Jonathan Maron
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
new file mode 100644
index 0000000..52b3c37
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
@@ -0,0 +1,1534 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.registry.server.dns;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.IOFileFilter;
+import org.apache.commons.net.util.Base64;
+import org.apache.commons.net.util.SubnetUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.registry.client.api.DNSOperations;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xbill.DNS.CNAMERecord;
+import org.xbill.DNS.DClass;
+import org.xbill.DNS.DNSKEYRecord;
+import org.xbill.DNS.DNSSEC;
+import org.xbill.DNS.DSRecord;
+import org.xbill.DNS.ExtendedFlags;
+import org.xbill.DNS.Flags;
+import org.xbill.DNS.Header;
+import org.xbill.DNS.Message;
+import org.xbill.DNS.NSRecord;
+import org.xbill.DNS.Name;
+import org.xbill.DNS.NameTooLongException;
+import org.xbill.DNS.OPTRecord;
+import org.xbill.DNS.Opcode;
+import org.xbill.DNS.RRSIGRecord;
+import org.xbill.DNS.RRset;
+import org.xbill.DNS.Rcode;
+import org.xbill.DNS.Record;
+import org.xbill.DNS.SOARecord;
+import org.xbill.DNS.Section;
+import org.xbill.DNS.SetResponse;
+import org.xbill.DNS.TSIG;
+import org.xbill.DNS.TSIGRecord;
+import org.xbill.DNS.TextParseException;
+import org.xbill.DNS.Type;
+import org.xbill.DNS.Zone;
+
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.SocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.channels.DatagramChannel;
+import java.nio.channels.ServerSocketChannel;
+import java.nio.channels.SocketChannel;
+import java.security.KeyFactory;
+import java.security.NoSuchAlgorithmException;
+import java.security.PrivateKey;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.RSAPrivateKeySpec;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.Properties;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.apache.hadoop.registry.client.api.RegistryConstants.*;
+
+/**
+ * A DNS service reflecting the state of the YARN registry. Records are created
+ * based on service records available in the YARN ZK-based registry.
+ */
+public class RegistryDNS extends AbstractService implements DNSOperations,
+ ZoneSelector {
+
+ public static final String CONTAINER = "container";
+
+ static final int FLAG_DNSSECOK = 1;
+ static final int FLAG_SIGONLY = 2;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RegistryDNS.class);
+ public static final String IN_ADDR_ARPA = "in-addr.arpa.";
+ public static final String ZONE_SUFFIX = ".zone";
+
+ private ExecutorService executor;
+ private ReentrantReadWriteLock zoneLock = new ReentrantReadWriteLock();
+ private CloseableLock readLock = new CloseableLock(zoneLock.readLock());
+ private CloseableLock writeLock = new CloseableLock(zoneLock.writeLock());
+ private String domainName;
+ private long ttl = 0L;
+
+ private static final Pattern USER_NAME = Pattern.compile("/users/(\\w*)/?");
+ private Boolean dnssecEnabled;
+ private PrivateKey privateKey;
+
+ private ConcurrentMap<Name, DNSKEYRecord> dnsKeyRecs =
+ new ConcurrentHashMap<>();
+ private ConcurrentMap<Name, Zone> zones = new ConcurrentHashMap<>();
+ private Name bindHost;
+
+ /**
+ * Construct the service.
+ *
+ * @param name service name
+ */
+ public RegistryDNS(String name) {
+ super(name);
+ executor = HadoopExecutors.newCachedThreadPool(
+ new ThreadFactory() {
+ private AtomicInteger counter = new AtomicInteger(1);
+
+ @Override
+ public Thread newThread(Runnable r) {
+ return new Thread(r,
+ "RegistryDNS "
+ + counter.getAndIncrement());
+ }
+ });
+ }
+
+ /**
+ * Initializes the registry.
+ *
+ * @param conf the hadoop configuration
+ * @throws Exception if there are tcp/udp issues
+ */
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+
+ // create the zone. for now create a "dummy" SOA record
+ try {
+ setDomainName(conf);
+
+ int port = initializeZones(conf);
+
+ InetAddress addr = InetAddress.getLocalHost();
+
+ String bindAddress = conf.get(KEY_DNS_BIND_ADDRESS);
+ if (bindAddress != null) {
+ addr = InetAddress.getByName(bindAddress);
+ }
+ addNIOUDP(addr, port);
+ addNIOTCP(addr, port);
+
+ } catch (IOException e) {
+ LOG.error("Error initializing Registry DNS Server", e);
+ throw e;
+ }
+ }
+
+ /**
+ * Initializes the registry based on available parameters in the hadoop
+ * configuration.
+ *
+ * @param conf the hadoop configuration
+ * @return the listener port
+ * @throws IOException
+ */
+ int initializeZones(Configuration conf) throws IOException {
+ int port = conf.getInt(KEY_DNS_PORT, DEFAULT_DNS_PORT);
+ ttl = conf.getTimeDuration(KEY_DNS_TTL, 1L, TimeUnit.SECONDS);
+ RecordCreatorFactory.setTtl(ttl);
+
+ setDNSSECEnabled(conf);
+
+ initializeZonesFromFiles(conf);
+
+ Zone registryZone = configureZone(Name.fromString(domainName), conf);
+ zones.put(registryZone.getOrigin(), registryZone);
+
+ initializeReverseLookupZone(conf);
+
+ return port;
+ }
+
+ /**
+ * Signs zone records if necessary (DNSSEC enabled). Zones may not have
+ * their NS and SOA records signed if they were initialized from master files.
+ */
+ private void signZones() throws IOException {
+ if (isDNSSECEnabled()) {
+ Collection<Zone> zoneCollection = zones.values();
+ for (Zone zone : zoneCollection) {
+ Iterator itor = zone.iterator();
+ while (itor.hasNext()) {
+ RRset rRset = (RRset) itor.next();
+ Iterator sigs = rRset.sigs();
+ if (!sigs.hasNext()) {
+ try {
+ signSiteRecord(zone, rRset.first());
+ } catch (DNSSEC.DNSSECException e) {
+ throw new IOException(e);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Initializes a zone by reading any zone file by the same name in the
+ * designated zone file directory.
+ *
+ * @param conf the Hadoop configuration object.
+ * @throws IOException
+ */
+ private void initializeZonesFromFiles(Configuration conf) throws IOException {
+ // should this be in HDFS?
+ String zonesDir = conf.get(KEY_DNS_ZONES_DIR);
+ if (zonesDir != null) {
+ Iterator<File> iterator = FileUtils.iterateFiles(new File(zonesDir),
+ new IOFileFilter() {
+ @Override
+ public boolean accept(
+ File file) {
+ return file.getName().endsWith(
+ ZONE_SUFFIX);
+ }
+
+ @Override
+ public boolean accept(
+ File file,
+ String s) {
+ return s.endsWith(
+ ZONE_SUFFIX);
+ }
+ }, null);
+ while (iterator.hasNext()) {
+ File file = iterator.next();
+ String name = file.getName();
+ name = name.substring(0, name.indexOf(ZONE_SUFFIX) + 1);
+ Zone zone = new SecureableZone(Name.fromString(name),
+ file.getAbsolutePath());
+ zones.putIfAbsent(zone.getOrigin(), zone);
+ }
+ }
+ }
+
+ /**
+ * Initializes the reverse lookup zone (mapping IP to name).
+ *
+ * @param conf the Hadoop configuration.
+ * @throws IOException
+ */
+ private void initializeReverseLookupZone(Configuration conf)
+ throws IOException {
+ Name reverseLookupZoneName = getReverseZoneName(conf);
+ Zone reverseLookupZone =
+ configureZone(reverseLookupZoneName, conf);
+ zones.put(reverseLookupZone.getOrigin(), reverseLookupZone);
+ }
+
+ /**
+ * Returns the list of reverse lookup zones.
+ *
+ * @param conf the hadoop configuration.
+ * @return the list of reverse zone names required based on the configuration
+ * properties.
+ */
+ protected Name getReverseZoneName(Configuration conf) {
+ Name name = null;
+ String zoneSubnet = getZoneSubnet(conf);
+ if (zoneSubnet == null) {
+ LOG.warn("Zone subnet is not configured. Reverse lookups disabled");
+ } else {
+ // is there a netmask
+ String mask = conf.get(KEY_DNS_ZONE_MASK);
+ if (mask != null) {
+ // get the range of IPs
+ SubnetUtils utils = new SubnetUtils(zoneSubnet, mask);
+ name = getReverseZoneName(utils, zoneSubnet);
+ } else {
+ name = getReverseZoneName(zoneSubnet);
+ }
+ }
+ return name;
+ }
+
+ /**
+ * Return the subnet for the zone. this should be a network address for the
+ * subnet (ends in ".0").
+ *
+ * @param conf the hadoop configuration.
+ * @return the zone subnet.
+ */
+ private String getZoneSubnet(Configuration conf) {
+ String subnet = conf.get(KEY_DNS_ZONE_SUBNET);
+ if (subnet != null) {
+ final String[] bytes = subnet.split("\\.");
+ if (bytes.length == 3) {
+ subnet += ".0";
+ }
+ }
+ return subnet;
+ }
+
+ /**
+ * Return the reverse zone name based on the address.
+ *
+ * @param networkAddress the network address.
+ * @return the reverse zone name.
+ */
+ private Name getReverseZoneName(String networkAddress) {
+ return getReverseZoneName(null, networkAddress);
+ }
+
+ /**
+ * Return the reverse zone name based on the address.
+ *
+ * @param utils subnet utils
+ * @param networkAddress the network address.
+ * @return the reverse zone name.
+ */
+ private Name getReverseZoneName(SubnetUtils utils, String networkAddress) {
+ Name reverseZoneName = null;
+ boolean isLargeNetwork = false;
+ if (utils != null) {
+ isLargeNetwork = utils.getInfo().getAddressCount() > 256;
+ }
+ final String[] bytes = networkAddress.split("\\.");
+ if (bytes.length == 4) {
+ String reverseLookupZoneName = null;
+ if (isLargeNetwork) {
+ reverseLookupZoneName =
+ String.format("%s.%s.%s",
+ bytes[1],
+ bytes[0],
+ IN_ADDR_ARPA);
+ } else {
+ reverseLookupZoneName =
+ String.format("%s.%s.%s.%s",
+ bytes[2],
+ bytes[1],
+ bytes[0],
+ IN_ADDR_ARPA);
+ }
+ try {
+ reverseZoneName = Name.fromString(reverseLookupZoneName);
+ } catch (TextParseException e) {
+ LOG.warn("Unable to convert {} to DNS name", reverseLookupZoneName);
+ }
+ }
+ return reverseZoneName;
+ }
+
+ /**
+ * Create the zone and its related zone associated DNS records (NS, SOA).
+ *
+ * @param zoneName domain name of the zone
+ * @param conf configuration reference.
+ * @return the zone.
+ * @throws IOException
+ */
+ private Zone configureZone(Name zoneName, Configuration conf)
+ throws IOException {
+ bindHost = Name.fromString(
+ InetAddress.getLocalHost().getCanonicalHostName() + ".");
+ SOARecord soaRecord = new SOARecord(zoneName, DClass.IN, ttl,
+ bindHost,
+ bindHost, getSerial(), 86000, 7200,
+ 1209600, 600);
+ NSRecord nsRecord = new NSRecord(zoneName, DClass.IN, ttl, bindHost);
+ Zone zone = zones.get(zoneName);
+ if (zone == null) {
+ zone = new SecureableZone(zoneName, new Record[] {soaRecord, nsRecord});
+ }
+
+ try {
+ enableDNSSECIfNecessary(zone, conf, soaRecord, nsRecord);
+ } catch (NoSuchAlgorithmException e) {
+ throw new IOException(e);
+ } catch (InvalidKeySpecException e) {
+ throw new IOException(e);
+ } catch (DNSSEC.DNSSECException e) {
+ throw new IOException(e);
+ }
+
+ return zone;
+ }
+
+ /**
+ * Return a serial number based on the current date and time.
+ *
+ * @return the serial number.
+ */
+ private long getSerial() {
+ Date curDate = new Date();
+ SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyyMMddHH");
+ String serial = simpleDateFormat.format(curDate);
+ return Long.parseLong(serial);
+ }
+
+ /**
+ * Set the value of the DNSSEC enabled property.
+ *
+ * @param conf the Hadoop configuration.
+ */
+ private void setDNSSECEnabled(Configuration conf) {
+ dnssecEnabled = conf.getBoolean(KEY_DNSSEC_ENABLED, false);
+ }
+
+ /**
+ * Is DNSSEC enabled?
+ *
+ * @return true if enabled, false otherwise.
+ */
+ private boolean isDNSSECEnabled() {
+ return dnssecEnabled;
+ }
+
+ /**
+ * Load the required public/private keys, create the zone DNSKEY record, and
+ * sign the zone level records.
+ *
+ * @param zone the zone.
+ * @param conf the configuration.
+ * @param soaRecord the SOA record.
+ * @param nsRecord the NS record.
+ * @throws IOException
+ * @throws NoSuchAlgorithmException
+ * @throws InvalidKeySpecException
+ * @throws DNSSEC.DNSSECException
+ */
+ private void enableDNSSECIfNecessary(Zone zone, Configuration conf,
+ SOARecord soaRecord,
+ NSRecord nsRecord)
+ throws IOException, NoSuchAlgorithmException, InvalidKeySpecException,
+ DNSSEC.DNSSECException {
+ if (isDNSSECEnabled()) {
+ // read in the DNSKEY and create the DNSKEYRecord
+ // TODO: reading these out of config seems wrong...
+ String publicKey = conf.get(KEY_DNSSEC_PUBLIC_KEY);
+ if (publicKey == null) {
+ throw new IOException("DNSSEC Key not configured");
+ }
+ //TODO - perhaps read in actual DNSKEY record structure?
+ Name zoneName = zone.getOrigin();
+ DNSKEYRecord dnskeyRecord = dnsKeyRecs.get(zoneName);
+ if (dnskeyRecord == null) {
+ byte[] key = Base64.decodeBase64(publicKey.getBytes("UTF-8"));
+ dnskeyRecord = new DNSKEYRecord(zoneName,
+ DClass.IN, ttl,
+ DNSKEYRecord.Flags.ZONE_KEY,
+ DNSKEYRecord.Protocol.DNSSEC,
+ DNSSEC.Algorithm.RSASHA256, key);
+ dnsKeyRecs.putIfAbsent(zoneName, dnskeyRecord);
+ }
+ LOG.info("Registering {}", dnskeyRecord);
+ try (CloseableLock lock = writeLock.lock()) {
+ zone.addRecord(dnskeyRecord);
+
+ String privateKeyFile = conf.get(KEY_DNSSEC_PRIVATE_KEY_FILE,
+ DEFAULT_DNSSEC_PRIVATE_KEY_FILE);
+
+ Properties props = new Properties();
+ try (
+ FileInputStream inputStream = new FileInputStream(privateKeyFile)) {
+ props.load(inputStream);
+ }
+
+ String privateModulus = props.getProperty("Modulus");
+ String privateExponent = props.getProperty("PrivateExponent");
+
+ RSAPrivateKeySpec privateSpec = new RSAPrivateKeySpec(
+ new BigInteger(1, Base64.decodeBase64(privateModulus)),
+ new BigInteger(1, Base64.decodeBase64(privateExponent)));
+
+ KeyFactory factory = KeyFactory.getInstance("RSA");
+ privateKey = factory.generatePrivate(privateSpec);
+
+ signSiteRecord(zone, dnskeyRecord);
+ signSiteRecord(zone, soaRecord);
+ signSiteRecord(zone, nsRecord);
+ }
+ // create required DS records
+
+ // domain
+// DSRecord dsRecord = new DSRecord(zoneName, DClass.IN, ttl,
+// DSRecord.Digest.SHA1, dnskeyRecord);
+// zone.addRecord(dsRecord);
+// signSiteRecord(zone, dsRecord);
+ }
+ }
+
+ /**
+ * Sign a DNS record.
+ *
+ * @param zone the zone reference
+ * @param record the record to sign.
+ * @throws DNSSEC.DNSSECException
+ */
+ private void signSiteRecord(Zone zone, Record record)
+ throws DNSSEC.DNSSECException {
+ RRset rrset = zone.findExactMatch(record.getName(),
+ record.getType());
+ Calendar cal = Calendar.getInstance();
+ Date inception = cal.getTime();
+ cal.add(Calendar.YEAR, 1);
+ Date expiration = cal.getTime();
+ RRSIGRecord rrsigRecord =
+ DNSSEC.sign(rrset, dnsKeyRecs.get(zone.getOrigin()),
+ privateKey, inception, expiration);
+ LOG.info("Adding {}", record);
+ rrset.addRR(rrsigRecord);
+ }
+
+ /**
+ * Sets the zone/domain name. The name will be read from the configuration
+ * and the code will ensure the name is absolute.
+ *
+ * @param conf the configuration.
+ * @throws IOException
+ */
+ void setDomainName(Configuration conf) throws IOException {
+ domainName = conf.get(KEY_DNS_DOMAIN);
+ if (domainName == null) {
+ throw new IOException("No DNS domain name specified");
+ }
+ if (!domainName.endsWith(".")) {
+ domainName += ".";
+ }
+ }
+
+ /**
+ * Stops the registry.
+ *
+ * @throws Exception if the service stop generates an issue.
+ */
+ @Override
+ protected void serviceStop() throws Exception {
+ stopExecutor();
+ super.serviceStop();
+ }
+
+ /**
+ * Shuts down the leveraged executor service.
+ */
+ protected synchronized void stopExecutor() {
+ if (executor != null) {
+ executor.shutdownNow();
+ }
+ }
+
+ /**
+ * Creates a DNS error response.
+ *
+ * @param in the byte array detailing the error.
+ * @return the error message, in bytes
+ */
+ public byte[] formErrorMessage(byte[] in) {
+ Header header;
+ try {
+ header = new Header(in);
+ } catch (IOException e) {
+ return null;
+ }
+ return buildErrorMessage(header, Rcode.FORMERR, null);
+ }
+
+ /**
+ * Process a TCP request.
+ *
+ * @param ch the socket channel for the request.
+ * @throws IOException if the tcp processing generates an issue.
+ */
+ public void nioTCPClient(SocketChannel ch) throws IOException {
+ try {
+ // query sizes are small, so the following two lines should work
+ // in all instances
+ ByteBuffer buf = ByteBuffer.allocate(1024);
+ ch.read(buf);
+ buf.flip();
+ int messageLength = getMessgeLength(buf);
+
+ byte[] in = new byte[messageLength];
+
+ buf.get(in, 0, messageLength);
+
+ Message query;
+ byte[] response = null;
+ try {
+ query = new Message(in);
+ response = generateReply(query, ch.socket());
+ if (response == null) {
+ return;
+ }
+ } catch (IOException e) {
+ response = formErrorMessage(in);
+ }
+
+ ByteBuffer out = ByteBuffer.allocate(response.length + 2);
+ out.putShort(0, (short) (response.length & 0xffff));
+ out.put(response);
+
+ ch.write(out);
+ } catch (IOException e) {
+ throw NetUtils.wrapException(ch.socket().getInetAddress().getHostName(),
+ ch.socket().getPort(),
+ ch.socket().getLocalAddress().getHostName(),
+ ch.socket().getLocalPort(), e);
+ } finally {
+ IOUtils.closeStream(ch);
+ }
+
+ }
+
+ /**
+ * Calculate the inbound message length, which is related in the message as an
+ * unsigned short value.
+ *
+ * @param buf the byte buffer containing the message.
+ * @return the message length
+ * @throws EOFException
+ */
+ private int getMessgeLength(ByteBuffer buf) throws EOFException {
+ int ch1 = buf.get();
+ int ch2 = buf.get();
+ if ((ch1 | ch2) < 0) {
+ throw new EOFException();
+ }
+ return (ch1 << 8) + (ch2 & 0xff);
+ }
+
+ /**
+ * Monitor the TCP socket for inbound requests.
+ *
+ * @param serverSocketChannel the server socket channel
+ * @param addr the local inet address
+ * @param port the listener (local) port
+ * @throws Exception if the tcp processing fails.
+ */
+ public void serveNIOTCP(ServerSocketChannel serverSocketChannel,
+ InetAddress addr, int port) throws Exception {
+ try {
+
+ while (true) {
+ final SocketChannel socketChannel = serverSocketChannel.accept();
+ if (socketChannel != null) {
+ executor.submit(new Callable<Boolean>() {
+ @Override
+ public Boolean call() throws Exception {
+ nioTCPClient(socketChannel);
+ return true;
+ }
+ });
+
+ }
+ }
+ } catch (IOException e) {
+ throw NetUtils.wrapException(addr.getHostName(), port,
+ addr.getHostName(), port, e);
+ }
+ }
+
+ /**
+ * Open the TCP listener.
+ *
+ * @param addr the host address.
+ * @param port the host port.
+ * @return the created server socket channel.
+ * @throws IOException
+ */
+ private ServerSocketChannel openTCPChannel(InetAddress addr, int port)
+ throws IOException {
+ ServerSocketChannel serverSocketChannel = ServerSocketChannel.open();
+ try {
+ serverSocketChannel.socket().bind(new InetSocketAddress(addr, port));
+ serverSocketChannel.configureBlocking(false);
+ } catch (IOException e) {
+ throw NetUtils.wrapException(null, 0,
+ InetAddress.getLocalHost().getHostName(),
+ port, e);
+ }
+ return serverSocketChannel;
+ }
+
+ /**
+ * Create the thread (Callable) monitoring the TCP listener.
+ *
+ * @param addr host address.
+ * @param port host port.
+ * @throws Exception if the tcp listener generates an error.
+ */
+ public void addNIOTCP(final InetAddress addr, final int port)
+ throws Exception {
+ final ServerSocketChannel tcpChannel = openTCPChannel(addr, port);
+ executor.submit(new Callable<Boolean>() {
+ @Override
+ public Boolean call() throws Exception {
+ try {
+ serveNIOTCP(tcpChannel, addr, port);
+ } catch (Exception e) {
+ LOG.error("Error initializing DNS TCP listener", e);
+ throw e;
+ }
+
+ return true;
+ }
+
+ });
+ }
+
+ /**
+ * Create the thread monitoring the socket for inbound UDP requests.
+ *
+ * @param addr host address.
+ * @param port host port.
+ * @throws Exception if the UDP listener creation generates an error.
+ */
+ public void addNIOUDP(final InetAddress addr, final int port)
+ throws Exception {
+ final DatagramChannel udpChannel = openUDPChannel(addr, port);
+ executor.submit(new Callable<Boolean>() {
+ @Override
+ public Boolean call() throws Exception {
+ try {
+ serveNIOUDP(udpChannel, addr, port);
+ } catch (Exception e) {
+ LOG.error("Error initializing DNS UDP listener", e);
+ throw e;
+ }
+ return true;
+ }
+ });
+ }
+
+ /**
+ * Process an inbound UDP request.
+ *
+ * @param channel the UDP datagram channel.
+ * @param addr local host address.
+ * @param port local port.
+ * @throws IOException if the UDP processing fails.
+ */
+ private void serveNIOUDP(DatagramChannel channel,
+ InetAddress addr, int port) throws Exception {
+ SocketAddress remoteAddress = null;
+ try {
+
+ ByteBuffer input = ByteBuffer.allocate(4096);
+ ByteBuffer output = ByteBuffer.allocate(4096);
+ byte[] in = null;
+
+ while (true) {
+ input.clear();
+ try {
+ remoteAddress = channel.receive(input);
+ } catch (IOException e) {
+ LOG.debug("Error during message receipt", e);
+ continue;
+ }
+ Message query;
+ byte[] response = null;
+ try {
+ int position = input.position();
+ in = new byte[position];
+ input.flip();
+ input.get(in);
+ query = new Message(in);
+ LOG.info("{}: received query {}", remoteAddress,
+ query.getQuestion());
+ response = generateReply(query, null);
+ if (response == null) {
+ continue;
+ }
+ } catch (IOException e) {
+ response = formErrorMessage(in);
+ }
+ output.clear();
+ output.put(response);
+ output.flip();
+
+ LOG.info("{}: sending response", remoteAddress);
+ channel.send(output, remoteAddress);
+ }
+ } catch (Exception e) {
+ if (e instanceof IOException && remoteAddress != null) {
+ throw NetUtils.wrapException(addr.getHostName(),
+ port,
+ ((InetSocketAddress) remoteAddress).getHostName(),
+ ((InetSocketAddress) remoteAddress).getPort(),
+ (IOException) e);
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ /**
+ * Create and UDP listener socket.
+ *
+ * @param addr host address.
+ * @param port host port.
+ * @return
+ * @throws IOException if listener creation fails.
+ */
+ private DatagramChannel openUDPChannel(InetAddress addr, int port)
+ throws IOException {
+ DatagramChannel channel = DatagramChannel.open();
+ try {
+ channel.socket().bind(new InetSocketAddress(addr, port));
+ } catch (IOException e) {
+ throw NetUtils.wrapException(null, 0,
+ InetAddress.getLocalHost().getHostName(),
+ port, e);
+ }
+ return channel;
+ }
+
+ /**
+ * Create an error message.
+ *
+ * @param header the response header.
+ * @param rcode the response code.
+ * @param question the question record.
+ * @return the error message.
+ */
+ byte[] buildErrorMessage(Header header, int rcode, Record question) {
+ Message response = new Message();
+ response.setHeader(header);
+ for (int i = 0; i < 4; i++) {
+ response.removeAllRecords(i);
+ }
+ if (rcode == Rcode.SERVFAIL) {
+ response.addRecord(question, Section.QUESTION);
+ }
+ header.setRcode(rcode);
+ return response.toWire();
+ }
+
+ /**
+ * Generate an error message based on inbound query.
+ *
+ * @param query the query.
+ * @param rcode the response code for the specific error.
+ * @return the error message.
+ */
+ public byte[] errorMessage(Message query, int rcode) {
+ return buildErrorMessage(query.getHeader(), rcode,
+ query.getQuestion());
+ }
+
+ /**
+ * Generate the response for the inbound DNS query.
+ *
+ * @param query the query.
+ * @param s the socket associated with the query.
+ * @return the response, in bytes.
+ * @throws IOException if reply generation fails.
+ */
+ byte[] generateReply(Message query, Socket s)
+ throws IOException {
+ Header header;
+ boolean badversion;
+ int maxLength;
+ int flags = 0;
+
+ OPTRecord queryOPT = query.getOPT();
+ maxLength = getMaxLength(s, queryOPT);
+
+ header = query.getHeader();
+ if (header.getFlag(Flags.QR)) {
+ LOG.debug("returning null");
+ return null;
+ }
+ if (header.getRcode() != Rcode.NOERROR) {
+ return errorMessage(query, Rcode.FORMERR);
+ }
+ if (header.getOpcode() != Opcode.QUERY) {
+ return errorMessage(query, Rcode.NOTIMP);
+ }
+
+ Record queryRecord = query.getQuestion();
+
+ if (queryOPT != null && (queryOPT.getFlags() & ExtendedFlags.DO) != 0) {
+ flags = FLAG_DNSSECOK;
+ }
+
+ Message response = new Message(query.getHeader().getID());
+ response.getHeader().setFlag(Flags.QR);
+ if (query.getHeader().getFlag(Flags.RD)) {
+ response.getHeader().setFlag(Flags.RD);
+ }
+ response.addRecord(queryRecord, Section.QUESTION);
+
+ Name name = queryRecord.getName();
+ int type = queryRecord.getType();
+ int dclass = queryRecord.getDClass();
+
+ TSIGRecord queryTSIG = query.getTSIG();
+ if (type == Type.AXFR && s != null) {
+ return doAXFR(name, query, null, queryTSIG, s);
+ }
+ if (!Type.isRR(type) && type != Type.ANY) {
+ return errorMessage(query, Rcode.NOTIMP);
+ }
+
+ LOG.debug("calling addAnswer");
+ byte rcode = addAnswer(response, name, type, dclass, 0, flags);
+ if (rcode != Rcode.NOERROR && rcode != Rcode.NXDOMAIN) {
+ return errorMessage(query, rcode);
+ }
+
+ addAdditional(response, flags);
+
+ if (queryOPT != null) {
+ int optflags = (flags == FLAG_DNSSECOK) ? ExtendedFlags.DO : 0;
+ OPTRecord opt = new OPTRecord((short) 4096, rcode >>> 16, (byte) 0,
+ optflags);
+ response.addRecord(opt, Section.ADDITIONAL);
+ }
+
+ return response.toWire(maxLength);
+ }
+
+ /**
+ * Create a query to forward to the primary DNS server (if configured).
+ * NOTE: Experimental
+ *
+ * @param query the inbound query.
+ * @return the query to forward to the primary server.
+ * @throws NameTooLongException
+ * @throws TextParseException if query creation fails.
+ */
+ private Message createPrimaryQuery(Message query)
+ throws NameTooLongException, TextParseException {
+ Name name = query.getQuestion().getName();
+ if (name.labels() > 0 && name.labels() <= 2) {
+ // short relative or absolute name. this code may not be necessary -
+ // OS resolution utilities probably append the search paths defined
+ // in resolv.conf prior to the lookup
+ int id = query.getHeader().getID();
+ String queryName = name.getLabelString(0);
+ Name qualifiedName = Name.concatenate(Name.fromString(queryName),
+ Name.fromString(domainName));
+ LOG.info("Received query {}. Forwarding query {}", name, qualifiedName);
+ Record question = Record.newRecord(qualifiedName,
+ query.getQuestion().getType(),
+ query.getQuestion().getDClass());
+ query = Message.newQuery(question);
+ query.getHeader().setID(id);
+ }
+ return query;
+ }
+
+ /**
+ * Calculate the max length for a response.
+ *
+ * @param s the request socket.
+ * @param queryOPT describes Extended DNS (EDNS) properties of a Message.
+ * @return the length of the response.
+ */
+ private int getMaxLength(Socket s, OPTRecord queryOPT) {
+ int maxLength;
+ if (s != null) {
+ maxLength = 65535;
+ } else if (queryOPT != null) {
+ maxLength = Math.max(queryOPT.getPayloadSize(), 512);
+ } else {
+ maxLength = 512;
+ }
+ return maxLength;
+ }
+
+ /**
+ * Add additional information to a DNS response section if a glue name is
+ * specified.
+ *
+ * @param response the response message.
+ * @param section the section of the response (e.g. ANSWER, AUTHORITY)
+ * @param flags the flags.
+ */
+ private void addAdditional2(Message response, int section, int flags) {
+ Record[] records = response.getSectionArray(section);
+ for (int i = 0; i < records.length; i++) {
+ Record r = records[i];
+ Name glueName = r.getAdditionalName();
+ if (glueName != null) {
+ addGlue(response, glueName, flags);
+ }
+ }
+ }
+
+ /**
+ * Process any additional records indicated for both the ANSWER and AUTHORITY
+ * sections of the response.
+ *
+ * @param response the response message.
+ * @param flags the flags.
+ */
+ private void addAdditional(Message response, int flags) {
+ addAdditional2(response, Section.ANSWER, flags);
+ addAdditional2(response, Section.AUTHORITY, flags);
+ }
+
+ /**
+ * Add the specific record indicated by the "glue", or the mapping to a
+ * specific host.
+ *
+ * @param response the response message.
+ * @param name the name of the glue record.
+ * @param flags the flags.
+ */
+ private void addGlue(Message response, Name name, int flags) {
+ RRset a = findExactMatch(name, Type.A);
+ if (a == null) {
+ return;
+ }
+ addRRset(name, response, a, Section.ADDITIONAL, flags);
+ }
+
+ /**
+ * Find the record set that matches the requested name and type.
+ *
+ * @param name the requested name.
+ * @param type the record type.
+ * @return the set of records with the given name and type.
+ */
+ public RRset findExactMatch(Name name, int type) {
+ try (CloseableLock lock = readLock.lock()) {
+ Zone zone = findBestZone(name);
+ if (zone != null) {
+ return zone.findExactMatch(name, type);
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * Find the zone that correlates to the provided name.
+ *
+ * @param name the name to be matched to a zone.
+ * @return the zone.
+ */
+ @Override public Zone findBestZone(Name name) {
+ Zone foundzone = null;
+ foundzone = zones.get(name);
+ if (foundzone != null) {
+ return foundzone;
+ }
+ int labels = name.labels();
+ for (int i = 1; i < labels; i++) {
+ Name tname = new Name(name, i);
+ foundzone = zones.get(tname);
+ if (foundzone != null) {
+ return foundzone;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Add the answer section to the response.
+ *
+ * @param response the response message.
+ * @param name the name of the answer record.
+ * @param type the type of record.
+ * @param dclass the DNS class.
+ * @param iterations iteration count.
+ * @param flags
+ * @return the response code.
+ */
+ byte addAnswer(Message response, Name name, int type, int dclass,
+ int iterations, int flags) {
+ SetResponse sr = null;
+ byte rcode = Rcode.NOERROR;
+
+ if (iterations > 6) {
+ return Rcode.NOERROR;
+ }
+
+ if (type == Type.SIG || type == Type.RRSIG) {
+ type = Type.ANY;
+ flags |= FLAG_SIGONLY;
+ }
+
+ Zone zone = findBestZone(name);
+
+ LOG.debug("finding record");
+ try (CloseableLock lock = readLock.lock()) {
+ if (zone != null) {
+ sr = zone.findRecords(name, type);
+ } else {
+ rcode = Rcode.NOTAUTH;
+ }
+ }
+ LOG.info("found record? {}", sr != null && sr.isSuccessful());
+
+ if (sr != null) {
+ if (sr.isCNAME()) {
+ CNAMERecord cname = sr.getCNAME();
+ RRset rrset = zone.findExactMatch(cname.getName(), Type.CNAME);
+ addRRset(name, response, rrset, Section.ANSWER, flags);
+ if (iterations == 0) {
+ response.getHeader().setFlag(Flags.AA);
+ }
+ rcode = addAnswer(response, cname.getTarget(),
+ type, dclass, iterations + 1, flags);
+ }
+ if (sr.isNXDOMAIN()) {
+ response.getHeader().setRcode(Rcode.NXDOMAIN);
+ if (isDNSSECEnabled()) {
+ try {
+ addNXT(response, flags);
+ } catch (Exception e) {
+ LOG.warn("Unable to add NXTRecord to AUTHORITY Section", e);
+ }
+ }
+ addSOA(response, zone, flags);
+ if (iterations == 0) {
+ response.getHeader().setFlag(Flags.AA);
+ }
+ rcode = Rcode.NXDOMAIN;
+ } else if (sr.isNXRRSET()) {
+ LOG.info("No data found the given name {} and type {}", name, type);
+ addSOA(response, zone, flags);
+ if (iterations == 0) {
+ response.getHeader().setFlag(Flags.AA);
+ }
+ } else if (sr.isSuccessful()) {
+ RRset[] rrsets = sr.answers();
+ LOG.info("found answers {}", rrsets);
+ for (int i = 0; i < rrsets.length; i++) {
+ addRRset(name, response, rrsets[i],
+ Section.ANSWER, flags);
+ }
+ addNS(response, zone, flags);
+ if (iterations == 0) {
+ response.getHeader().setFlag(Flags.AA);
+ }
+ }
+ } else {
+ if (zone != null) {
+ Name defaultDomain = null;
+ try {
+ defaultDomain = Name.fromString(domainName);
+ zone = zones.get(defaultDomain);
+ addNS(response, zone, flags);
+ if (iterations == 0) {
+ response.getHeader().setFlag(Flags.AA);
+ }
+ } catch (TextParseException e) {
+ LOG.warn("Unable to obtain default zone for unknown name response",
+ e);
+ }
+ }
+ }
+ return rcode;
+ }
+
+ /**
+ * Add the SOA record (describes the properties of the zone) to the authority
+ * section of the response.
+ *
+ * @param response the response message.
+ * @param zone the DNS zone.
+ */
+ private void addSOA(Message response, Zone zone, int flags) {
+ RRset soa = zone.findExactMatch(zone.getOrigin(), Type.SOA);
+ addRRset(soa.getName(), response, soa,
+ Section.AUTHORITY, flags);
+ }
+
+ /**
+ * Add the NXT record to the authority
+ * section of the response.
+ *
+ * @param response the response message.
+ */
+ private void addNXT(Message response, int flags)
+ throws DNSSEC.DNSSECException, IOException {
+ Record nxtRecord = getNXTRecord(
+ response.getSectionArray(Section.QUESTION)[0]);
+ Zone zone = findBestZone(nxtRecord.getName());
+ addRecordCommand.exec(zone, nxtRecord);
+ RRset nxtRR = zone.findExactMatch(nxtRecord.getName(), Type.NXT);
+ addRRset(nxtRecord.getName(), response, nxtRR, Section.AUTHORITY, flags);
+
+ removeRecordCommand.exec(zone, nxtRecord);
+ }
+
+ /**
+ * Return an NXT record required to validate negative responses. If there is
+ * an issue returning the NXT record, a SOA record will be returned.
+ *
+ * @param query the query record.
+ * @return an NXT record.
+ */
+ private Record getNXTRecord(Record query) {
+ Record response = null;
+ SecureableZone zone = (SecureableZone) findBestZone(query.getName());
+ if (zone != null) {
+ response = zone.getNXTRecord(query, zone);
+ if (response == null) {
+ response = zone.getSOA();
+ }
+ }
+
+ return response;
+ }
+
+ /**
+ * Add the name server info to the authority section.
+ *
+ * @param response the response message.
+ * @param zone the DNS zone.
+ * @param flags the flags.
+ */
+ private void addNS(Message response, Zone zone, int flags) {
+ RRset nsRecords = zone.getNS();
+ addRRset(nsRecords.getName(), response, nsRecords,
+ Section.AUTHORITY, flags);
+ }
+
+ /**
+ * Add the provided record set to the response section specified.
+ *
+ * @param name the name associated with the record set.
+ * @param response the response message.
+ * @param rrset the record set.
+ * @param section the response section to which the record set will be added.
+ * @param flags the flags.
+ */
+ private void addRRset(Name name, Message response, RRset rrset, int section,
+ int flags) {
+ for (int s = 1; s <= section; s++) {
+ if (response.findRRset(name, rrset.getType(), s)) {
+ return;
+ }
+ }
+ if ((flags & FLAG_SIGONLY) == 0) {
+ Iterator it = rrset.rrs();
+ while (it.hasNext()) {
+ Record r = (Record) it.next();
+ if (r.getName().isWild() && !name.isWild()) {
+ r = r.withName(name);
+ }
+ response.addRecord(r, section);
+ }
+ }
+ if ((flags & (FLAG_SIGONLY | FLAG_DNSSECOK)) != 0) {
+ Iterator it = rrset.sigs();
+ while (it.hasNext()) {
+ Record r = (Record) it.next();
+ if (r.getName().isWild() && !name.isWild()) {
+ r = r.withName(name);
+ }
+ response.addRecord(r, section);
+ }
+ }
+ }
+
+ /**
+ * Perform a zone transfer.
+ *
+ * @param name the zone name.
+ * @param query the query.
+ * @param tsig the query signature.
+ * @param qtsig the signature record.
+ * @param s the connection socket.
+ * @return an error message if there is no matching zone
+ * or null due to error.
+ */
+ byte[] doAXFR(Name name, Message query, TSIG tsig, TSIGRecord qtsig,
+ Socket s) {
+ boolean first = true;
+ Zone zone = findBestZone(name);
+ if (zone == null) {
+ return errorMessage(query, Rcode.REFUSED);
+ }
+ Iterator it = zone.AXFR();
+ try {
+ DataOutputStream dataOut;
+ dataOut = new DataOutputStream(s.getOutputStream());
+ int id = query.getHeader().getID();
+ while (it.hasNext()) {
+ RRset rrset = (RRset) it.next();
+ Message response = new Message(id);
+ Header header = response.getHeader();
+ header.setFlag(Flags.QR);
+ header.setFlag(Flags.AA);
+ addRRset(rrset.getName(), response, rrset,
+ Section.ANSWER, FLAG_DNSSECOK);
+ if (tsig != null) {
+ tsig.applyStream(response, qtsig, first);
+ qtsig = response.getTSIG();
+ }
+ first = false;
+ byte[] out = response.toWire();
+ dataOut.writeShort(out.length);
+ dataOut.write(out);
+ }
+ } catch (IOException ex) {
+ System.out.println("AXFR failed");
+ }
+ try {
+ s.close();
+ } catch (IOException ex) {
+ }
+ return null;
+ }
+
+ /**
+ * Perform the registry operation (register or delete). This method will take
+ * the provided service record and either add or remove the DNS records
+ * indicated.
+ *
+ * @param path the ZK path for the service record.
+ * @param record the service record.
+ * @param command the registry command (REGISTER or DELETE).
+ * @throws IOException if the is an error performing registry operation.
+ */
+ private void op(String path, ServiceRecord record, RegistryCommand command)
+ throws IOException {
+ ServiceRecordProcessor processor;
+ try {
+ if (record.get(YarnRegistryAttributes.YARN_PERSISTENCE)
+ .equals(CONTAINER)) {
+ // container registration. the logic to identify and create the
+ // container entry needs to be enhanced/more accurate and associate to
+ // correct host
+ processor =
+ new ContainerServiceRecordProcessor(record, path, domainName, this);
+ } else {
+ processor =
+ new ApplicationServiceRecordProcessor(record, path, domainName,
+ this);
+ }
+ processor.manageDNSRecords(command);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ }
+
+ /**
+ * Return the username found in the ZK path.
+ *
+ * @param path the ZK path.
+ * @return the user name.
+ */
+ private String getUsername(String path) {
+ String user = "anonymous";
+ Matcher matcher = USER_NAME.matcher(path);
+ if (matcher.find()) {
+ user = matcher.group(1);
+ }
+ return user;
+ }
+
+ /**
+ * Register DNS records based on the provided service record.
+ *
+ * @param path the ZK path of the service record.
+ * @param record record providing DNS registration info.
+ * @throws IOException if registration causes an error.
+ */
+ @Override
+ public void register(String path, ServiceRecord record) throws IOException {
+ op(path, record, addRecordCommand);
+ }
+
+ /**
+ * Delete the DNS records generated by the provided service record.
+ *
+ * @param path the ZK path for the given record.
+ * @param record the service record
+ * @throws IOException if deletion causes and error.
+ */
+ @Override
+ public void delete(String path, ServiceRecord record) throws IOException {
+ op(path, record, removeRecordCommand);
+ }
+
+ /**
+ * An interface representing a registry associated function/command (see
+ * command pattern).
+ */
+ interface RegistryCommand {
+ void exec(Zone zone, Record record) throws IOException;
+
+ String getLogDescription();
+ }
+
+ /**
+ * The "add record" command.
+ */
+ private final RegistryCommand addRecordCommand = new RegistryCommand() {
+ @Override
+ public void exec(Zone zone, Record record) throws IOException {
+ if (zone != null) {
+ try (CloseableLock lock = writeLock.lock()) {
+ zone.addRecord(record);
+ LOG.info("Registered {}", record);
+ if (isDNSSECEnabled()) {
+ Calendar cal = Calendar.getInstance();
+ Date inception = cal.getTime();
+ cal.add(Calendar.YEAR, 1);
+ Date expiration = cal.getTime();
+ RRset rRset =
+ zone.findExactMatch(record.getName(), record.getType());
+ try {
+ DNSKEYRecord dnskeyRecord = dnsKeyRecs.get(zone.getOrigin());
+ RRSIGRecord rrsigRecord =
+ DNSSEC.sign(rRset, dnskeyRecord, privateKey,
+ inception, expiration);
+ LOG.info("Adding {}", rrsigRecord);
+ rRset.addRR(rrsigRecord);
+
+ //addDSRecord(zone, record.getName(), record.getDClass(),
+ // record.getTTL(), inception, expiration);
+
+ } catch (DNSSEC.DNSSECException e) {
+ throw new IOException(e);
+ }
+ }
+ }
+ } else {
+ LOG.warn("Unable to find zone matching record {}", record);
+ }
+ }
+
+ /**
+ * Add a DS record associated with the input name.
+ * @param zone the zone.
+ * @param name the record name.
+ * @param dClass the DNS class.
+ * @param dsTtl the ttl value.
+ * @param inception the time of inception of the record.
+ * @param expiration the expiry time of the record.
+ * @throws DNSSEC.DNSSECException if the addition of DS record fails.
+ */
+ private void addDSRecord(Zone zone,
+ Name name, int dClass, long dsTtl,
+ Date inception,
+ Date expiration) throws DNSSEC.DNSSECException {
+ RRset rRset;
+ RRSIGRecord rrsigRecord;
+
+ DNSKEYRecord dnskeyRecord = dnsKeyRecs.get(zone.getOrigin());
+ DSRecord dsRecord = new DSRecord(name, dClass,
+ dsTtl, DSRecord.Digest.SHA1,
+ dnskeyRecord);
+ zone.addRecord(dsRecord);
+ LOG.info("Adding {}", dsRecord);
+ rRset = zone.findExactMatch(dsRecord.getName(), dsRecord.getType());
+
+ rrsigRecord = DNSSEC.sign(rRset, dnskeyRecord, privateKey,
+ inception, expiration);
+ rRset.addRR(rrsigRecord);
+ }
+
+ @Override
+ public String getLogDescription() {
+ return "Registering ";
+ }
+ };
+
+ /**
+ * The "remove record" command.
+ */
+ private final RegistryCommand removeRecordCommand = new RegistryCommand() {
+ @Override
+ public void exec(Zone zone, Record record) throws IOException {
+ zone.removeRecord(record);
+ LOG.info("Removed {}", record);
+ if (isDNSSECEnabled()) {
+ RRset rRset = zone.findExactMatch(record.getName(), Type.DS);
+ if (rRset != null) {
+ zone.removeRecord(rRset.first());
+ }
+ }
+ }
+
+ @Override
+ public String getLogDescription() {
+ return "Deleting ";
+ }
+ };
+
+ /**
+ * An implementation allowing for obtaining and releasing a lock.
+ */
+ public static class CloseableLock implements AutoCloseable {
+ private Lock lock;
+
+ public CloseableLock(Lock lock) {
+ this.lock = lock;
+ }
+
+ public CloseableLock lock() {
+ lock.lock();
+ return this;
+ }
+
+ @Override
+ public void close() {
+ lock.unlock();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNSServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNSServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNSServer.java
new file mode 100644
index 0000000..faa5fe1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNSServer.java
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.registry.server.dns;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.registry.client.api.DNSOperationsFactory;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.impl.zk.PathListener;
+import org.apache.hadoop.registry.client.impl.zk.RegistryOperationsService;
+import org.apache.hadoop.registry.client.types.RegistryPathStatus;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.ShutdownHookManager;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+/**
+ * A server/service that starts and manages the lifecycle of a DNS registry
+ * instance.
+ */
+public class RegistryDNSServer extends CompositeService {
+
+
+ public static final int SHUTDOWN_HOOK_PRIORITY = 30;
+ private RegistryDNS registryDNS;
+ private RegistryOperationsService registryOperations;
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RegistryDNS.class);
+ private ConcurrentMap<String, ServiceRecord> pathToRecordMap;
+
+ /**
+ * Creates the DNS server.
+ * @param name the server name.
+ */
+ public RegistryDNSServer(String name) {
+ super(name);
+ }
+
+ /**
+ * Initializes the DNS server.
+ * @param conf the hadoop configuration instance.
+ * @throws Exception if service initialization fails.
+ */
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+
+ pathToRecordMap = new ConcurrentHashMap<>();
+
+ registryOperations = new RegistryOperationsService("RegistryDNSOperations");
+ addService(registryOperations);
+
+ // probably need to populate with existing apps?
+ registryDNS = (RegistryDNS) DNSOperationsFactory.createInstance(conf);
+ addService(registryDNS);
+
+ super.serviceInit(conf);
+ }
+
+ /**
+ * Starts the server.
+ * @throws Exception if service start fails.
+ */
+ @Override
+ protected void serviceStart() throws Exception {
+ super.serviceStart();
+ manageRegistryDNS();
+ }
+
+ /**
+ * Performs operations required to setup the DNS registry instance (e.g. sets
+ * up a path listener to react to service record creation/deletion and invoke
+ * the appropriate registry method).
+ */
+ private void manageRegistryDNS() {
+
+ try {
+ registryOperations.monitorRegistryEntries();
+ registryOperations.registerPathListener(new PathListener() {
+ private String registryRoot = getConfig().
+ get(RegistryConstants.KEY_REGISTRY_ZK_ROOT,
+ RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
+
+ @Override
+ public void nodeAdded(String path) throws IOException {
+ // get a listing of service records
+ String relativePath = getPathRelativeToRegistryRoot(path);
+ String child = RegistryPathUtils.lastPathEntry(path);
+ Map<String, RegistryPathStatus> map = new HashMap<>();
+ map.put(child, registryOperations.stat(relativePath));
+ Map<String, ServiceRecord> records =
+ RegistryUtils.extractServiceRecords(registryOperations,
+ getAdjustedParentPath(path),
+ map);
+ processServiceRecords(records, register);
+ pathToRecordMap.putAll(records);
+ }
+
+ private String getAdjustedParentPath(String path) {
+ Preconditions.checkNotNull(path);
+ String adjustedPath = null;
+ adjustedPath = getPathRelativeToRegistryRoot(path);
+ try {
+ return RegistryPathUtils.parentOf(adjustedPath);
+ } catch (PathNotFoundException e) {
+ // attempt to use passed in path
+ return path;
+ }
+ }
+
+ private String getPathRelativeToRegistryRoot(String path) {
+ String adjustedPath;
+ if (path.equals(registryRoot)) {
+ adjustedPath = "/";
+ } else {
+ adjustedPath = path.substring(registryRoot.length());
+ }
+ return adjustedPath;
+ }
+
+ @Override
+ public void nodeRemoved(String path) throws IOException {
+ ServiceRecord record = pathToRecordMap.remove(path.substring(
+ registryRoot.length()));
+ processServiceRecord(path, record, delete);
+ }
+
+ });
+
+ // create listener for record deletions
+
+ } catch (Exception e) {
+ LOG.warn("Unable to monitor the registry. DNS support disabled.", e);
+ }
+ }
+
+ /**
+ * A registry management command interface.
+ */
+ interface ManagementCommand {
+ void exec(String path, ServiceRecord record) throws IOException;
+ }
+
+ /**
+ * Performs registry service record registration.
+ */
+ private final ManagementCommand register = new ManagementCommand() {
+ @Override
+ public void exec(String path, ServiceRecord record) throws IOException {
+ if (record != null) {
+ LOG.info("Registering DNS records for {}", path);
+ registryDNS.register(path, record);
+ }
+ }
+ };
+
+ /**
+ * Performs registry service record deletion.
+ */
+ private ManagementCommand delete = new ManagementCommand() {
+ @Override
+ public void exec(String path, ServiceRecord record) throws IOException {
+ if (record != null) {
+ LOG.info("Deleting DNS records for {}", path);
+ registryDNS.delete(path, record);
+ }
+ }
+ };
+
+ /**
+ * iterates thru the supplied service records, executing the provided registry
+ * command.
+ * @param records the service records.
+ * @param command the registry command.
+ * @throws IOException
+ */
+ private void processServiceRecords(Map<String, ServiceRecord> records,
+ ManagementCommand command)
+ throws IOException {
+ for (Map.Entry<String, ServiceRecord> entry : records.entrySet()) {
+ processServiceRecord(entry.getKey(), entry.getValue(), command);
+ }
+ }
+
+ /**
+ * Process the service record, parsing the information and creating the
+ * required DNS records.
+ * @param path the service record path.
+ * @param record the record.
+ * @param command the registry command to execute.
+ * @throws IOException
+ */
+ private void processServiceRecord(String path, ServiceRecord record,
+ ManagementCommand command)
+ throws IOException {
+ command.exec(path, record);
+ }
+
+ /**
+ * Launch the server.
+ * @param args command line args.
+ * @return
+ */
+ static RegistryDNSServer launchDNSServer(String[] args) {
+ RegistryDNSServer dnsServer = null;
+
+ Thread
+ .setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
+ StringUtils.startupShutdownMessage(RegistryDNSServer.class, args,
+ LOG);
+ try {
+ dnsServer = new RegistryDNSServer("RegistryDNSServer");
+ ShutdownHookManager.get().addShutdownHook(
+ new CompositeService.CompositeServiceShutdownHook(dnsServer),
+ SHUTDOWN_HOOK_PRIORITY);
+ YarnConfiguration conf = new YarnConfiguration();
+ processCommandLine(args, conf);
+ new GenericOptionsParser(conf, args);
+ dnsServer.init(conf);
+ dnsServer.start();
+ } catch (Throwable t) {
+ LOG.error("Error starting Registry DNS Server", t);
+ ExitUtil.terminate(-1, "Error starting Registry DNS Server");
+ }
+ return dnsServer;
+ }
+
+ /**
+ * Process input command line arguments.
+ * @param args the command line argument array.
+ * @param conf the configuration.
+ */
+ private static void processCommandLine(String[] args,
+ YarnConfiguration conf) {
+ Options options = new Options();
+ options.addOption("p", "port", true,
+ "the server listening port (override)");
+
+ CommandLineParser parser = new BasicParser();
+ try {
+ CommandLine cmd = parser.parse(options, args);
+ if (cmd.hasOption("p")) {
+ conf.set(RegistryConstants.KEY_DNS_PORT, cmd.getOptionValue("p"));
+ }
+ } catch (ParseException e) {
+ LOG.error("Error parsing the command line options", e);
+ }
+ }
+
+ /**
+ * Lanches the server instance.
+ * @param args the command line args.
+ */
+ public static void main(String[] args) {
+ launchDNSServer(args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/SecureableZone.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/SecureableZone.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/SecureableZone.java
new file mode 100644
index 0000000..4b0a852
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/SecureableZone.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.registry.server.dns;
+
+import org.xbill.DNS.DClass;
+import org.xbill.DNS.NXTRecord;
+import org.xbill.DNS.Name;
+import org.xbill.DNS.RRset;
+import org.xbill.DNS.Record;
+import org.xbill.DNS.SetResponse;
+import org.xbill.DNS.Type;
+import org.xbill.DNS.Zone;
+import org.xbill.DNS.ZoneTransferException;
+import org.xbill.DNS.ZoneTransferIn;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+/**
+ * A zone implementation geared to support some DNSSEC functionality.
+ */
+public class SecureableZone extends Zone {
+ private List<Record> records;
+
+ /**
+ * Creates a Zone by doing the specified zone transfer.
+ * @param xfrin The incoming zone transfer to execute.
+ * @throws IOException if there is an error.
+ * @throws ZoneTransferException if there is an error.
+ */
+ public SecureableZone(ZoneTransferIn xfrin)
+ throws IOException, ZoneTransferException {
+ super(xfrin);
+ }
+
+ /**
+ * Creates a Zone by performing a zone transfer to the specified host.
+ * @param zone zone name.
+ * @param dclass the dclass
+ * @param remote the remote host.
+ * @throws IOException if there is an error.
+ * @throws ZoneTransferException if there is an error.
+ */
+ public SecureableZone(Name zone, int dclass, String remote)
+ throws IOException, ZoneTransferException {
+ super(zone, dclass, remote);
+ }
+
+ /**
+ * Creates a Zone from the records in the specified master file.
+ * @param zone The name of the zone.
+ * @param file The master file to read from.
+ * @throws IOException if there is an error.
+ */
+ public SecureableZone(Name zone, String file) throws IOException {
+ super(zone, file);
+ }
+
+ /**
+ * Creates a Zone from an array of records.
+ * @param zone The name of the zone.
+ * @param records The records to add to the zone.
+ * @throws IOException if there is an error.
+ */
+ public SecureableZone(Name zone, Record[] records)
+ throws IOException {
+ super(zone, records);
+ }
+
+ /**
+ * Adds a Record to the Zone.
+ * @param r The record to be added
+ * @see Record
+ */
+ @Override public void addRecord(Record r) {
+ if (records == null) {
+ records = new ArrayList<Record>();
+ }
+ super.addRecord(r);
+ records.add(r);
+ }
+
+ /**
+ * Removes a record from the Zone.
+ * @param r The record to be removed
+ * @see Record
+ */
+ @Override public void removeRecord(Record r) {
+ if (records == null) {
+ records = new ArrayList<Record>();
+ }
+ super.removeRecord(r);
+ records.remove(r);
+ }
+
+ /**
+ * Return a NXT record appropriate for the query.
+ * @param queryRecord the query record.
+ * @param zone the zone to search.
+ * @return the NXT record describing the insertion point.
+ */
+ @SuppressWarnings({"unchecked"})
+ public Record getNXTRecord(Record queryRecord, Zone zone) {
+ Collections.sort(records);
+
+ int index = Collections.binarySearch(records, queryRecord,
+ new Comparator<Record>() {
+ @Override public int compare(Record r1, Record r2) {
+ return r1.compareTo(r2);
+ }
+ });
+ if (index >= 0) {
+ return null;
+ }
+ index = -index - 1;
+ if (index >= records.size()) {
+ index = records.size() - 1;
+ }
+ Record base = records.get(index);
+ SetResponse sr = zone.findRecords(base.getName(), Type.ANY);
+ BitSet bitMap = new BitSet();
+ bitMap.set(Type.NXT);
+ RRset[] rRsets = sr.answers();
+ for (RRset rRset : rRsets) {
+ int typeCode = rRset.getType();
+ if (typeCode > 0 && typeCode < 128) {
+ bitMap.set(typeCode);
+ }
+ }
+ return new NXTRecord(base.getName(), DClass.IN, zone.getSOA().getMinimum(),
+ queryRecord.getName(), bitMap);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ServiceRecordProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ServiceRecordProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ServiceRecordProcessor.java
new file mode 100644
index 0000000..b67cc7d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ServiceRecordProcessor.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.registry.server.dns;
+
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+
+import java.io.IOException;
+
+/**
+ * Manage the processing of service records in order to create DNS records.
+ */
+public interface ServiceRecordProcessor {
+ /**
+ * Initialize the mapping between DNS record type and record information
+ * for the given service record.
+ * @param serviceRecord the registry service record.
+ * @throws Exception if encountering an error.
+ */
+ void initTypeToInfoMapping(ServiceRecord serviceRecord)
+ throws Exception;
+
+ /**
+ * Return the DNS record types valid for this processor.
+ * @return the array of DNS record types.
+ */
+ int[] getRecordTypes();
+
+ /**
+ * Manage the creation and registration of DNS records generated by parsing
+ * a service record.
+ * @param command the DNS registration command object (e.g. add_record,
+ * remove record)
+ * @throws IOException if the creation or registration generates an issue.
+ */
+ void manageDNSRecords(RegistryDNS.RegistryCommand command)
+ throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ZoneSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ZoneSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ZoneSelector.java
new file mode 100644
index 0000000..5043b85
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ZoneSelector.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.registry.server.dns;
+
+import org.xbill.DNS.Name;
+import org.xbill.DNS.Zone;
+
+/**
+ * A selector that returns the zone associated with a provided name.
+ */
+public interface ZoneSelector {
+ /**
+ * Finds the best matching zone given the provided name.
+ * @param name the record name for which a zone is requested.
+ * @return the matching zone.
+ */
+ Zone findBestZone(Name name);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/package-info.java
new file mode 100644
index 0000000..00d8c9db
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/package-info.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * DNS Server classes.
+ * <p>
+ * These classes are leveraged to create a DNS server that can provide the
+ * facilities necessary for YARN application and/or service discovery.
+ * </p>
+ */
+package org.apache.hadoop.registry.server.dns;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/51] [abbrv] hadoop git commit: YARN-5735. Make the service REST
API use the app timeout feature YARN-4205. Contributed by Jian He
Posted by ji...@apache.org.
YARN-5735. Make the service REST API use the app timeout feature YARN-4205. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/de70e984
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/de70e984
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/de70e984
Branch: refs/heads/yarn-native-services
Commit: de70e984ecea1c850a33f6bb8acdb4bc7922d968
Parents: 39e18c1
Author: Gour Saha <go...@apache.org>
Authored: Fri Oct 14 17:40:51 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../api/impl/ApplicationApiService.java | 10 ++++--
.../org/apache/slider/client/SliderClient.java | 33 ++++++++++----------
.../AbstractClusterBuildingActionArgs.java | 5 +++
.../slider/common/params/ActionThawArgs.java | 6 ++++
.../apache/slider/common/params/Arguments.java | 1 +
5 files changed, 36 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de70e984/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index 21cf113..73df4a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -347,7 +347,7 @@ public class ApplicationApiService implements ApplicationApi {
if (queueName != null && queueName.trim().length() > 0) {
createArgs.queue = queueName.trim();
}
-
+ createArgs.lifetime = application.getLifetime();
return invokeSliderClientRunnable(new SliderClientContextRunnable<String>() {
@Override
public String run(SliderClient sliderClient) throws YarnException,
@@ -1246,13 +1246,17 @@ public class ApplicationApiService implements ApplicationApi {
});
}
- private Response startSliderApplication(final String appName)
+ private Response startSliderApplication(final String appName, Application app)
throws IOException, YarnException, InterruptedException {
return invokeSliderClientRunnable(new SliderClientContextRunnable<Response>() {
@Override
public Response run(SliderClient sliderClient) throws YarnException,
IOException, InterruptedException {
ActionThawArgs thawArgs = new ActionThawArgs();
+ if (app.getLifetime() == null) {
+ app.setLifetime(DEFAULT_UNLIMITED_LIFETIME);
+ }
+ thawArgs.lifetime = app.getLifetime();
int returnCode = sliderClient.actionThaw(appName, thawArgs);
if (returnCode == 0) {
logger.info("Successfully started application {}", appName);
@@ -1344,7 +1348,7 @@ public class ApplicationApiService implements ApplicationApi {
try {
int livenessCheck = getSliderList(appName);
if (livenessCheck != 0) {
- return startSliderApplication(appName);
+ return startSliderApplication(appName, updateAppData);
} else {
logger.info("Application {} is already running", appName);
ApplicationStatus applicationStatus = new ApplicationStatus();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de70e984/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index fe4f1d2..2840c4b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.NodeReport;
@@ -734,7 +735,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
sliderFileSystem.getFileSystem().delete(clusterDirectory, true);
throw e;
}
- return startCluster(clustername, createArgs);
+ return startCluster(clustername, createArgs, createArgs.lifetime);
}
@Override
@@ -1960,14 +1961,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
*
* @param clustername name of the cluster.
* @param launchArgs launch arguments
+ * @param lifetime
* @return the exit code
* @throws YarnException
* @throws IOException
*/
- protected int startCluster(String clustername,
- LaunchArgsAccessor launchArgs) throws
- YarnException,
- IOException {
+ protected int startCluster(String clustername, LaunchArgsAccessor launchArgs,
+ long lifetime) throws YarnException, IOException {
Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(
clustername,
@@ -1975,7 +1975,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
LaunchedApplication launchedApplication =
launchApplication(clustername, clusterDirectory, instanceDefinition,
- serviceArgs.isDebug());
+ serviceArgs.isDebug(), lifetime);
if (launchArgs.getOutputFile() != null) {
// output file has been requested. Get the app report and serialize it
@@ -2044,9 +2044,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
protected AppMasterLauncher setupAppMasterLauncher(String clustername,
- Path clusterDirectory,
- AggregateConf instanceDefinition,
- boolean debugAM)
+ Path clusterDirectory, AggregateConf instanceDefinition, boolean debugAM,
+ long lifetime)
throws YarnException, IOException{
deployedClusterName = clustername;
validateClusterName(clustername);
@@ -2119,7 +2118,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
ApplicationId appId = amLauncher.getApplicationId();
// set the application name;
amLauncher.setKeepContainersOverRestarts(true);
-
+ // set lifetime in submission context;
+ Map<ApplicationTimeoutType, Long> appTimeout = new HashMap<>();
+ appTimeout.put(ApplicationTimeoutType.LIFETIME, lifetime);
+ amLauncher.submissionContext.setApplicationTimeouts(appTimeout);
int maxAppAttempts = config.getInt(KEY_AM_RESTART_LIMIT, 0);
amLauncher.setMaxAppAttempts(maxAppAttempts);
@@ -2383,20 +2385,19 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @param clusterDirectory cluster dir
* @param instanceDefinition the instance definition
* @param debugAM enable debug AM options
+ * @param lifetime
* @return the launched application
* @throws YarnException
* @throws IOException
*/
- public LaunchedApplication launchApplication(String clustername,
- Path clusterDirectory,
- AggregateConf instanceDefinition,
- boolean debugAM)
+ public LaunchedApplication launchApplication(String clustername, Path clusterDirectory,
+ AggregateConf instanceDefinition, boolean debugAM, long lifetime)
throws YarnException, IOException {
AppMasterLauncher amLauncher = setupAppMasterLauncher(clustername,
clusterDirectory,
instanceDefinition,
- debugAM);
+ debugAM, lifetime);
applicationId = amLauncher.getApplicationId();
log.info("Submitting application {}", applicationId);
@@ -3254,7 +3255,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
verifyNoLiveClusters(clustername, "Start");
//start the cluster
- return startCluster(clustername, thaw);
+ return startCluster(clustername, thaw, thaw.lifetime);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de70e984/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
index 2a5eedc..3cb75e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
@@ -102,6 +102,11 @@ public abstract class AbstractClusterBuildingActionArgs extends
description = "Queue to submit the application")
public String queue;
+ @Parameter(names = {ARG_LIFETIME},
+ description = "Life time of the application since application started at"
+ + " running state")
+ public long lifetime;
+
@ParametersDelegate
public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de70e984/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java
index b43a14e..2bd856f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionThawArgs.java
@@ -18,6 +18,7 @@
package org.apache.slider.common.params;
+import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
import com.beust.jcommander.ParametersDelegate;
@@ -43,6 +44,11 @@ public class ActionThawArgs extends AbstractActionArgs implements
@ParametersDelegate
LaunchArgsDelegate launchArgs = new LaunchArgsDelegate();
+ @Parameter(names = {ARG_LIFETIME},
+ description = "Life time of the application since application started at"
+ + " running state")
+ public long lifetime;
+
@Override
public String getRmAddress() {
return launchArgs.getRmAddress();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de70e984/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index aec4e26..cbf7e59 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -103,6 +103,7 @@ public interface Arguments {
String ARG_PRINCIPAL = "--principal";
String ARG_PROVIDER = "--provider";
String ARG_QUEUE = "--queue";
+ String ARG_LIFETIME = "--lifetime";
String ARG_REPLACE_PKG = "--replacepkg";
String ARG_RESOURCE = "--resource";
String ARG_RESOURCES = "--resources";
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/51] [abbrv] hadoop git commit: YARN-5909. Remove agent related
code in slider AM. Contributed by Jian He
Posted by ji...@apache.org.
YARN-5909. Remove agent related code in slider AM. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/340967d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/340967d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/340967d7
Branch: refs/heads/yarn-native-services
Commit: 340967d7202412452fcd91beef4dba4add6d8aef
Parents: 1805e36
Author: Jian He <ji...@apache.org>
Authored: Tue Nov 29 13:06:45 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
hadoop-project/pom.xml | 5 -
.../services/webapp/ApplicationApiWebApp.java | 2 +-
.../hadoop-yarn-slider-core/pom.xml | 23 +-
.../apache/slider/common/tools/SliderUtils.java | 1 -
.../providers/AbstractProviderService.java | 21 +-
.../slider/providers/ProviderService.java | 9 +-
.../providers/agent/AgentClientProvider.java | 713 -----
.../providers/agent/AgentLaunchParameter.java | 130 -
.../providers/agent/AgentProviderFactory.java | 47 -
.../providers/agent/AgentProviderService.java | 2850 ------------------
.../slider/providers/agent/AgentRoles.java | 38 -
.../slider/providers/agent/AgentUtils.java | 150 -
.../apache/slider/providers/agent/Command.java | 59 -
.../slider/providers/agent/CommandResult.java | 40 -
.../providers/agent/ComponentCommandOrder.java | 225 --
.../providers/agent/ComponentInstanceState.java | 340 ---
.../providers/agent/ComponentTagProvider.java | 127 -
.../slider/providers/agent/ContainerState.java | 41 -
.../providers/agent/HeartbeatMonitor.java | 130 -
.../apache/slider/providers/agent/State.java | 199 --
.../application/metadata/AbstractComponent.java | 80 -
.../metadata/AbstractMetainfoParser.java | 130 -
.../metadata/AbstractMetainfoSchema.java | 69 -
.../metadata/AddonPackageMetainfoParser.java | 53 -
.../agent/application/metadata/Application.java | 193 --
.../metadata/ApplicationPackage.java | 69 -
.../application/metadata/CommandOrder.java | 61 -
.../application/metadata/CommandScript.java | 72 -
.../agent/application/metadata/Component.java | 217 --
.../application/metadata/ComponentCommand.java | 85 -
.../application/metadata/ComponentExport.java | 54 -
.../metadata/ComponentsInAddonPackage.java | 26 -
.../agent/application/metadata/ConfigFile.java | 59 -
.../application/metadata/DefaultConfig.java | 39 -
.../metadata/DefaultConfigParser.java | 54 -
.../application/metadata/DockerContainer.java | 187 --
.../metadata/DockerContainerInputFile.java | 50 -
.../metadata/DockerContainerMount.java | 60 -
.../metadata/DockerContainerPort.java | 66 -
.../agent/application/metadata/Export.java | 61 -
.../agent/application/metadata/ExportGroup.java | 71 -
.../agent/application/metadata/Metainfo.java | 118 -
.../application/metadata/MetainfoParser.java | 97 -
.../agent/application/metadata/OSPackage.java | 51 -
.../agent/application/metadata/OSSpecific.java | 57 -
.../agent/application/metadata/Package.java | 60 -
.../application/metadata/PropertyInfo.java | 54 -
.../agent/application/metadata/Validate.java | 27 -
.../org/apache/slider/providers/agent/todo.md | 22 -
.../server/appmaster/SliderAppMaster.java | 140 +-
.../server/appmaster/web/AgentService.java | 37 -
.../slider/server/appmaster/web/WebAppApi.java | 7 -
.../server/appmaster/web/WebAppApiImpl.java | 6 -
.../appmaster/web/rest/AMWebServices.java | 1 -
.../web/rest/agent/AgentCommandType.java | 23 -
.../appmaster/web/rest/agent/AgentEnv.java | 376 ---
.../appmaster/web/rest/agent/AgentResource.java | 118 -
.../web/rest/agent/AgentRestOperations.java | 28 -
.../appmaster/web/rest/agent/AgentWebApp.java | 258 --
.../web/rest/agent/AgentWebServices.java | 40 -
.../appmaster/web/rest/agent/CommandReport.java | 207 --
.../web/rest/agent/ComponentStatus.java | 129 -
.../appmaster/web/rest/agent/DiskInfo.java | 128 -
.../web/rest/agent/ExecutionCommand.java | 310 --
.../appmaster/web/rest/agent/HeartBeat.java | 149 -
.../web/rest/agent/HeartBeatResponse.java | 147 -
.../appmaster/web/rest/agent/HostInfo.java | 398 ---
.../appmaster/web/rest/agent/HostStatus.java | 63 -
.../appmaster/web/rest/agent/Register.java | 193 --
.../web/rest/agent/RegistrationCommand.java | 43 -
.../web/rest/agent/RegistrationResponse.java | 133 -
.../web/rest/agent/RegistrationStatus.java | 22 -
.../appmaster/web/rest/agent/StatusCommand.java | 152 -
.../agent/TestAgentClientProvider.java | 77 -
.../agent/TestAgentLaunchParameter.java | 76 -
.../slider/providers/agent/TestAgentUtils.java | 94 -
.../agent/TestAppDefinitionPersister.java | 264 --
.../agent/TestComponentTagProvider.java | 115 -
.../slider/providers/agent/TestState.java | 33 -
.../application/metadata/TestConfigParser.java | 107 -
.../metadata/TestMetainfoParser.java | 177 --
.../publisher/TestAgentProviderService.java | 60 -
.../publisher/TestSliderProviderFactory.java | 40 -
83 files changed, 26 insertions(+), 11517 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 409dd93..d46bde0 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -921,11 +921,6 @@
<version>${jackson2.version}</version>
</dependency>
<dependency>
- <groupId>com.fasterxml.jackson.jaxrs</groupId>
- <artifactId>jackson-jaxrs-json-provider</artifactId>
- <version>${jackson2.version}</version>
- </dependency>
- <dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-cbor</artifactId>
<version>${jackson2.version}</version>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java
index 52a9de6..e1bddb5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.services.api.impl.ApplicationApiService;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
-import org.mortbay.jetty.webapp.Configuration;
+import org.eclipse.jetty.webapp.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index a2c67c0..66e9ee9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -225,7 +225,7 @@
<dependency>
<groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
+ <artifactId>javax.servlet-api</artifactId>
</dependency>
<dependency>
@@ -289,21 +289,6 @@
</dependency>
<dependency>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-sslengine</artifactId>
- </dependency>
-
- <dependency>
<groupId>javax.servlet.jsp</groupId>
<artifactId>jsp-api</artifactId>
<scope>runtime</scope>
@@ -315,12 +300,6 @@
</dependency>
<dependency>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-sslengine</artifactId>
- <version>6.1.26</version>
- </dependency>
-
- <dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.16</version>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 713cd02..b101d34 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -70,7 +70,6 @@ import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.ClasspathConstructor;
import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.slider.providers.agent.AgentKeys;
-import org.apache.slider.providers.agent.application.metadata.Component;
import org.apache.slider.server.services.utility.PatternValidator;
import org.apache.slider.server.services.workflow.ForkedProcessService;
import org.apache.zookeeper.server.util.KerberosUtil;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index 19fa07b..00fc606 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -19,17 +19,17 @@
package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
+import org.apache.hadoop.registry.client.types.AddressTypes;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
-import org.apache.hadoop.registry.client.types.AddressTypes;
-import org.apache.hadoop.registry.client.types.Endpoint;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.ConfigHelper;
@@ -44,7 +44,6 @@ import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
import org.apache.slider.server.services.workflow.ForkedProcessService;
import org.apache.slider.server.services.workflow.ServiceParent;
import org.apache.slider.server.services.workflow.WorkflowSequenceService;
@@ -77,7 +76,6 @@ public abstract class AbstractProviderService
private static final Logger log =
LoggerFactory.getLogger(AbstractProviderService.class);
protected StateAccessForProviders amState;
- protected AgentRestOperations restOps;
protected URL amWebAPI;
protected YarnRegistryViewForProviders yarnRegistry;
protected QueueAccess queueAccess;
@@ -127,18 +125,9 @@ public abstract class AbstractProviderService
}
@Override
- public AgentRestOperations getAgentRestOperations() {
- return restOps;
- }
-
- @Override
public void notifyContainerCompleted(ContainerId containerId) {
}
- public void setAgentRestOperations(AgentRestOperations agentRestOperations) {
- this.restOps = agentRestOperations;
- }
-
/**
* Load default Configuration
* @param confDir configuration directory
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index b62510a..4ca9326 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -20,10 +20,10 @@ package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.tools.SliderFileSystem;
@@ -37,7 +37,6 @@ import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions;
import org.apache.slider.server.appmaster.state.ContainerReleaseSelector;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
import java.io.File;
@@ -176,12 +175,6 @@ public interface ProviderService extends ProviderCore,
void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry);
/**
- * Returns the agent rest operations interface.
- * @return the interface if available, null otherwise.
- */
- AgentRestOperations getAgentRestOperations();
-
- /**
* Build up the endpoint details for this service
* @param details
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
deleted file mode 100644
index 7ca469f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ /dev/null
@@ -1,713 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import com.google.common.io.Files;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
-import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.slider.api.InternalKeys;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.client.ClientUtils;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.conf.MapOperations;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.AbstractLauncher;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.providers.AbstractClientProvider;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.providers.ProviderUtils;
-import org.apache.slider.providers.agent.application.metadata.Application;
-import org.apache.slider.providers.agent.application.metadata.Component;
-import org.apache.slider.providers.agent.application.metadata.ConfigFile;
-import org.apache.slider.providers.agent.application.metadata.Metainfo;
-import org.apache.slider.providers.agent.application.metadata.MetainfoParser;
-import org.apache.slider.providers.agent.application.metadata.OSPackage;
-import org.apache.slider.providers.agent.application.metadata.OSSpecific;
-import org.apache.slider.providers.agent.application.metadata.Package;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.Charset;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
-
-import static org.apache.slider.common.tools.SliderUtils.getApplicationDefinitionPath;
-
-/** This class implements the client-side aspects of the agent deployer */
-public class AgentClientProvider extends AbstractClientProvider
- implements AgentKeys, SliderKeys {
-
-
- protected static final Logger log =
- LoggerFactory.getLogger(AgentClientProvider.class);
- protected static final String NAME = "agent";
- private static final ProviderUtils providerUtils = new ProviderUtils(log);
- public static final String E_COULD_NOT_READ_METAINFO
- = "Not a valid app package. Could not read metainfo.";
-
- protected Map<String, Metainfo> metaInfoMap = new ConcurrentHashMap<String, Metainfo>();
-
- protected AgentClientProvider(Configuration conf) {
- super(conf);
- }
-
- @Override
- public String getName() {
- return NAME;
- }
-
- @Override
- public List<ProviderRole> getRoles() {
- return AgentRoles.getRoles();
- }
-
- @Override //Client
- public void preflightValidateClusterConfiguration(SliderFileSystem sliderFileSystem,
- String clustername,
- Configuration configuration,
- AggregateConf instanceDefinition,
- Path clusterDirPath,
- Path generatedConfDirPath,
- boolean secure) throws
- SliderException,
- IOException {
- super.preflightValidateClusterConfiguration(sliderFileSystem, clustername,
- configuration,
- instanceDefinition,
- clusterDirPath,
- generatedConfDirPath, secure);
-
- String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition
- .getAppConfOperations());
- Path appDefPath = new Path(appDef);
- sliderFileSystem.verifyFileExists(appDefPath);
-
- String agentConf = instanceDefinition.getAppConfOperations().
- getGlobalOptions().getOption(AGENT_CONF, "");
- if (StringUtils.isNotEmpty(agentConf)) {
- sliderFileSystem.verifyFileExists(new Path(agentConf));
- }
-
- String appHome = instanceDefinition.getAppConfOperations().
- getGlobalOptions().get(PACKAGE_PATH);
- if (SliderUtils.isUnset(appHome)) {
- String agentImage = instanceDefinition.getInternalOperations().
- get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
- sliderFileSystem.verifyFileExists(new Path(agentImage));
- }
- }
-
- @Override
- public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws
- SliderException {
- super.validateInstanceDefinition(instanceDefinition, fs);
- log.debug("Validating conf {}", instanceDefinition);
- ConfTreeOperations resources =
- instanceDefinition.getResourceOperations();
-
- providerUtils.validateNodeCount(instanceDefinition, ROLE_NODE,
- 0, -1);
-
- String appDef = null;
- try {
- // Validate the app definition
- appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition
- .getAppConfOperations());
- } catch (BadConfigException bce) {
- throw new BadConfigException("Application definition must be provided. " + bce.getMessage());
- }
-
- log.info("Validating app definition {}", appDef);
- String extension = appDef.substring(appDef.lastIndexOf(".") + 1, appDef.length());
- if (!"zip".equals(extension.toLowerCase(Locale.ENGLISH))) {
- throw new BadConfigException("App definition must be packaged as a .zip file. File provided is " + appDef);
- }
-
- Set<String> names = resources.getComponentNames();
- names.remove(COMPONENT_AM);
- Map<Integer, String> priorityMap = new HashMap<Integer, String>();
-
- for (String name : names) {
- try {
- // Validate the app definition
- appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition
- .getAppConfOperations(), name);
- } catch (BadConfigException bce) {
- throw new BadConfigException("Application definition must be provided. " + bce.getMessage());
- }
- Metainfo metaInfo = getMetainfo(fs, appDef);
-
- MapOperations component = resources.getMandatoryComponent(name);
-
- if (metaInfo != null) {
- Component componentDef = metaInfo.getApplicationComponent(
- AgentUtils.getMetainfoComponentName(name,
- instanceDefinition.getAppConfOperations()));
- if (componentDef == null) {
- throw new BadConfigException(
- "Component %s is not a member of application.", name);
- }
- }
-
- int priority =
- component.getMandatoryOptionInt(ResourceKeys.COMPONENT_PRIORITY);
- if (priority <= 0) {
- throw new BadConfigException("Component %s %s value out of range %d",
- name,
- ResourceKeys.COMPONENT_PRIORITY,
- priority);
- }
-
- String existing = priorityMap.get(priority);
- if (existing != null) {
- throw new BadConfigException(
- "Component %s has a %s value %d which duplicates that of %s",
- name,
- ResourceKeys.COMPONENT_PRIORITY,
- priority,
- existing);
- }
- priorityMap.put(priority, name);
-
- // fileSystem may be null for tests
- if (metaInfo != null) {
- Component componentDef = metaInfo.getApplicationComponent(
- AgentUtils.getMetainfoComponentName(name,
- instanceDefinition.getAppConfOperations()));
-
- // ensure that intance count is 0 for client components
- if ("CLIENT".equals(componentDef.getCategory())) {
- MapOperations componentConfig = resources.getMandatoryComponent(name);
- int count =
- componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES);
- if (count > 0) {
- throw new BadConfigException("Component %s is of type CLIENT and cannot be instantiated."
- + " Use \"slider client install ...\" command instead.",
- name);
- }
- } else {
- MapOperations componentConfig = resources.getMandatoryComponent(name);
- int count =
- componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES);
- int definedMinCount = componentDef.getMinInstanceCountInt();
- int definedMaxCount = componentDef.getMaxInstanceCountInt();
- if (count < definedMinCount || count > definedMaxCount) {
- throw new BadConfigException("Component %s, %s value %d out of range. "
- + "Expected minimum is %d and maximum is %d",
- name,
- ResourceKeys.COMPONENT_INSTANCES,
- count,
- definedMinCount,
- definedMaxCount);
- }
- }
- }
- }
- }
-
-
- @Override
- public void prepareAMAndConfigForLaunch(SliderFileSystem fileSystem,
- Configuration serviceConf,
- AbstractLauncher launcher,
- AggregateConf instanceDefinition,
- Path snapshotConfDirPath,
- Path generatedConfDirPath,
- Configuration clientConfExtras,
- String libdir,
- Path tempPath,
- boolean miniClusterTestRun) throws
- IOException,
- SliderException {
- String agentImage = instanceDefinition.getInternalOperations().
- get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
- if (SliderUtils.isUnset(agentImage)) {
- Path agentPath = new Path(tempPath.getParent(), PROVIDER_AGENT);
- log.info("Automatically uploading the agent tarball at {}", agentPath);
- fileSystem.getFileSystem().mkdirs(agentPath);
- if (ProviderUtils.addAgentTar(this, AGENT_TAR, fileSystem, agentPath)) {
- instanceDefinition.getInternalOperations().set(
- InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH,
- new Path(agentPath, AGENT_TAR).toUri());
- }
- }
- }
-
- @Override
- public Set<String> getApplicationTags(SliderFileSystem fileSystem,
- ConfTreeOperations appConf) throws SliderException {
- return getApplicationTags(fileSystem,
- getApplicationDefinitionPath(appConf));
- }
-
- public Set<String> getApplicationTags(SliderFileSystem fileSystem,
- String appDef) throws SliderException {
- Set<String> tags;
- Metainfo metaInfo = getMetainfo(fileSystem, appDef);
-
- if (metaInfo == null) {
- log.error("Error retrieving metainfo from {}", appDef);
- throw new SliderException("Error parsing metainfo file, possibly bad structure.");
- }
-
- Application application = metaInfo.getApplication();
- tags = new HashSet<String>();
- tags.add("Name: " + application.getName());
- tags.add("Version: " + application.getVersion());
- tags.add("Description: " + SliderUtils.truncate(application.getComment(), 80));
-
- return tags;
- }
-
- @Override
- public void processClientOperation(SliderFileSystem fileSystem,
- RegistryOperations rops,
- Configuration configuration,
- String operation,
- File clientInstallPath,
- File appPackage,
- JSONObject config,
- String name) throws SliderException {
- // create temp folder
- // create sub-folders app_pkg, agent_pkg, command
- File tmpDir = Files.createTempDir();
- log.info("Command is being executed at {}", tmpDir.getAbsolutePath());
- File appPkgDir = new File(tmpDir, "app_pkg");
- appPkgDir.mkdir();
-
- File agentPkgDir = new File(tmpDir, "agent_pkg");
- agentPkgDir.mkdir();
-
- File cmdDir = new File(tmpDir, "command");
- cmdDir.mkdir();
-
- Metainfo metaInfo = null;
- JSONObject defaultConfig = null;
- try {
- // expand app package into /app_pkg
- ZipInputStream zipInputStream = null;
- try {
- zipInputStream = new ZipInputStream(new FileInputStream(appPackage));
- {
- ZipEntry zipEntry = zipInputStream.getNextEntry();
- while (zipEntry != null) {
- log.info("Processing {}", zipEntry.getName());
- String filePath = appPkgDir + File.separator + zipEntry.getName();
- if (!zipEntry.isDirectory()) {
- log.info("Extracting file {}", filePath);
- extractFile(zipInputStream, filePath);
-
- if ("metainfo.xml".equals(zipEntry.getName())) {
- FileInputStream input = null;
- try {
- input = new FileInputStream(filePath);
- metaInfo = new MetainfoParser().fromXmlStream(input);
- } finally {
- IOUtils.closeStream(input);
- }
- } else if ("metainfo.json".equals(zipEntry.getName())) {
- FileInputStream input = null;
- try {
- input = new FileInputStream(filePath);
- metaInfo = new MetainfoParser().fromJsonStream(input);
- } finally {
- IOUtils.closeStream(input);
- }
- } else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) {
- try {
- defaultConfig = new JSONObject(FileUtils.readFileToString(new File(filePath), Charset.defaultCharset()));
- } catch (JSONException jex) {
- throw new SliderException("Unable to read default client config.", jex);
- }
- }
- } else {
- log.info("Creating dir {}", filePath);
- File dir = new File(filePath);
- dir.mkdir();
- }
- zipInputStream.closeEntry();
- zipEntry = zipInputStream.getNextEntry();
- }
- }
- } finally {
- zipInputStream.close();
- }
-
- if (metaInfo == null) {
- throw new BadConfigException(E_COULD_NOT_READ_METAINFO);
- }
-
- String clientScript = null;
- String clientComponent = null;
- for (Component component : metaInfo.getApplication().getComponents()) {
- if (component.getCategory().equals("CLIENT")) {
- clientComponent = component.getName();
- if (component.getCommandScript() != null) {
- clientScript = component.getCommandScript().getScript();
- }
- break;
- }
- }
-
- if (SliderUtils.isUnset(clientScript)) {
- log.info("Installing CLIENT without script");
- List<Package> packages = metaInfo.getApplication().getPackages();
- if (packages.size() > 0) {
- // retrieve package resources from HDFS and extract
- for (Package pkg : packages) {
- Path pkgPath = fileSystem.buildResourcePath(pkg.getName());
- if (!fileSystem.isFile(pkgPath) && name != null) {
- pkgPath = fileSystem.buildResourcePath(name, pkg.getName());
- }
- if (!fileSystem.isFile(pkgPath)) {
- throw new IOException("Package doesn't exist as a resource: " +
- pkg.getName());
- }
- if ("archive".equals(pkg.getType())) {
- File pkgFile = new File(tmpDir, pkg.getName());
- fileSystem.copyHdfsFileToLocal(pkgPath, pkgFile);
- expandTar(pkgFile, clientInstallPath);
- } else {
- File pkgFile = new File(clientInstallPath, pkg.getName());
- fileSystem.copyHdfsFileToLocal(pkgPath, pkgFile);
- }
- }
- } else {
- // extract tarball from app def
- for (OSSpecific osSpecific : metaInfo.getApplication()
- .getOSSpecifics()) {
- for (OSPackage pkg : osSpecific.getPackages()) {
- if ("tarball".equals(pkg.getType())) {
- File pkgFile = new File(appPkgDir, pkg.getName());
- expandTar(pkgFile, clientInstallPath);
- }
- }
- }
- }
- if (name == null) {
- log.warn("Conf files not being generated because no app name was " +
- "provided");
- return;
- }
- File confInstallDir;
- String clientRoot = null;
- if (config != null) {
- try {
- clientRoot = config.getJSONObject("global")
- .getString(APP_CLIENT_ROOT);
- } catch (JSONException e) {
- log.info("Couldn't read {} from provided client config, falling " +
- "back on default", APP_CLIENT_ROOT);
- }
- }
- if (clientRoot == null && defaultConfig != null) {
- try {
- clientRoot = defaultConfig.getJSONObject("global")
- .getString(APP_CLIENT_ROOT);
- } catch (JSONException e) {
- log.info("Couldn't read {} from default client config, using {}",
- APP_CLIENT_ROOT, clientInstallPath);
- }
- }
- if (clientRoot == null) {
- confInstallDir = clientInstallPath;
- } else {
- confInstallDir = new File(new File(clientInstallPath, clientRoot), "conf");
- if (!confInstallDir.exists()) {
- confInstallDir.mkdirs();
- }
- }
- String user = RegistryUtils.currentUser();
- for (ConfigFile configFile : metaInfo.getComponentConfigFiles(clientComponent)) {
- retrieveConfigFile(rops, configuration, configFile, name, user,
- confInstallDir);
- }
- } else {
- log.info("Installing CLIENT using script {}", clientScript);
- expandAgentTar(agentPkgDir);
-
- JSONObject commandJson = getCommandJson(defaultConfig, config, metaInfo, clientInstallPath, name);
- FileWriter file = new FileWriter(new File(cmdDir, "command.json"));
- try {
- file.write(commandJson.toString());
-
- } catch (IOException e) {
- log.error("Couldn't write command.json to file");
- } finally {
- file.flush();
- file.close();
- }
-
- runCommand(appPkgDir, agentPkgDir, cmdDir, clientScript);
- }
-
- } catch (IOException ioex) {
- log.warn("Error while executing INSTALL command {}", ioex.getMessage());
- throw new SliderException("INSTALL client failed.");
- }
- }
-
- protected void runCommand(
- File appPkgDir,
- File agentPkgDir,
- File cmdDir,
- String clientScript) throws SliderException {
- int exitCode = 0;
- Exception exp = null;
- try {
- String clientScriptPath = appPkgDir.getAbsolutePath() + File.separator + "package" +
- File.separator + clientScript;
- List<String> command = Arrays.asList(PYTHON_EXE,
- "-S",
- clientScriptPath,
- "INSTALL",
- cmdDir.getAbsolutePath() + File.separator + "command.json",
- appPkgDir.getAbsolutePath() + File.separator + "package",
- cmdDir.getAbsolutePath() + File.separator + "command-out.json",
- "DEBUG");
- ProcessBuilder pb = new ProcessBuilder(command);
- log.info("Command: " + StringUtils.join(pb.command(), " "));
- pb.environment().put(PYTHONPATH,
- agentPkgDir.getAbsolutePath()
- + File.separator + "slider-agent" + File.pathSeparator
- + agentPkgDir.getAbsolutePath()
- + File.separator + "slider-agent/jinja2");
- log.info("{}={}", PYTHONPATH, pb.environment().get(PYTHONPATH));
-
- Process proc = pb.start();
- InputStream stderr = proc.getErrorStream();
- InputStream stdout = proc.getInputStream();
- BufferedReader stdOutReader = new BufferedReader(new InputStreamReader(stdout));
- BufferedReader stdErrReader = new BufferedReader(new InputStreamReader(stderr));
-
- proc.waitFor();
-
- String line;
- while ((line = stdOutReader.readLine()) != null) {
- log.info("Stdout: " + line);
- }
- while ((line = stdErrReader.readLine()) != null) {
- log.info("Stderr: " + line);
- }
-
- exitCode = proc.exitValue();
- log.info("Exit value is {}", exitCode);
- } catch (IOException e) {
- exp = e;
- } catch (InterruptedException e) {
- exp = e;
- }
-
- if (exitCode != 0) {
- throw new SliderException("INSTALL client failed with exit code " + exitCode);
- }
-
- if (exp != null) {
- log.error("Error while executing INSTALL command {}. Stack trace {}",
- exp.getMessage(),
- ExceptionUtils.getStackTrace(exp));
- throw new SliderException("INSTALL client failed.", exp);
- }
- }
-
- private void expandAgentTar(File agentPkgDir) throws IOException {
- String libDirProp = SliderUtils.getLibDir();
- File tarFile = new File(libDirProp, AGENT_TAR);
- expandTar(tarFile, agentPkgDir);
- }
-
- private void expandTar(File tarFile, File destDir) throws IOException {
- log.info("Expanding tar {} to {}", tarFile, destDir);
- TarArchiveInputStream tarIn = new TarArchiveInputStream(
- new GzipCompressorInputStream(
- new BufferedInputStream(
- new FileInputStream(tarFile)
- )
- )
- );
- try {
- TarArchiveEntry tarEntry = tarIn.getNextTarEntry();
- while (tarEntry != null) {
- File destPath = new File(destDir, tarEntry.getName());
- File parent = destPath.getParentFile();
- if (!parent.exists()) {
- parent.mkdirs();
- }
- if (tarEntry.isDirectory()) {
- destPath.mkdirs();
- } else {
- byte[] byteToRead = new byte[1024];
- BufferedOutputStream buffOut =
- new BufferedOutputStream(new FileOutputStream(destPath));
- try {
- int len;
- while ((len = tarIn.read(byteToRead)) != -1) {
- buffOut.write(byteToRead, 0, len);
- }
- } finally {
- buffOut.close();
- }
- }
- if ((tarEntry.getMode() & 0100) != 0) {
- destPath.setExecutable(true);
- }
- tarEntry = tarIn.getNextTarEntry();
- }
- } finally {
- tarIn.close();
- }
- }
-
- private void retrieveConfigFile(RegistryOperations rops,
- Configuration configuration, ConfigFile configFile, String name,
- String user, File destDir) throws IOException, SliderException {
- log.info("Retrieving config {} to {}", configFile.getDictionaryName(),
- destDir);
- PublishedConfiguration published = ClientUtils.getConfigFromRegistry(rops,
- configuration, configFile.getDictionaryName(), name, user, true);
- ClientUtils.saveOrReturnConfig(published, configFile.getType(),
- destDir, configFile.getFileName());
- }
-
- protected JSONObject getCommandJson(JSONObject defaultConfig,
- JSONObject inputConfig,
- Metainfo metainfo,
- File clientInstallPath,
- String name) throws SliderException {
- try {
- JSONObject pkgList = new JSONObject();
- pkgList.put(PACKAGE_LIST,
- AgentProviderService.getPackageListFromApplication(metainfo.getApplication()));
- JSONObject obj = new JSONObject();
- obj.put("hostLevelParams", pkgList);
-
- String user = RegistryUtils.currentUser();
- JSONObject configuration = new JSONObject();
- JSONObject global = new JSONObject();
- global.put("app_install_dir", clientInstallPath.getAbsolutePath());
- global.put("app_user", user);
- if (name != null) {
- global.put("app_name", name);
- }
-
- if (defaultConfig != null) {
- readConfigEntries(defaultConfig, clientInstallPath, global, name, user);
- }
- if (inputConfig != null) {
- readConfigEntries(inputConfig, clientInstallPath, global, name, user);
- }
-
- configuration.put("global", global);
- obj.put("configurations", configuration);
- return obj;
- } catch (JSONException jex) {
- log.warn("Error while executing INSTALL command {}", jex.getMessage());
- throw new SliderException("INSTALL client failed.");
- }
- }
-
- private void readConfigEntries(JSONObject inpConfig,
- File clientInstallPath,
- JSONObject globalConfig,
- String name, String user)
- throws JSONException {
- JSONObject globalSection = inpConfig.getJSONObject("global");
- Iterator it = globalSection.keys();
- while (it.hasNext()) {
- String key = (String) it.next();
- String value = globalSection.getString(key);
- if (SliderUtils.isSet(value)) {
- value = value.replace("{app_install_dir}", clientInstallPath.getAbsolutePath());
- value = value.replace("{app_user}", user);
- if (name != null) {
- value = value.replace("{app_name}", name);
- }
- }
- if (globalConfig.has(key)) {
- // last one wins
- globalConfig.remove(key);
- }
- globalConfig.put(key, value);
- }
- }
-
- private void extractFile(ZipInputStream zipInputStream, String filePath) throws IOException {
- BufferedOutputStream output = new BufferedOutputStream(new FileOutputStream(filePath));
- try {
- byte[] bytesRead = new byte[4096];
- int read = 0;
- while ((read = zipInputStream.read(bytesRead)) != -1) {
- output.write(bytesRead, 0, read);
- }
- } finally {
- output.close();
- }
- }
-
- private Metainfo getMetainfo(SliderFileSystem fs, String appDef) {
- Metainfo metaInfo = metaInfoMap.get(appDef);
- if (fs != null && metaInfo == null) {
- try {
- metaInfo = AgentUtils.getApplicationMetainfo(fs, appDef, false);
- metaInfoMap.put(appDef, metaInfo);
- } catch (IOException ioe) {
- // Ignore missing metainfo file for now
- log.info("Missing metainfo {}", ioe.getMessage());
- } catch (BadConfigException bce) {
- log.info("Bad Configuration {}", bce.getMessage());
- }
- }
- return metaInfo;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java
deleted file mode 100644
index 18c6374..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.regex.Pattern;
-
-class AgentLaunchParameter {
- public static final Logger log =
- LoggerFactory.getLogger(AgentLaunchParameter.class);
- private static final String DEFAULT_PARAMETER = "";
- private static final String ANY_COMPONENT = "ANY";
- private static final String NONE_VALUE = "NONE";
- private final Map<String, CommandTracker> launchParameterTracker;
-
- public AgentLaunchParameter(String parameters) {
- launchParameterTracker = parseExpectedLaunchParameters(parameters);
- }
-
- /**
- * Get command for the component type
- *
- * @param componentGroup
- *
- * @return
- */
- public String getNextLaunchParameter(String componentGroup) {
- if (launchParameterTracker != null) {
- if (launchParameterTracker.containsKey(componentGroup)
- || launchParameterTracker.containsKey(ANY_COMPONENT)) {
- synchronized (this) {
- CommandTracker indexTracker = null;
- if (launchParameterTracker.containsKey(componentGroup)) {
- indexTracker = launchParameterTracker.get(componentGroup);
- } else {
- indexTracker = launchParameterTracker.get(ANY_COMPONENT);
- }
-
- return indexTracker.getNextCommand();
- }
- }
- }
-
- return DEFAULT_PARAMETER;
- }
-
- /**
- * Parse launch parameters of the form ANY:PARAM_FOR_FIRST:PARAM_FOR_SECOND:...:PARAM_FOR_REST|HBASE_MASTER:...
- *
- * E.g. ANY:DO_NOT_REGISTER:DO_NOT_HEARTBEAT:NONE For any container, first one gets DO_NOT_REGISTER second one gets
- * DO_NOT_HEARTBEAT, then all of the rest get nothing
- *
- * E.g. HBASE_MASTER:FAIL_AFTER_START:NONE For HBASE_MASTER, first one gets FAIL_AFTER_START then "" for all
- *
- * @param launchParameters
- *
- * @return
- */
- Map<String, CommandTracker> parseExpectedLaunchParameters(String launchParameters) {
- Map<String, CommandTracker> trackers = null;
- if (launchParameters != null && launchParameters.length() > 0) {
- String[] componentSpecificParameters = launchParameters.split(Pattern.quote("|"));
- for (String componentSpecificParameter : componentSpecificParameters) {
- if (componentSpecificParameter.length() != 0) {
- String[] parameters = componentSpecificParameter.split(Pattern.quote(":"));
-
- if (parameters.length > 1 && parameters[0].length() > 0) {
-
- for (int index = 1; index < parameters.length; index++) {
- if (parameters[index].equals(NONE_VALUE)) {
- parameters[index] = DEFAULT_PARAMETER;
- }
- }
-
- if (trackers == null) {
- trackers = new HashMap<String, CommandTracker>(10);
- }
- String componentName = parameters[0];
- CommandTracker tracker = new CommandTracker(Arrays.copyOfRange(parameters, 1, parameters.length));
- trackers.put(componentName, tracker);
- }
- }
- }
- }
-
- return trackers;
- }
-
- class CommandTracker {
- private final int maxIndex;
- private final String[] launchCommands;
- private int currentIndex;
-
- CommandTracker(String[] launchCommands) {
- this.currentIndex = 0;
- this.maxIndex = launchCommands.length - 1;
- this.launchCommands = launchCommands;
- }
-
- String getNextCommand() {
- String retVal = launchCommands[currentIndex];
- if (currentIndex != maxIndex) {
- currentIndex++;
- }
-
- return retVal;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderFactory.java
deleted file mode 100644
index d5ca749..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderFactory.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.slider.providers.AbstractClientProvider;
-import org.apache.slider.providers.ProviderService;
-import org.apache.slider.providers.SliderProviderFactory;
-
-public class AgentProviderFactory extends SliderProviderFactory {
-
- public static final String CLASSNAME =
- "org.apache.slider.providers.agent.AgentProviderFactory";
-
- public AgentProviderFactory() {
- }
-
- public AgentProviderFactory(Configuration conf) {
- super(conf);
- }
-
- @Override
- public AbstractClientProvider createClientProvider() {
- return new AgentClientProvider(getConf());
- }
-
- @Override
- public ProviderService createServerProvider() {
- return new AgentProviderService();
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/51] [abbrv] hadoop git commit: YARN-5796. Convert enums values in
service code to upper case and special handling of an error. Contributed by
Gour Saha
Posted by ji...@apache.org.
YARN-5796. Convert enums values in service code to upper case and special handling of an error. Contributed by Gour Saha
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a79a0704
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a79a0704
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a79a0704
Branch: refs/heads/yarn-native-services
Commit: a79a070451751e90807dfa6f4ee9757163ccb2b2
Parents: 6c857ea
Author: Jian He <ji...@apache.org>
Authored: Tue Nov 1 11:00:11 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../yarn/services/api/impl/ApplicationApiService.java | 4 ++++
.../apache/hadoop/yarn/services/resource/Artifact.java | 2 +-
.../apache/hadoop/yarn/services/resource/ConfigFile.java | 4 ++--
.../hadoop/yarn/services/resource/ReadinessCheck.java | 2 +-
.../hadoop/yarn/services/utils/RestApiErrorMessages.java | 10 +++++++---
5 files changed, 15 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a79a0704/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index cf43ac2..37bd134 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -172,6 +172,10 @@ public class ApplicationApiService implements ApplicationApi {
applicationStatus.setDiagnostics(ERROR_APPLICATION_IN_USE);
return Response.status(Status.BAD_REQUEST).entity(applicationStatus)
.build();
+ } else if (se.getExitCode() == SliderExitCodes.EXIT_INSTANCE_EXISTS) {
+ applicationStatus.setDiagnostics(ERROR_APPLICATION_INSTANCE_EXISTS);
+ return Response.status(Status.BAD_REQUEST).entity(applicationStatus)
+ .build();
} else {
applicationStatus.setDiagnostics(se.getMessage());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a79a0704/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
index 9ac2bc7..87fcf89 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
@@ -40,7 +40,7 @@ public class Artifact implements Serializable {
private String id = null;
public enum TypeEnum {
- DOCKER("docker"), TARBALL("tarball"), APPLICATION("application");
+ DOCKER("DOCKER"), TARBALL("TARBALL"), APPLICATION("APPLICATION");
private String value;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a79a0704/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
index 3ced153..01d976f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
@@ -43,8 +43,8 @@ public class ConfigFile implements Serializable {
private static final long serialVersionUID = -7009402089417704612L;
public enum TypeEnum {
- XML("xml"), PROPERTIES("properties"), JSON("json"), YAML("yaml"), TEMPLATE(
- "template"), ENV("env"), HADOOP_XML("hadoop_xml");
+ XML("XML"), PROPERTIES("PROPERTIES"), JSON("JSON"), YAML("YAML"), TEMPLATE(
+ "TEMPLATE"), ENV("ENV"), HADOOP_XML("HADOOP_XML");
private String value;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a79a0704/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ReadinessCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ReadinessCheck.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ReadinessCheck.java
index f549746..26cd39a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ReadinessCheck.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ReadinessCheck.java
@@ -39,7 +39,7 @@ public class ReadinessCheck implements Serializable {
private static final long serialVersionUID = -3836839816887186801L;
public enum TypeEnum {
- HTTP("http");
+ HTTP("HTTP");
private String value;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a79a0704/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java
index 685f85a..2d739a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java
@@ -26,6 +26,11 @@ public interface RestApiErrorMessages {
String ERROR_APPLICATION_NOT_RUNNING = "Application not running";
String ERROR_APPLICATION_DOES_NOT_EXIST = "Application not found";
+ String ERROR_APPLICATION_IN_USE = "Application already exists in started"
+ + " state";
+ String ERROR_APPLICATION_INSTANCE_EXISTS = "Application already exists in"
+ + " stopped/failed state (either restart with PUT or destroy with DELETE"
+ + " before creating a new one)";
String ERROR_SUFFIX_FOR_COMPONENT =
" for component %s (nor at the global level)";
@@ -66,7 +71,6 @@ public interface RestApiErrorMessages {
String ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET =
"Resource profile is not " + "supported yet. Please specify cpus/memory.";
- String ERROR_APPLICATION_IN_USE = "Application name is already in use";
String ERROR_NULL_ARTIFACT_ID =
"Artifact Id can not be null if artifact type is none";
String ERROR_ABSENT_NUM_OF_INSTANCE =
@@ -74,6 +78,6 @@ public interface RestApiErrorMessages {
String ERROR_ABSENT_LAUNCH_COMMAND =
"launch command should appear if type is slider-zip or none";
- String ERROR_QUICKLINKS_FOR_COMP_INVALID =
- "Quicklinks specified at component level, needs corresponding values set at application level";
+ String ERROR_QUICKLINKS_FOR_COMP_INVALID = "Quicklinks specified at"
+ + " component level, needs corresponding values set at application level";
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/51] [abbrv] hadoop git commit: YARN-5740. Add a new field in
Slider status output - lifetime (remaining). Contributed by Jian He
Posted by ji...@apache.org.
YARN-5740. Add a new field in Slider status output - lifetime (remaining). Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b39605fb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b39605fb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b39605fb
Branch: refs/heads/yarn-native-services
Commit: b39605fbd154e00db0935b1f92e2c1420101ddc5
Parents: cef1ce4
Author: Gour Saha <go...@apache.org>
Authored: Fri Dec 16 12:03:51 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../api/impl/ApplicationApiService.java | 82 ++++++++++++--------
.../org/apache/slider/client/SliderClient.java | 62 +++++++++++----
.../slider/common/params/ActionStatusArgs.java | 4 +
3 files changed, 103 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b39605fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index c4f5d43..b11da2c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -52,6 +52,8 @@ import org.apache.commons.lang.SerializationUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.services.api.ApplicationApi;
@@ -771,7 +773,7 @@ public class ApplicationApiService implements ApplicationApi {
// Get all applications in a specific state - lighter projection. For full
// detail, call getApplication on a specific app.
- Set<String> applications;
+ Set<ApplicationReport> applications;
try {
if (StringUtils.isNotEmpty(state)) {
ApplicationStatus appStatus = new ApplicationStatus();
@@ -793,13 +795,12 @@ public class ApplicationApiService implements ApplicationApi {
Set<Application> apps = new HashSet<Application>();
if (applications.size() > 0) {
try {
- for (String app : applications) {
+ for (ApplicationReport app : applications) {
Application application = new Application();
- // TODO: Need to get lifetime, launch-time and privileged container
- // status from YARN
- application.setLifetime(null);
- application.setLaunchTime(new Date());
- application.setName(app);
+ application.setLifetime(app.getApplicationTimeouts().get(
+ ApplicationTimeoutType.LIFETIME).getRemainingTime());
+ application.setLaunchTime(new Date(app.getStartTime()));
+ application.setName(app.getName());
// Containers not required, setting to null to avoid empty list
application.setContainers(null);
apps.add(application);
@@ -930,9 +931,7 @@ public class ApplicationApiService implements ApplicationApi {
app.setLaunchTime(appStatus.get("createTime") == null ? null
: new Date(appStatus.get("createTime").getAsLong()));
- // lifetime - set it to unlimited for now
- // TODO: Once YARN-3813 and YARN-4205 are available - get it from YARN
- app.setLifetime(DEFAULT_UNLIMITED_LIFETIME);
+ app.setLifetime(queryLifetime(appName));
// Quicklinks
Map<String, String> appQuicklinks = new HashMap<>();
@@ -1062,6 +1061,24 @@ public class ApplicationApiService implements ApplicationApi {
return object.get(key) == null ? null : object.get(key).getAsJsonObject();
}
+ private long queryLifetime(String appName) {
+ try {
+ return invokeSliderClientRunnable(
+ new SliderClientContextRunnable<Long>() {
+ @Override
+ public Long run(SliderClient sliderClient)
+ throws YarnException, IOException, InterruptedException {
+ ApplicationReport report = sliderClient.findInstance(appName);
+ return report.getApplicationTimeouts()
+ .get(ApplicationTimeoutType.LIFETIME).getRemainingTime();
+ }
+ });
+ } catch (Exception e) {
+ logger.error("Error when querying lifetime for " + appName, e);
+ return DEFAULT_UNLIMITED_LIFETIME;
+ }
+ }
+
private JsonObject getSliderApplicationStatus(final String appName)
throws IOException, YarnException, InterruptedException {
@@ -1142,36 +1159,37 @@ public class ApplicationApiService implements ApplicationApi {
});
}
- private Set<String> getSliderApplications(final String state)
+ private Set<ApplicationReport> getSliderApplications(final String state)
throws IOException, YarnException, InterruptedException {
return getSliderApplications(false, state);
}
- private Set<String> getSliderApplications(final boolean liveOnly)
+ private Set<ApplicationReport> getSliderApplications(final boolean liveOnly)
throws IOException, YarnException, InterruptedException {
return getSliderApplications(liveOnly, null);
}
- private Set<String> getSliderApplications(final boolean liveOnly,
- final String state) throws IOException, YarnException,
- InterruptedException {
- return invokeSliderClientRunnable(new SliderClientContextRunnable<Set<String>>() {
- @Override
- public Set<String> run(SliderClient sliderClient) throws YarnException,
- IOException, InterruptedException {
- Set<String> apps;
- ActionListArgs listArgs = new ActionListArgs();
- if (liveOnly) {
- apps = sliderClient.getApplicationList(null);
- } else if (StringUtils.isNotEmpty(state)) {
- listArgs.state = state;
- apps = sliderClient.getApplicationList(null, listArgs);
- } else {
- apps = sliderClient.getApplicationList(null, listArgs);
- }
- return apps;
- }
- });
+ private Set<ApplicationReport> getSliderApplications(final boolean liveOnly,
+ final String state)
+ throws IOException, YarnException, InterruptedException {
+ return invokeSliderClientRunnable(
+ new SliderClientContextRunnable<Set<ApplicationReport>>() {
+ @Override
+ public Set<ApplicationReport> run(SliderClient sliderClient)
+ throws YarnException, IOException, InterruptedException {
+ Set<ApplicationReport> apps;
+ ActionListArgs listArgs = new ActionListArgs();
+ if (liveOnly) {
+ apps = sliderClient.getApplicationList(null);
+ } else if (StringUtils.isNotEmpty(state)) {
+ listArgs.state = state;
+ apps = sliderClient.getApplicationList(null, listArgs);
+ } else {
+ apps = sliderClient.getApplicationList(null, listArgs);
+ }
+ return apps;
+ }
+ });
}
@DELETE
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b39605fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index ef45d10..1c126ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -178,6 +179,7 @@ import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.ByteArrayOutputStream;
import java.io.Console;
import java.io.File;
import java.io.FileNotFoundException;
@@ -185,6 +187,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
+import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.StringWriter;
@@ -2706,11 +2709,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
@Override
public int actionList(String clustername, ActionListArgs args)
throws IOException, YarnException {
- Set<String> appInstances = getApplicationList(clustername, args);
- // getApplicationList never returns null
- return !appInstances.isEmpty() ? EXIT_SUCCESS
- : ((appInstances.isEmpty() && isUnset(clustername)) ? EXIT_SUCCESS
- : EXIT_FALSE);
+ Set<ApplicationReport> appInstances = getApplicationList(clustername, args);
+ if (!appInstances.isEmpty()) {
+ return EXIT_SUCCESS;
+ } else {
+ return EXIT_FALSE;
+ }
}
/**
@@ -2723,8 +2727,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws IOException
* @throws YarnException
*/
- public Set<String> getApplicationList(String clustername) throws IOException,
- YarnException {
+ public Set<ApplicationReport> getApplicationList(String clustername)
+ throws IOException, YarnException {
ActionListArgs args = new ActionListArgs();
args.live = true;
return getApplicationList(clustername, args);
@@ -2743,8 +2747,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws UnknownApplicationInstanceException
* if a specific instance was named but it was not found
*/
- public Set<String> getApplicationList(String clustername, ActionListArgs args)
- throws IOException, YarnException {
+ public Set<ApplicationReport> getApplicationList(String clustername,
+ ActionListArgs args) throws IOException, YarnException {
if (args.help) {
actionHelp(ACTION_LIST);
// the above call throws an exception so the return is not really required
@@ -2830,13 +2834,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
// at this point there is either the entire list or a stripped down instance
- Set<String> listedInstances = new HashSet<String>();
+ Set<ApplicationReport> listedInstances = new HashSet<ApplicationReport>();
for (String name : persistentInstances.keySet()) {
ApplicationReport report = reportMap.get(name);
if (!listOnlyInState || report != null) {
// list the details if all were requested, or the filtering contained
// a report
- listedInstances.add(name);
+ listedInstances.add(report);
// containers will be non-null when only one instance is requested
String details = instanceDetailsToString(name, report,
containers, version, components, verbose);
@@ -3055,7 +3059,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws YarnException YARN issues
* @throws IOException IO problems
*/
- private ApplicationReport findInstance(String appname)
+ public ApplicationReport findInstance(String appname)
throws YarnException, IOException {
return yarnAppListClient.findInstance(appname);
}
@@ -3106,6 +3110,11 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
@VisibleForTesting
public int actionStatus(String clustername, ActionStatusArgs statusArgs)
throws YarnException, IOException {
+ if (statusArgs.lifetime) {
+ queryAndPrintLifetime(clustername);
+ return EXIT_SUCCESS;
+ }
+
ClusterDescription status = verifyAndGetClusterDescription(clustername);
String outfile = statusArgs.getOutput();
if (outfile == null) {
@@ -3122,6 +3131,32 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return verifyAndGetClusterDescription(clustername).toJsonString();
}
+ private void queryAndPrintLifetime(String appName)
+ throws YarnException, IOException {
+ ApplicationReport appReport = findInstance(appName);
+ if (appReport == null) {
+ throw new YarnException("No application found for " + appName);
+ }
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter timeoutStr =
+ new PrintWriter(new OutputStreamWriter(baos, Charset.forName("UTF-8")));
+ try {
+ ApplicationTimeout lifetime = appReport.getApplicationTimeouts()
+ .get(ApplicationTimeoutType.LIFETIME);
+ if (lifetime.getRemainingTime() == -1L) {
+ timeoutStr.append(appName + " has no lifetime configured.");
+ } else {
+ timeoutStr.append("\t" + ApplicationTimeoutType.LIFETIME);
+ timeoutStr.print(" expires at : " + lifetime.getExpiryTime());
+ timeoutStr.println(
+ ".\tRemaining Time : " + lifetime.getRemainingTime() + " seconds");
+ }
+ System.out.println(baos.toString("UTF-8"));
+ } finally {
+ timeoutStr.close();
+ }
+ }
+
private ClusterDescription verifyAndGetClusterDescription(String clustername)
throws YarnException, IOException {
verifyBindingsDefined();
@@ -3547,7 +3582,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws IOException
*/
@VisibleForTesting
- public List<ApplicationReport> getApplications() throws YarnException, IOException {
+ public List<ApplicationReport> getApplications()
+ throws YarnException, IOException {
return yarnClient.getApplications();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b39605fb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java
index 00178df..6fbd96d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java
@@ -35,6 +35,10 @@ public class ActionStatusArgs extends AbstractActionArgs {
description = "Output file for the status information")
public String output;
+ @Parameter(names = {ARG_LIFETIME},
+ description = "Lifetime of the application from the time of request")
+ public boolean lifetime;
+
public String getOutput() {
return output;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/51] [abbrv] hadoop git commit: YARN-5909. Remove agent related
code in slider AM. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
deleted file mode 100644
index 281895a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import org.apache.slider.providers.ProviderRole;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class AgentRoles {
-
- /**
- * List of roles Agent provider does not have any roles by default. All roles are read from the application
- * specification.
- */
- protected static final List<ProviderRole> ROLES =
- new ArrayList<ProviderRole>();
-
- public static List<ProviderRole> getRoles() {
- return ROLES;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
deleted file mode 100644
index 23e05a3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.providers.agent.application.metadata.AbstractMetainfoParser;
-import org.apache.slider.providers.agent.application.metadata.AddonPackageMetainfoParser;
-import org.apache.slider.providers.agent.application.metadata.DefaultConfig;
-import org.apache.slider.providers.agent.application.metadata.DefaultConfigParser;
-import org.apache.slider.providers.agent.application.metadata.Metainfo;
-import org.apache.slider.providers.agent.application.metadata.MetainfoParser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-
-import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
-
-/**
- *
- */
-public class AgentUtils {
- private static final Logger log = LoggerFactory.getLogger(AgentUtils.class);
-
- public static Metainfo getApplicationMetainfoFromSummaryFile(
- SliderFileSystem fileSystem, String metainfoPath, boolean metainfoForAddon) {
- FileSystem fs = fileSystem.getFileSystem();
- Path appPathXML = new Path(metainfoPath + ".metainfo.xml");
- Path appPathJson = new Path(metainfoPath + ".metainfo.json");
- Path appPathUsed = null;
- try {
- FSDataInputStream appStream = null;
- if (fs.exists(appPathXML)) {
- appPathUsed = appPathXML;
- appStream = fs.open(appPathXML);
- return parseMetainfo(appStream, metainfoForAddon, "xml");
- } else if (fs.exists(appPathJson)) {
- appPathUsed = appPathJson;
- appStream = fs.open(appPathJson);
- return parseMetainfo(appStream, metainfoForAddon, "json");
- }
- } catch (IOException e) {
- log.info("Failed to get metainfo from summary file {} - {}", appPathUsed,
- e.getMessage());
- log.debug("Failed to get metainfo", e);
- }
- return null;
- }
-
- public static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
- String metainfoPath, boolean metainfoForAddon) throws IOException,
- BadConfigException {
- log.info("Reading metainfo at {}", metainfoPath);
- Metainfo metainfo = getApplicationMetainfoFromSummaryFile(fileSystem,
- metainfoPath, metainfoForAddon);
- if (metainfo != null) {
- log.info("Got metainfo from summary file");
- return metainfo;
- }
-
- FileSystem fs = fileSystem.getFileSystem();
- Path appPath = new Path(metainfoPath);
-
- InputStream metainfoJsonStream = SliderUtils.getApplicationResourceInputStream(
- fs, appPath, "metainfo.json");
- if (metainfoJsonStream == null) {
- InputStream metainfoXMLStream = SliderUtils.getApplicationResourceInputStream(
- fs, appPath, "metainfo.xml");
- if (metainfoXMLStream != null) {
- metainfo = parseMetainfo(metainfoXMLStream, metainfoForAddon, "xml");
- }
- } else {
- metainfo = parseMetainfo(metainfoJsonStream, metainfoForAddon, "json");
- }
-
- if (metainfo == null) {
- log.error("metainfo is unavailable at {}.", metainfoPath);
- throw new FileNotFoundException("metainfo.xml/json is required in app package. " +
- appPath);
- }
- return metainfo;
- }
-
- private static Metainfo parseMetainfo(InputStream stream,
- boolean metainfoForAddon, String type) throws IOException {
- AbstractMetainfoParser metainfoParser = null;
- if (metainfoForAddon) {
- metainfoParser = new AddonPackageMetainfoParser();
- } else {
- metainfoParser = new MetainfoParser();
- }
- if (type.equals("xml")) {
- return metainfoParser.fromXmlStream(stream);
- } else if (type.equals("json")) {
- return metainfoParser.fromJsonStream(stream);
- }
- return null;
- }
-
- static DefaultConfig getDefaultConfig(SliderFileSystem fileSystem,
- String appDef, String configFileName)
- throws IOException {
- // this is the path inside the zip file
- String fileToRead = "configuration/" + configFileName;
- log.info("Reading default config file {} at {}", fileToRead, appDef);
- InputStream configStream = SliderUtils.getApplicationResourceInputStream(
- fileSystem.getFileSystem(), new Path(appDef), fileToRead);
- if (configStream == null) {
- log.error("{} is unavailable at {}.", fileToRead, appDef);
- throw new IOException("Expected config file " + fileToRead + " is not available.");
- }
-
- return new DefaultConfigParser().parse(configStream);
- }
-
- static String getMetainfoComponentName(String roleGroup,
- ConfTreeOperations appConf) throws BadConfigException {
- String prefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, null);
- if (prefix == null) {
- return roleGroup;
- }
- if (!roleGroup.startsWith(prefix)) {
- throw new BadConfigException("Component " + roleGroup + " doesn't start" +
- " with prefix " + prefix);
- }
- return roleGroup.substring(prefix.length());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/Command.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/Command.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/Command.java
deleted file mode 100644
index 647cb86..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/Command.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-/** The states a component instance can be. */
-public enum Command {
- NOP, // do nothing
- INSTALL, // Install the component
- INSTALL_ADDON, // Install add on packages if any
- START, // Start the component
- STOP, // Stop the component
- UPGRADE, // The component will undergo upgrade
- TERMINATE; // Send terminate signal to agent
-
- public static Command getCommand(String commandVal) {
- if (commandVal.equals(Command.START.toString())) {
- return Command.START;
- }
- if (commandVal.equals(Command.INSTALL.toString())) {
- return Command.INSTALL;
- }
- if (commandVal.equals(Command.STOP.toString())) {
- return Command.STOP;
- }
- if (commandVal.equals(Command.UPGRADE.toString())) {
- return Command.UPGRADE;
- }
- if (commandVal.equals(Command.TERMINATE.toString())) {
- return Command.TERMINATE;
- }
-
- return Command.NOP;
- }
-
- public static String transform(Command command, boolean isUpgrade) {
- switch (command) {
- case STOP:
- return isUpgrade ? "UPGRADE_STOP" : command.name();
- default:
- return command.name();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/CommandResult.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/CommandResult.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/CommandResult.java
deleted file mode 100644
index 35d9116..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/CommandResult.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-/** Command results. */
-public enum CommandResult {
- IN_PROGRESS, // Command is in progress
- COMPLETED, // Command has successfully completed
- FAILED; // Command has failed
-
- public static CommandResult getCommandResult(String commandResVal) {
- if (commandResVal.equals(CommandResult.COMPLETED.toString())) {
- return CommandResult.COMPLETED;
- }
- if (commandResVal.equals(CommandResult.FAILED.toString())) {
- return CommandResult.FAILED;
- }
- if (commandResVal.equals(CommandResult.IN_PROGRESS.toString())) {
- return CommandResult.IN_PROGRESS;
- }
-
- throw new IllegalArgumentException("Unrecognized value " + commandResVal);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
deleted file mode 100644
index 4abac7a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.providers.agent.application.metadata.CommandOrder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
-
-/**
- * Stores the command dependency order for all components in a service. <commandOrder>
- * <command>SUPERVISOR-START</command> <requires>NIMBUS-STARTED</requires> </commandOrder> Means, SUPERVISOR START
- * requires NIMBUS to be STARTED
- */
-public class ComponentCommandOrder {
- public static final Logger log =
- LoggerFactory.getLogger(ComponentCommandOrder.class);
- private static char SPLIT_CHAR = '-';
- Map<Command, Map<String, List<ComponentState>>> dependencies =
- new HashMap<Command, Map<String, List<ComponentState>>>();
- Map<String, Collection<String>> prefixRoleMap = new HashMap<>();
- Map<String, String> rolePrefixMap = new HashMap<>();
-
- public ComponentCommandOrder() {}
-
- public ComponentCommandOrder(List<CommandOrder> commandOrders,
- ConfTreeOperations resources) {
- mergeCommandOrders(commandOrders, resources);
- }
-
- void mergeCommandOrders(List<CommandOrder> commandOrders,
- ConfTreeOperations resources) {
- for (String component : resources.getComponentNames()) {
- String prefix = SliderUtils.trimPrefix(
- resources.getComponentOpt(component, ROLE_PREFIX, null));
- if (prefix != null) {
- rolePrefixMap.put(component, prefix);
- if (!prefixRoleMap.containsKey(prefix)) {
- prefixRoleMap.put(prefix, new HashSet<String>());
- }
- prefixRoleMap.get(prefix).add(component);
- }
- }
- if (commandOrders != null && commandOrders.size() > 0) {
- for (CommandOrder commandOrder : commandOrders) {
- ComponentCommand componentCmd = getComponentCommand(
- commandOrder.getCommand(), resources);
- String requires = commandOrder.getRequires();
- List<ComponentState> requiredStates = parseRequiredStates(requires,
- resources);
- if (requiredStates.size() > 0) {
- Map<String, List<ComponentState>> compDep = dependencies.get(componentCmd.command);
- if (compDep == null) {
- compDep = new HashMap<>();
- dependencies.put(componentCmd.command, compDep);
- }
-
- List<ComponentState> requirements = compDep.get(componentCmd.componentName);
- if (requirements == null) {
- requirements = new ArrayList<>();
- compDep.put(componentCmd.componentName, requirements);
- }
-
- requirements.addAll(requiredStates);
- }
- }
- }
- }
-
- private List<ComponentState> parseRequiredStates(String requires,
- ConfTreeOperations resources) {
- if (requires == null || requires.length() < 2) {
- throw new IllegalArgumentException("Input cannot be null and must contain component and state.");
- }
-
- String[] componentStates = requires.split(",");
- List<ComponentState> retList = new ArrayList<ComponentState>();
- for (String componentStateStr : componentStates) {
- retList.add(getComponentState(componentStateStr, resources));
- }
-
- return retList;
- }
-
- private ComponentCommand getComponentCommand(String compCmdStr,
- ConfTreeOperations resources) {
- if (compCmdStr == null || compCmdStr.trim().length() < 2) {
- throw new IllegalArgumentException("Input cannot be null and must contain component and command.");
- }
-
- compCmdStr = compCmdStr.trim();
- int splitIndex = compCmdStr.lastIndexOf(SPLIT_CHAR);
- if (splitIndex == -1 || splitIndex == 0 || splitIndex == compCmdStr.length() - 1) {
- throw new IllegalArgumentException("Input does not appear to be well-formed.");
- }
- String compStr = compCmdStr.substring(0, splitIndex);
- String cmdStr = compCmdStr.substring(splitIndex + 1);
-
- if (resources.getComponent(compStr) == null && !prefixRoleMap.containsKey(compStr)) {
- throw new IllegalArgumentException("Component " + compStr + " specified" +
- " in command order does not exist");
- }
-
- Command cmd = Command.valueOf(cmdStr);
-
- if (cmd != Command.START) {
- throw new IllegalArgumentException("Dependency order can only be specified for START.");
- }
- return new ComponentCommand(compStr, cmd);
- }
-
- private ComponentState getComponentState(String compStStr,
- ConfTreeOperations resources) {
- if (compStStr == null || compStStr.trim().length() < 2) {
- throw new IllegalArgumentException("Input cannot be null.");
- }
-
- compStStr = compStStr.trim();
- int splitIndex = compStStr.lastIndexOf(SPLIT_CHAR);
- if (splitIndex == -1 || splitIndex == 0 || splitIndex == compStStr.length() - 1) {
- throw new IllegalArgumentException("Input does not appear to be well-formed.");
- }
- String compStr = compStStr.substring(0, splitIndex);
- String stateStr = compStStr.substring(splitIndex + 1);
-
- if (resources.getComponent(compStr) == null && !prefixRoleMap.containsKey(compStr)) {
- throw new IllegalArgumentException("Component " + compStr + " specified" +
- " in command order does not exist");
- }
-
- State state = State.valueOf(stateStr);
- if (state != State.STARTED && state != State.INSTALLED) {
- throw new IllegalArgumentException("Dependency order can only be specified against STARTED/INSTALLED.");
- }
- return new ComponentState(compStr, state);
- }
-
- // dependency is still on component level, but not package level
- // so use component name to check dependency, not component-package
- public boolean canExecute(String component, Command command, Collection<ComponentInstanceState> currentStates) {
- if (!dependencies.containsKey(command)) {
- return true;
- }
- List<ComponentState> required = new ArrayList<>();
- if (dependencies.get(command).containsKey(component)) {
- required.addAll(dependencies.get(command).get(component));
- }
- String prefix = rolePrefixMap.get(component);
- if (prefix != null && dependencies.get(command).containsKey(prefix)) {
- required.addAll(dependencies.get(command).get(prefix));
- }
-
- for (ComponentState stateToMatch : required) {
- for (ComponentInstanceState currState : currentStates) {
- log.debug("Checking schedule {} {} against dependency {} is {}",
- component, command, currState.getComponentName(), currState.getState());
- if (currState.getComponentName().equals(stateToMatch.componentName) ||
- (prefixRoleMap.containsKey(stateToMatch.componentName) &&
- prefixRoleMap.get(stateToMatch.componentName).contains(currState.getComponentName()))) {
- if (currState.getState() != stateToMatch.state) {
- if (stateToMatch.state == State.STARTED) {
- log.info("Cannot schedule {} {} as dependency {} is {}",
- component, command, currState.getComponentName(), currState.getState());
- return false;
- } else {
- //state is INSTALLED
- if (currState.getState() != State.STARTING && currState.getState() != State.STARTED) {
- log.info("Cannot schedule {} {} as dependency {} is {}",
- component, command, currState.getComponentName(), currState.getState());
- return false;
- }
- }
- }
- }
- }
- }
- return true;
- }
-
- static class ComponentState {
- public String componentName;
- public State state;
-
- public ComponentState(String componentName, State state) {
- this.componentName = componentName;
- this.state = state;
- }
- }
-
- static class ComponentCommand {
- public String componentName;
- public Command command;
-
- public ComponentCommand(String componentName, Command command) {
- this.componentName = componentName;
- this.command = command;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
deleted file mode 100644
index 6ee0ebb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
+++ /dev/null
@@ -1,340 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import java.util.Map;
-import java.util.TreeMap;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.providers.agent.application.metadata.Component;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/** This class implements a simple state machine for component instances. */
-public class ComponentInstanceState {
- public static final Logger log =
- LoggerFactory.getLogger(ComponentInstanceState.class);
- private static int MAX_FAILURE_TOLERATED = 3;
- private static String INVALID_TRANSITION_ERROR =
- "Result %s for command %s is not expected for component %s in state %s.";
-
- private final String componentName;
- private final ContainerId containerId;
- private final String containerIdAsString;
- private final String applicationId;
- private State state = State.INIT;
- private State targetState = State.STARTED;
- private int failuresSeen = 0;
- private Boolean configReported = false;
- private long lastHeartbeat = 0;
- private String ip;
- private String hostname;
- private ContainerState containerState;
-
- private Map<String, State> pkgStatuses;
- private String nextPkgToInstall;
-
- private boolean stopInitiated;
-
- public ComponentInstanceState(String componentName,
- ContainerId containerId,
- String applicationId) {
- this(componentName, containerId, applicationId,
- new TreeMap<String, State>());
- }
-
- public ComponentInstanceState(String componentName,
- ContainerId containerId,
- String applicationId, Map<String, State> pkgStatuses) {
- this.componentName = componentName;
- this.containerId = containerId;
- this.containerIdAsString = containerId.toString();
- this.applicationId = applicationId;
- this.containerState = ContainerState.INIT;
- this.lastHeartbeat = System.currentTimeMillis();
- this.pkgStatuses = pkgStatuses;
- }
-
- public String getComponentName() {
- return componentName;
- }
-
- public Boolean getConfigReported() {
- return configReported;
- }
-
- public void setConfigReported(Boolean configReported) {
- this.configReported = configReported;
- }
-
- public ContainerState getContainerState() {
- return containerState;
- }
-
- public void setContainerState(ContainerState containerState) {
- this.containerState = containerState;
- }
-
- public long getLastHeartbeat() {
- return lastHeartbeat;
- }
-
- /**
- * Update the heartbeat, and change container state
- * to mark as healthy if appropriate
- * @param heartbeatTime last time the heartbeat was seen
- * @return the current container state
- */
- public ContainerState heartbeat(long heartbeatTime) {
- this.lastHeartbeat = heartbeatTime;
- if(containerState == ContainerState.UNHEALTHY ||
- containerState == ContainerState.INIT) {
- containerState = ContainerState.HEALTHY;
- }
- return containerState;
- }
-
-
- public ContainerId getContainerId() {
- return containerId;
- }
-
- public void commandIssued(Command command) {
- commandIssued(command, false);
- }
-
- public void commandIssued(Command command, boolean isInUpgradeMode) {
- Command expected = getNextCommand(isInUpgradeMode);
- if (expected != command) {
- throw new IllegalArgumentException("Command " + command + " is not allowed in state " + state);
- }
- if (expected == Command.INSTALL_ADDON) {
- // for add on packages, the pkg must be nextPkgToInstall
- State currentState = pkgStatuses.get(nextPkgToInstall);
- log.debug("Command issued: component: {} is in {}", componentName,
- currentState);
- State nextState = currentState.getNextState(command);
- pkgStatuses.put(nextPkgToInstall, nextState);
- log.debug("Command issued: component: {} is now in {}", componentName,
- nextState);
- } else {
- // for master package
- state = state.getNextState(command);
- }
- }
-
- public void applyCommandResult(CommandResult result, Command command,
- String pkg) {
- // if the heartbeat is for a package
- // update that package's state in the component status
- // and don't bother with the master pkg
- if (StringUtils.isNotEmpty(pkg)
- && !Component.MASTER_PACKAGE_NAME.equals(pkg)) {
- log.debug("This result is for component: {} pkg: {}", componentName, pkg);
- State previousPkgState = pkgStatuses.get(pkg);
- log.debug("Currently component: {} pkg: {} is in state: {}",
- componentName, pkg, previousPkgState.toString());
- State nextPkgState = previousPkgState.getNextState(result);
- pkgStatuses.put(pkg, nextPkgState);
- log.debug("Component: {} pkg: {} next state: {}", componentName, pkg,
- nextPkgState);
- } else {
- log.debug("This result is for component: {} master package",
- componentName);
- applyCommandResult(result, command);
- }
- }
-
- public void applyCommandResult(CommandResult result, Command command) {
- if (!this.state.couldHaveIssued(command)) {
- throw new IllegalStateException("Invalid command " + command + " for state " + this.state);
- }
-
- try {
- if (result == CommandResult.FAILED) {
- failuresSeen++;
- } else if (result == CommandResult.COMPLETED) {
- failuresSeen = 0;
- }
- state = state.getNextState(result);
- } catch (IllegalArgumentException e) {
- String message = String.format(INVALID_TRANSITION_ERROR,
- result.toString(),
- command.toString(),
- componentName,
- state.toString());
- log.warn(message);
- throw new IllegalStateException(message);
- }
- }
-
- public boolean hasPendingCommand() {
- if (state.canIssueCommands() &&
- state != targetState &&
- failuresSeen < MAX_FAILURE_TOLERATED) {
- return true;
- }
-
- return false;
- }
-
- public Command getNextCommand() {
- return getNextCommand(false);
- }
-
- public Command getNextCommand(boolean isInUpgradeMode) {
- if (!hasPendingCommand()) {
- nextPkgToInstall = null;
- return Command.NOP;
- }
-
- log.debug("In getNextCommand, checking for component: {} ", componentName);
- // if the master pkg is just installed, check if any add on pkg need to be
- // installed
- nextPkgToInstall = null;
- if (state == State.INSTALLED) {
- for (Map.Entry<String, State> pkgStatus : pkgStatuses.entrySet()) {
- String pkg = pkgStatus.getKey();
- State pkgState = pkgStatus.getValue();
- log.debug("In getNextCommand, pkg: {} is in {}", pkg, pkgState);
- if (pkgState == State.INSTALLING) {
- // first check if any pkg is install in progress, if so, wait
- // so we don't need to do anything, just return NOP
- log.debug("In getNextCommand, pkg: {} we are issuing NOP", pkg);
- nextPkgToInstall = pkg;
- return Command.NOP;
- } else if (pkgState == State.INIT) {
- // temporarily storing pkg here
- // in case no pkg in 'installing' state
- // will return the package to install
- nextPkgToInstall = pkg;
- }
- }
- // when we reach here, no pkg is in 'installing' state
- if (nextPkgToInstall != null) {
- // nextPkgToInstall != null means some pkg is in INIT state
- // issue 'install' to the pkg we have stored in nextPkgToInstall
- log.debug("In getNextCommand, pkg: {} we are issuing install addon",
- nextPkgToInstall);
- return Command.INSTALL_ADDON;
- }
- }
- return this.state.getSupportedCommand(isInUpgradeMode, stopInitiated);
- }
-
- public State getState() {
- return state;
- }
-
- @VisibleForTesting
- protected void setState(State state) {
- this.state = state;
- }
-
- public State getTargetState() {
- return targetState;
- }
-
- public void setTargetState(State targetState) {
- this.targetState = targetState;
- }
-
- public String getNextPkgToInstall() {
- return nextPkgToInstall;
- }
-
- public boolean isStopInitiated() {
- return stopInitiated;
- }
-
- public void setStopInitiated(boolean stopInitiated) {
- this.stopInitiated = stopInitiated;
- }
-
- @Override
- public int hashCode() {
- int hashCode = 1;
-
- hashCode = hashCode ^ (componentName != null ? componentName.hashCode() : 0);
- hashCode = hashCode ^ (containerIdAsString != null ? containerIdAsString.hashCode() : 0);
- hashCode = hashCode ^ (applicationId != null ? applicationId.hashCode() : 0);
- return hashCode;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
-
- if (o == null || getClass() != o.getClass()) return false;
-
- ComponentInstanceState that = (ComponentInstanceState) o;
-
- if (this.componentName != null ?
- !this.componentName.equals(that.componentName) : this.componentName != null) {
- return false;
- }
-
- if (this.containerIdAsString != null ?
- !this.containerIdAsString.equals(that.containerIdAsString) : this.containerIdAsString != null) {
- return false;
- }
-
- if (this.applicationId != null ?
- !this.applicationId.equals(that.applicationId) : this.applicationId != null) {
- return false;
- }
-
- return true;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("ComponentInstanceState{");
- sb.append("containerIdAsString='").append(containerIdAsString).append('\'');
- sb.append(", state=").append(state);
- sb.append(", failuresSeen=").append(failuresSeen);
- sb.append(", lastHeartbeat=").append(lastHeartbeat);
- sb.append(", containerState=").append(containerState);
- sb.append(", componentName='").append(componentName).append('\'');
- sb.append(", ip=").append(ip);
- sb.append(", hostname='").append(hostname).append('\'');
- sb.append('}');
- return sb.toString();
- }
-
- public String getIp() {
- return ip;
- }
-
- public void setIp(String ip) {
- this.ip = ip;
- }
-
- public String getHostname() {
- return hostname;
- }
-
- public void setHostname(String hostname) {
- this.hostname = hostname;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentTagProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentTagProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentTagProvider.java
deleted file mode 100644
index 68f63fa..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ComponentTagProvider.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent;
-
-import org.apache.slider.common.tools.SliderUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.concurrent.ConcurrentHashMap;
-
-/** A simple tag provider that attempts to associate tags from 1-N to all container of a component */
-public class ComponentTagProvider {
- private static final Logger log = LoggerFactory.getLogger(ComponentTagProvider.class);
- private static String FREE = "free";
- private final ConcurrentHashMap<String, ConcurrentHashMap<String, String>> allTags;
-
- public ComponentTagProvider() {
- allTags = new ConcurrentHashMap<String, ConcurrentHashMap<String, String>>();
- }
-
- /**
- * Record an assigned tag to a container
- *
- * @param component
- * @param containerId
- * @param tag
- */
- public void recordAssignedTag(String component, String containerId, String tag) {
- if (SliderUtils.isSet(component) && SliderUtils.isSet(containerId)) {
- Integer key = null;
- try {
- key = Integer.valueOf(tag);
- } catch (NumberFormatException nfe) {
- //ignore
- }
- if (key != null && key > 0) {
- ConcurrentHashMap<String, String> compTags = getComponentSpecificTags(component);
- synchronized (compTags) {
- for (int index = 1; index <= key.intValue(); index++) {
- String tempKey = new Integer(index).toString();
- if (!compTags.containsKey(tempKey)) {
- compTags.put(tempKey, FREE);
- }
- }
- compTags.put(key.toString(), containerId);
- }
- }
- }
- }
-
- /**
- * Get a tag for container
- *
- * @param component
- * @param containerId
- *
- * @return
- */
- public String getTag(String component, String containerId) {
- if (SliderUtils.isSet(component) && SliderUtils.isSet(containerId)) {
- ConcurrentHashMap<String, String> compTags = getComponentSpecificTags(component);
- synchronized (compTags) {
- for (String key : compTags.keySet()) {
- if (compTags.get(key).equals(containerId)) {
- return key;
- }
- }
- for (String key : compTags.keySet()) {
- if (compTags.get(key).equals(FREE)) {
- compTags.put(key, containerId);
- return key;
- }
- }
- String newKey = new Integer(compTags.size() + 1).toString();
- compTags.put(newKey, containerId);
- return newKey;
- }
- }
- return "";
- }
-
- /**
- * Release a tag associated with a container
- *
- * @param component
- * @param containerId
- */
- public void releaseTag(String component, String containerId) {
- if (SliderUtils.isSet(component) && SliderUtils.isSet(containerId)) {
- ConcurrentHashMap<String, String> compTags = allTags.get(component);
- if (compTags != null) {
- synchronized (compTags) {
- for (String key : compTags.keySet()) {
- if (compTags.get(key).equals(containerId)) {
- compTags.put(key, FREE);
- }
- }
- }
- }
- }
- }
-
- private ConcurrentHashMap<String, String> getComponentSpecificTags(String component) {
- if (!allTags.containsKey(component)) {
- synchronized (allTags) {
- if (!allTags.containsKey(component)) {
- allTags.put(component, new ConcurrentHashMap<String, String>());
- }
- }
- }
- return allTags.get(component);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ContainerState.java
deleted file mode 100644
index 0394ba2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/ContainerState.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-/** The states a component instance can be. */
-public enum ContainerState {
- INIT, // Container is not net activated
- HEALTHY, // Agent is heartbeating
- UNHEALTHY, // Container is unhealthy - no heartbeat for some interval
- HEARTBEAT_LOST; // Container is lost - request a new instance
-
- /**
- * Indicates whether or not it is a valid state to produce a command.
- *
- * @return true if command can be issued for this state.
- */
- public boolean canIssueCommands() {
- switch (this) {
- case HEALTHY:
- return true;
- default:
- return false;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java
deleted file mode 100644
index 4293916..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/HeartbeatMonitor.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-
-/** Monitors the container state and heartbeats. */
-public class HeartbeatMonitor implements Runnable {
- protected static final Logger log =
- LoggerFactory.getLogger(HeartbeatMonitor.class);
- private final int threadWakeupInterval; //1 minute
- private final AgentProviderService provider;
- private volatile boolean shouldRun = true;
- private Thread monitorThread = null;
-
- public HeartbeatMonitor(AgentProviderService provider, int threadWakeupInterval) {
- this.provider = provider;
- this.threadWakeupInterval = threadWakeupInterval;
- }
-
- public void shutdown() {
- shouldRun = false;
- }
-
- public void start() {
- log.info("Starting heartbeat monitor with interval {}", threadWakeupInterval);
- monitorThread = new Thread(this);
- monitorThread.start();
- }
-
- void join(long millis) throws InterruptedException {
- if (isAlive()) {
- monitorThread.join(millis);
- }
- }
-
- public boolean isAlive() {
- return monitorThread != null && monitorThread.isAlive();
- }
-
- @Override
- public void run() {
- while (shouldRun) {
- try {
- log.debug("Putting monitor to sleep for " + threadWakeupInterval + " " +
- "milliseconds");
- Thread.sleep(threadWakeupInterval);
- doWork(System.currentTimeMillis());
- } catch (InterruptedException ex) {
- log.warn("Scheduler thread is interrupted going to stop", ex);
- shouldRun = false;
- } catch (Exception ex) {
- log.warn("Exception received", ex);
- } catch (Throwable t) {
- log.warn("ERROR", t);
- }
- }
- }
-
- /**
- * Every interval the current state of the container are checked. If the state is INIT or HEALTHY and no HB are
- * received in last check interval they are marked as UNHEALTHY. INIT is when the agent is started but it did not
- * communicate at all. HEALTHY being the AM has received heartbeats. After an interval as UNHEALTHY the container is
- * declared unavailable
- * @param now current time in milliseconds ... tests can set this explicitly
- */
- @VisibleForTesting
- public void doWork(long now) {
- Map<String, ComponentInstanceState> componentStatuses = provider.getComponentStatuses();
- if (componentStatuses != null) {
- for (String containerLabel : componentStatuses.keySet()) {
- ComponentInstanceState componentInstanceState = componentStatuses.get(containerLabel);
- long timeSinceLastHeartbeat = now - componentInstanceState.getLastHeartbeat();
-
- if (timeSinceLastHeartbeat > threadWakeupInterval) {
- switch (componentInstanceState.getContainerState()) {
- case INIT:
- case HEALTHY:
- componentInstanceState.setContainerState(ContainerState.UNHEALTHY);
- log.warn(
- "Component {} marked UNHEALTHY. Last heartbeat received at {} approx. {} ms. back.",
- componentInstanceState,
- componentInstanceState.getLastHeartbeat(),
- timeSinceLastHeartbeat);
- break;
- case UNHEALTHY:
- if (timeSinceLastHeartbeat > threadWakeupInterval * 2) {
- componentInstanceState.setContainerState(
- ContainerState.HEARTBEAT_LOST);
- log.warn(
- "Component {} marked HEARTBEAT_LOST. Last heartbeat received at {} approx. {} ms. back.",
- componentInstanceState, componentInstanceState.getLastHeartbeat(),
- timeSinceLastHeartbeat);
- ContainerId containerId =
- componentInstanceState.getContainerId();
- provider.lostContainer(containerLabel, containerId);
- }
- break;
- case HEARTBEAT_LOST:
- // unexpected case
- log.warn("Heartbeat from lost component: {}", componentInstanceState);
- break;
- }
-
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/State.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/State.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/State.java
deleted file mode 100644
index 5603f8d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/State.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-/** The states a component instance can be. */
-public enum State {
- INIT, // Not installed
- INSTALLING, // Being installed
- INSTALLED, // Installed (or stopped)
- STARTING, // Starting
- STARTED, // Started
- INSTALL_FAILED, // Install failed, start failure in INSTALLED
- UPGRADING, // Undergoing upgrade, perform necessary pre-upgrade steps
- UPGRADED, // Pre-upgrade steps completed
- STOPPING, // Stop has been issued
- STOPPED, // Agent has stopped
- TERMINATING; // Terminate signal to ask the agent to kill itself
- // No need for state TERMINATED (as the agent is dead by then)
-
- /**
- * Indicates whether or not it is a valid state to produce a command.
- *
- * @return true if command can be issued for this state.
- */
- public boolean canIssueCommands() {
- switch (this) {
- case INSTALLING:
- case STARTING:
- case UPGRADING:
- case STOPPING:
- case TERMINATING:
- return false;
- default:
- return true;
- }
- }
-
- /**
- * Returns valid command in this state.
- *
- * @return command allowed in this state.
- */
- public Command getSupportedCommand() {
- return getSupportedCommand(false);
- }
-
- public Command getSupportedCommand(boolean isInUpgradeMode) {
- return getSupportedCommand(isInUpgradeMode, false);
- }
-
- public Command getSupportedCommand(boolean isInUpgradeMode,
- boolean stopInitiated) {
- switch (this) {
- case INIT:
- case INSTALL_FAILED:
- return Command.INSTALL;
- case INSTALLED:
- return Command.START;
- case STARTED:
- return isInUpgradeMode ? Command.UPGRADE : (stopInitiated) ? Command.STOP
- : Command.NOP;
- case UPGRADED:
- return Command.STOP;
- case STOPPED:
- return Command.TERMINATE;
- default:
- return Command.NOP;
- }
- }
-
- /**
- * Returns next state based on the command result.
- *
- * @return next state.
- */
- public State getNextState(CommandResult result) throws IllegalArgumentException {
- switch (result) {
- case IN_PROGRESS:
- if (this == State.INSTALLING || this == State.STARTING
- || this == State.UPGRADING || this == State.STOPPING
- || this == State.TERMINATING) {
- return this;
- } else {
- throw new IllegalArgumentException(result + " is not valid for " + this);
- }
- case COMPLETED:
- if (this == State.INSTALLING) {
- return State.INSTALLED;
- } else if (this == State.STARTING) {
- return State.STARTED;
- } else if (this == State.UPGRADING) {
- return State.UPGRADED;
- } else if (this == State.STOPPING) {
- return State.STOPPED;
- } else {
- throw new IllegalArgumentException(result + " is not valid for " + this);
- }
- case FAILED:
- if (this == State.INSTALLING) {
- return State.INSTALL_FAILED;
- } else if (this == State.STARTING) {
- return State.INSTALLED;
- } else if (this == State.UPGRADING) {
- // if pre-upgrade failed, force stop now, so mark it upgraded
- // what other options can be exposed to app owner?
- return State.UPGRADED;
- } else if (this == State.STOPPING) {
- // if stop fails, force mark it stopped (and let container terminate)
- return State.STOPPED;
- } else if (this == State.STOPPED) {
- // if in stopped state, force mark it as terminating
- return State.TERMINATING;
- } else {
- throw new IllegalArgumentException(result + " is not valid for " + this);
- }
- default:
- throw new IllegalArgumentException("Bad command result " + result);
- }
- }
-
- /**
- * Returns next state based on the command.
- *
- * @return next state.
- */
- public State getNextState(Command command) throws IllegalArgumentException {
- switch (command) {
- case INSTALL:
- if (this == State.INIT || this == State.INSTALL_FAILED) {
- return State.INSTALLING;
- } else {
- throw new IllegalArgumentException(command + " is not valid for " + this);
- }
- case INSTALL_ADDON:
- if (this == State.INIT || this == State.INSTALL_FAILED) {
- return State.INSTALLING;
- } else {
- throw new IllegalArgumentException(command + " is not valid for " + this);
- }
- case START:
- if (this == State.INSTALLED) {
- return State.STARTING;
- } else {
- throw new IllegalArgumentException(command + " is not valid for " + this);
- }
- case UPGRADE:
- if (this == State.STARTED) {
- return State.UPGRADING;
- } else {
- throw new IllegalArgumentException(command + " is not valid for " + this);
- }
- case STOP:
- if (this == State.STARTED || this == State.UPGRADED) {
- return State.STOPPING;
- } else {
- throw new IllegalArgumentException(command + " is not valid for " + this);
- }
- case TERMINATE:
- if (this == State.STOPPED) {
- return State.TERMINATING;
- } else {
- throw new IllegalArgumentException(command + " is not valid for " + this);
- }
- case NOP:
- return this;
- default:
- throw new IllegalArgumentException("Bad command " + command);
- }
- }
-
- public boolean couldHaveIssued(Command command) {
- if ((this == State.INSTALLING && command == Command.INSTALL)
- || (this == State.STARTING && command == Command.START)
- || (this == State.UPGRADING && command == Command.UPGRADE)
- || (this == State.STOPPING
- && (command == Command.STOP || command == Command.NOP))
- || (this == State.TERMINATING && command == Command.TERMINATE)
- ) {
- return true;
- }
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java
deleted file mode 100644
index b6ae4de..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractComponent.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.codehaus.jackson.annotate.JsonProperty;
-
-/**
- * Component defined in master package metainfo.json
- */
-public abstract class AbstractComponent implements Validate {
- public static final String TYPE_STANDARD = "STANDARD";
- public static final String TYPE_DOCKER = "DOCKER";
- public static final String TYPE_PYTHON = "PYTHON";
- public static final String CATEGORY_MASTER = "MASTER";
- public static final String CATEGORY_SLAVE = "SLAVE";
- public static final String CATEGORY_CLIENT = "CLIENT";
- public static final String MASTER_PACKAGE_NAME = "MASTER";
-
- protected String name;
- protected CommandScript commandScript;
- protected List<ComponentCommand> commands = new ArrayList<>();
-
- public AbstractComponent() {
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public CommandScript getCommandScript() {
- return commandScript;
- }
-
- public void addCommandScript(CommandScript commandScript) {
- this.commandScript = commandScript;
- }
-
- @JsonProperty("commands")
- public List<ComponentCommand> getCommands() {
- return commands;
- }
-
- public void setCommands(List<ComponentCommand> commands) {
- this.commands = commands;
- }
-
- public void addCommand(ComponentCommand command) {
- commands.add(command);
- }
-
- @Override
- public String toString() {
- final StringBuilder sb = new StringBuilder("{");
- sb.append("\n\"name\": ").append(name);
- sb.append(",\n\"commandScript\" :").append(commandScript);
- sb.append('}');
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java
deleted file mode 100644
index 67d1f15..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-
-import org.apache.commons.digester.Digester;
-import org.apache.commons.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.xml.sax.SAXException;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.StringWriter;
-
-/**
- * This abstract class provide common functionality to parse metainfo.json for
- * either master package or add on packages.
- */
-public abstract class AbstractMetainfoParser {
- protected final GsonBuilder gsonBuilder = new GsonBuilder();
- protected final Gson gson;
- private static final Logger log = LoggerFactory
- .getLogger(AbstractMetainfoParser.class);
-
- public AbstractMetainfoParser() {
- gson = gsonBuilder.create();
- }
-
- /**
- * Convert to a JSON string
- *
- * @return a JSON string description
- *
- * @throws IOException Problems mapping/writing the object
- */
- public String toJsonString(Metainfo metaInfo) throws IOException {
- return gson.toJson(metaInfo);
- }
-
- /**
- * Convert from JSON
- *
- * @param json input
- *
- * @return the parsed JSON
- *
- * @throws IOException IO
- */
- public Metainfo fromJsonString(String json)
- throws IOException {
- return gson.fromJson(json, Metainfo.class);
- }
-
- /**
- * Parse metainfo from an IOStream
- *
- * @param is
- *
- * @return
- *
- * @throws IOException
- */
- public Metainfo fromJsonStream(InputStream is) throws IOException {
- log.debug("loading from xml stream");
- StringWriter writer = new StringWriter();
- IOUtils.copy(is, writer);
- return fromJsonString(writer.toString());
- }
-
- /**
- * Parse metainfo from an XML formatted IOStream
- *
- * @param metainfoStream
- *
- * @return
- *
- * @throws IOException
- */
- public Metainfo fromXmlStream(InputStream metainfoStream) throws IOException {
- log.debug("loading from xml stream");
- Digester digester = new Digester();
- digester.setValidating(false);
-
- composeSchema(digester);
-
- try {
- return (Metainfo) digester.parse(metainfoStream);
- } catch (IOException e) {
- log.debug("IOException in metainfoparser during fromXmlStream: "
- + e.getMessage());
- } catch (SAXException e) {
- log.debug("SAXException in metainfoparser during fromXmlStream: "
- + e.getMessage());
- } finally {
- if (metainfoStream != null) {
- metainfoStream.close();
- }
- }
-
- return null;
- }
-
- /**
- * Compose the schema for the metainfo
- *
- * @param Digester - The Digester object we passed in to compose the schema
- *
- * @return
- *
- * @throws IOException
- */
- abstract protected void composeSchema(Digester digester);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoSchema.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoSchema.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoSchema.java
deleted file mode 100644
index cfa2895..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoSchema.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.codehaus.jackson.annotate.JsonProperty;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Application type defined in the metainfo
- */
-public abstract class AbstractMetainfoSchema implements Validate {
- protected String name;
- protected String comment;
- protected String version;
- protected List<ConfigFile> configFiles = new ArrayList<>();
-
- public AbstractMetainfoSchema() {
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getComment() {
- return comment;
- }
-
- public void setComment(String comment) {
- this.comment = comment;
- }
-
- public String getVersion() {
- return version;
- }
-
- public void setVersion(String version) {
- this.version = version;
- }
-
- public void addConfigFile(ConfigFile configFile) {
- this.configFiles.add(configFile);
- }
-
- @JsonProperty("configFiles")
- public List<ConfigFile> getConfigFiles() {
- return configFiles;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AddonPackageMetainfoParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AddonPackageMetainfoParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AddonPackageMetainfoParser.java
deleted file mode 100644
index c75837f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AddonPackageMetainfoParser.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.commons.digester.Digester;
-
-/**
- *
- */
-public class AddonPackageMetainfoParser extends AbstractMetainfoParser {
-
- protected void composeSchema(Digester digester) {
- digester.addObjectCreate("metainfo", Metainfo.class);
- digester.addBeanPropertySetter("metainfo/schemaVersion");
-
- digester.addObjectCreate("*/applicationPackage", ApplicationPackage.class);
- digester.addBeanPropertySetter("*/applicationPackage/name");
- digester.addBeanPropertySetter("*/applicationPackage/comment");
- digester.addBeanPropertySetter("*/applicationPackage/version");
-
- digester.addObjectCreate("*/component", ComponentsInAddonPackage.class);
- digester.addBeanPropertySetter("*/component/name");
- digester.addSetNext("*/component", "addComponent");
-
- digester.addObjectCreate("*/commandScript", CommandScript.class);
- digester.addBeanPropertySetter("*/commandScript/script");
- digester.addBeanPropertySetter("*/commandScript/scriptType");
- digester.addBeanPropertySetter("*/commandScript/timeout");
- digester.addSetNext("*/commandScript", "addCommandScript");
-
- digester.addObjectCreate("*/configFile", ConfigFile.class);
- digester.addBeanPropertySetter("*/configFile/type");
- digester.addBeanPropertySetter("*/configFile/fileName");
- digester.addBeanPropertySetter("*/configFile/dictionaryName");
- digester.addSetNext("*/configFile", "addConfigFile");
-
- digester.addSetRoot("*/applicationPackage", "setApplicationPackage");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
deleted file mode 100644
index 5556c7f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.codehaus.jackson.annotate.JsonIgnore;
-import org.codehaus.jackson.annotate.JsonProperty;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Application type defined in the metainfo
- */
-public class Application extends AbstractMetainfoSchema {
- String exportedConfigs;
- List<ExportGroup> exportGroups = new ArrayList<>();
- List<OSSpecific> osSpecifics = new ArrayList<>();
- List<CommandOrder> commandOrders = new ArrayList<>();
- List<Package> packages = new ArrayList<>();
- private List<Component> components = new ArrayList<>();
-
- public Application() {
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getComment() {
- return comment;
- }
-
- public void setComment(String comment) {
- this.comment = comment;
- }
-
- public String getVersion() {
- return version;
- }
-
- public void setVersion(String version) {
- this.version = version;
- }
-
- public String getExportedConfigs() {
- return exportedConfigs;
- }
-
- public void setExportedConfigs(String exportedConfigs) {
- this.exportedConfigs = exportedConfigs;
- }
-
- public void addConfigFile(ConfigFile configFile) {
- this.configFiles.add(configFile);
- }
-
- @JsonProperty("configFiles")
- public List<ConfigFile> getConfigFiles() {
- return configFiles;
- }
-
- public void addComponent(Component component) {
- components.add(component);
- }
-
- @JsonProperty("components")
- public List<Component> getComponents() {
- return components;
- }
-
- public void addExportGroup(ExportGroup exportGroup) {
- exportGroups.add(exportGroup);
- }
-
- @JsonProperty("exportGroups")
- public List<ExportGroup> getExportGroups() {
- return exportGroups;
- }
-
- public void addOSSpecific(OSSpecific osSpecific) {
- osSpecifics.add(osSpecific);
- }
-
- @JsonIgnore
- public List<OSSpecific> getOSSpecifics() {
- return osSpecifics;
- }
-
- public void addCommandOrder(CommandOrder commandOrder) {
- commandOrders.add(commandOrder);
- }
-
- @JsonProperty("commandOrders")
- public List<CommandOrder> getCommandOrders() {
- return commandOrders;
- }
-
- public void addPackage(Package pkg) {
- packages.add(pkg);
- }
-
- @JsonProperty("packages")
- public List<Package> getPackages() {
- return packages;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("{");
- sb.append(",\n\"name\": ").append(name);
- sb.append(",\n\"comment\": ").append(comment);
- sb.append(",\n\"version\" :").append(version);
- sb.append(",\n\"components\" : {");
- for (Component component : components) {
- sb.append("\n").append(component.toString());
- }
- sb.append("\n},");
- sb.append('}');
- return sb.toString();
- }
-
- public void validate(String version) throws SliderException {
- if(SliderUtils.isUnset(version)) {
- throw new BadCommandArgumentsException("schema version cannot be null");
- }
-
- Metainfo.checkNonNull(getName(), "name", "application");
-
- Metainfo.checkNonNull(getVersion(), "version", "application");
-
- if(getComponents().size() == 0) {
- throw new SliderException("application must contain at least one component");
- }
-
- if(version.equals(Metainfo.VERSION_TWO_ZERO)) {
- if(getPackages().size() > 0) {
- throw new SliderException("packages is not supported in version " + version);
- }
- }
-
- if(version.equals(Metainfo.VERSION_TWO_ONE)) {
- if(getOSSpecifics().size() > 0) {
- throw new SliderException("osSpecifics is not supported in version " + version);
- }
- }
-
- for(CommandOrder co : getCommandOrders()) {
- co.validate(version);
- }
-
- for(Component comp : getComponents()) {
- comp.validate(version);
- }
-
- for(ConfigFile cf : getConfigFiles()) {
- cf.validate(version);
- }
-
- for(ExportGroup eg : getExportGroups()) {
- eg.validate(version);
- }
-
- for(Package pkg : getPackages()) {
- pkg.validate(version);
- }
-
- for(OSSpecific os : getOSSpecifics()) {
- os.validate(version);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ApplicationPackage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ApplicationPackage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ApplicationPackage.java
deleted file mode 100644
index a94a213..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ApplicationPackage.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.slider.core.exceptions.SliderException;
-
-public class ApplicationPackage extends AbstractMetainfoSchema{
- private List<ComponentsInAddonPackage> components = new ArrayList<ComponentsInAddonPackage>();
-
- public void addComponent(ComponentsInAddonPackage component) {
- components.add(component);
- }
-
- // we must override getcomponent() as well. otherwise it is pointing to the
- // overriden components of type List<Component>
- public List<ComponentsInAddonPackage> getComponents(){
- return this.components;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb = new StringBuilder("{");
- sb.append("\n\"name\": ").append(name);
- sb.append(",\n\"comment\": ").append(comment);
- sb.append(",\n\"version\" :").append(version);
- sb.append(",\n\"components\" : {");
- for (ComponentsInAddonPackage component : components) {
- sb.append("\n").append(component);
- }
- sb.append("\n},");
- sb.append('}');
- return sb.toString();
- }
-
- @Override
- public void validate(String version) throws SliderException {
- if (name == null || name.isEmpty()) {
- throw new SliderException(
- "Missing name in metainfo.json for add on packages");
- }
- if (components.isEmpty()) {
- throw new SliderException(
- "Missing components in metainfo.json for add on packages");
- }
- for (ComponentsInAddonPackage component : components) {
- if (component.name == null || component.name.isEmpty()) {
- throw new SliderException(
- "Missing name of components in metainfo.json for add on packages");
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java
deleted file mode 100644
index 40d8cc6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-
-/**
- *
- */
-public class CommandOrder implements Validate {
- String command;
- String requires;
-
- public CommandOrder() {
- }
-
- public String getCommand() {
- return command;
- }
-
- public void setCommand(String command) {
- this.command = command;
- }
-
- public String getRequires() {
- return requires;
- }
-
- public void setRequires(String requires) {
- this.requires = requires;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("{");
- sb.append(",\n\"command\": ").append(command);
- sb.append(",\n\"requires\": ").append(requires);
- sb.append('}');
- return sb.toString();
- }
-
- public void validate(String version) throws SliderException {
- Metainfo.checkNonNull(getCommand(), "command", "package");
- Metainfo.checkNonNull(getRequires(), "requires", "package");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java
deleted file mode 100644
index 9915ba1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-
-/**
- * CommandScript that implements all component commands
- */
-public class CommandScript implements Validate {
- String script;
- String scriptType;
- long timeout;
-
- public CommandScript() {
-
- }
-
- public String getScript() {
- return script;
- }
-
- public void setScript(String script) {
- this.script = script;
- }
-
- public String getScriptType() {
- return scriptType;
- }
-
- public void setScriptType(String scriptType) {
- this.scriptType = scriptType;
- }
-
- public long getTimeout() {
- return timeout;
- }
-
- public void setTimeout(long timeout) {
- this.timeout = timeout;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("{");
- sb.append(",\n\"script\": ").append(script);
- sb.append(",\n\"scriptType\": ").append(scriptType);
- sb.append(",\n\"timeout\" :").append(timeout);
- sb.append('}');
- return sb.toString();
- }
-
- public void validate(String version) throws SliderException {
- Metainfo.checkNonNull(getScript(), "script", "commandScript");
- Metainfo.checkNonNull(getScriptType(), "scriptType", "commandScript");
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/51] [abbrv] hadoop git commit: YARN-5808. Add gc log options to
the yarn daemon script when starting services-api. Contributed by Billie
Rinaldi
Posted by ji...@apache.org.
YARN-5808. Add gc log options to the yarn daemon script when starting services-api. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1805e369
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1805e369
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1805e369
Branch: refs/heads/yarn-native-services
Commit: 1805e36903ef7c0c7e908ffc75f20b4de57fa3ce
Parents: af1025c
Author: Gour Saha <go...@apache.org>
Authored: Thu Nov 10 11:35:02 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
hadoop-yarn-project/hadoop-yarn/bin/yarn | 16 ++++++++--------
hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh | 12 ++++++++++++
2 files changed, 20 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1805e369/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 26d54b8..2396a7a 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -47,7 +47,7 @@ function hadoop_usage
hadoop_add_subcommand "resourcemanager" "run the ResourceManager"
hadoop_add_subcommand "rmadmin" "admin tools"
hadoop_add_subcommand "scmadmin" "SharedCacheManager admin tools"
- hadoop_add_subcommand "services-api" "run slider services api"
+ hadoop_add_subcommand "servicesapi" "run slider services api"
hadoop_add_subcommand "sharedcachemanager" "run the SharedCacheManager daemon"
hadoop_add_subcommand "slider" "run a slider app"
hadoop_add_subcommand "timelinereader" "run the timeline reader server"
@@ -144,20 +144,20 @@ function yarncmd_case
scmadmin)
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
;;
- services-api)
+ servicesapi)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services-api"'/*'
HADOOP_CLASSNAME='org.apache.hadoop.yarn.services.webapp.ApplicationApiWebApp'
- hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS} \
--Dslider.libdir=${HADOOP_YARN_HOME}/${YARN_DIR},\
+ local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
${HADOOP_HDFS_HOME}/${HDFS_DIR},\
${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
+ hadoop_translate_cygwin_path sld
+ hadoop_add_param HADOOP_OPTS slider.libdir "-Dslider.libdir=${sld}"
;;
sharedcachemanager)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
@@ -166,15 +166,15 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
slider)
hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
HADOOP_CLASSNAME='org.apache.slider.Slider'
- hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS} \
--Dslider.libdir=${HADOOP_YARN_HOME}/${YARN_DIR},\
+ local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
${HADOOP_HDFS_HOME}/${HDFS_DIR},\
${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
+ hadoop_translate_cygwin_path sld
+ hadoop_add_param HADOOP_OPTS slider.libdir "-Dslider.libdir=${sld}"
;;
timelinereader)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1805e369/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
index d003adb..3828897 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
+++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
@@ -136,3 +136,15 @@
# See ResourceManager for some examples
#
#export YARN_SHAREDCACHEMANAGER_OPTS=
+
+###
+# Services API specific parameters
+###
+# Specify the JVM options to be used when starting the services API.
+#
+# These options will be appended to the options specified as HADOOP_OPTS
+# and therefore may override any similar flags set in HADOOP_OPTS
+#
+# See ResourceManager for some examples
+#
+#export YARN_SERVICESAPI_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-servicesapi.log-$(date +'%Y%m%d%H%M')"
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/51] [abbrv] hadoop git commit: YARN-6010. Fix findbugs,
site warnings in yarn-services-api module. Contributed by Jian He
Posted by ji...@apache.org.
YARN-6010. Fix findbugs, site warnings in yarn-services-api module. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa46df92
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa46df92
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa46df92
Branch: refs/heads/yarn-native-services
Commit: fa46df9295cf571e27ad83c2c93ba4f5ca341bc2
Parents: 01813cf
Author: Billie Rinaldi <bi...@apache.org>
Authored: Mon Dec 19 16:15:58 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../dev-support/findbugs-exclude.xml | 4 +++-
.../hadoop-yarn-services-api/pom.xml | 13 -------------
2 files changed, 3 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa46df92/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
index b89146a..2843338 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
@@ -16,5 +16,7 @@
limitations under the License.
-->
<FindBugsFilter>
-
+ <Match>
+ <Package name="org.apache.hadoop.yarn.services.resource" />
+ </Match>
</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa46df92/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
index c198f28..e467f7f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -108,19 +108,6 @@
</configuration>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-site-plugin</artifactId>
- <version>${maven-site-plugin.version}</version>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.maven.doxia</groupId>
- <artifactId>doxia-module-markdown</artifactId>
- <version>${maven-doxia-module-markdown.version}</version>
- </dependency>
- </dependencies>
- </plugin>
</plugins>
</build>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/51] [abbrv] hadoop git commit: YARN-5610. Initial code for native
services REST API. Contributed by Gour Saha
Posted by ji...@apache.org.
YARN-5610. Initial code for native services REST API. Contributed by Gour Saha
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/07710758
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/07710758
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/07710758
Branch: refs/heads/yarn-native-services
Commit: 077107586cefcc10813325312ff191fae068a340
Parents: 09010e7
Author: Jian He <ji...@apache.org>
Authored: Tue Oct 11 11:36:57 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
hadoop-project/pom.xml | 20 +
.../dev-support/findbugs-exclude.xml | 20 +
.../hadoop-yarn-services-api/pom.xml | 225 +++
.../yarn/services/api/ApplicationApi.java | 38 +
.../api/impl/ApplicationApiService.java | 1527 ++++++++++++++++++
.../yarn/services/resource/Application.java | 452 ++++++
.../services/resource/ApplicationState.java | 25 +
.../services/resource/ApplicationStatus.java | 147 ++
.../hadoop/yarn/services/resource/Artifact.java | 155 ++
.../yarn/services/resource/BaseResource.java | 48 +
.../yarn/services/resource/Component.java | 377 +++++
.../yarn/services/resource/ConfigFile.java | 190 +++
.../yarn/services/resource/Configuration.java | 147 ++
.../yarn/services/resource/Container.java | 256 +++
.../yarn/services/resource/ContainerState.java | 25 +
.../hadoop/yarn/services/resource/Error.java | 125 ++
.../yarn/services/resource/PlacementPolicy.java | 97 ++
.../yarn/services/resource/ReadinessCheck.java | 161 ++
.../hadoop/yarn/services/resource/Resource.java | 149 ++
.../yarn/services/utils/RestApiConstants.java | 66 +
.../services/utils/RestApiErrorMessages.java | 79 +
.../services/webapp/ApplicationApiWebApp.java | 127 ++
.../src/main/resources/log4j-server.properties | 76 +
.../resources/webapps/services-rest-api/app | 16 +
.../src/main/scripts/run_rest_service.sh | 28 +
.../src/main/webapp/WEB-INF/web.xml | 36 +
.../api/impl/TestApplicationApiService.java | 232 +++
.../hadoop-yarn-applications/pom.xml | 2 +-
28 files changed, 4845 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index a935292..a633873 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -130,6 +130,9 @@
<!-- the version of Hadoop declared in the version resources; can be overridden
so that Hadoop 3.x can declare itself a 2.x artifact. -->
<declared.hadoop.version>${project.version}</declared.hadoop.version>
+
+ <swagger-annotations-version>1.5.4</swagger-annotations-version>
+ <maven-doxia-module-markdown.version>1.4</maven-doxia-module-markdown.version>
</properties>
<dependencyManagement>
@@ -1254,6 +1257,23 @@
<artifactId>kerb-simplekdc</artifactId>
<version>1.0.0-RC2</version>
</dependency>
+
+ <dependency>
+ <groupId>io.swagger</groupId>
+ <artifactId>swagger-annotations</artifactId>
+ <version>${swagger-annotations-version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.jaxrs</groupId>
+ <artifactId>jackson-jaxrs-json-provider</artifactId>
+ <version>${jackson2.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.doxia</groupId>
+ <artifactId>doxia-module-markdown</artifactId>
+ <version>${maven-doxia-module-markdown.version}</version>
+ </dependency>
+
</dependencies>
</dependencyManagement>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
new file mode 100644
index 0000000..b89146a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<FindBugsFilter>
+
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
new file mode 100644
index 0000000..78b7855
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -0,0 +1,225 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-applications</artifactId>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
+ </parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-services-api</artifactId>
+ <name>Apache Hadoop YARN Services API</name>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
+ <packaging>jar</packaging>
+ <description>Hadoop YARN REST APIs for services</description>
+
+ <properties>
+ <test.failIfNoTests>false</test.failIfNoTests>
+ <powermock.version>1.6.5</powermock.version>
+ </properties>
+
+ <build>
+
+ <!-- resources are filtered for dynamic updates. This gets build info in-->
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ <resource>
+ <directory>src/main/scripts/</directory>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+
+ <plugins>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>${maven-jar-plugin.version}</version>
+ <!-- The configuration of the plugin -->
+ <configuration>
+ <!-- Configuration of the archiver -->
+ <archive>
+ <manifestEntries>
+ <mode>development</mode>
+ <url>${project.url}</url>
+ </manifestEntries>
+ <!-- Manifest specific configuration -->
+ <manifest>
+ </manifest>
+ </archive>
+ <excludes>
+ <exclude>**/run_rest_service.sh</exclude>
+ </excludes>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${maven-surefire-plugin.version}</version>
+ <configuration>
+ <reuseForks>${test.reuseForks}</reuseForks>
+ <forkMode>${test.forkMode}</forkMode>
+ <forkCount>1</forkCount>
+ <forkedProcessTimeoutInSeconds>${test.forkedProcessTimeoutInSeconds}
+ </forkedProcessTimeoutInSeconds>
+ <threadCount>1</threadCount>
+ <argLine>${test.argLine}</argLine>
+ <failIfNoTests>${test.failIfNoTests}</failIfNoTests>
+ <redirectTestOutputToFile>${build.redirect.test.output.to.file}</redirectTestOutputToFile>
+ <environmentVariables>
+ <PATH>${test.env.path}</PATH>
+ </environmentVariables>
+ <systemPropertyVariables>
+ <java.net.preferIPv4Stack>true</java.net.preferIPv4Stack>
+ <java.awt.headless>true</java.awt.headless>
+ </systemPropertyVariables>
+ <includes>
+ <include>**/Test*.java</include>
+ </includes>
+ <excludes>
+ <exclude>**/Test*$*.java</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <version>${maven-site-plugin.version}</version>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.maven.doxia</groupId>
+ <artifactId>doxia-module-markdown</artifactId>
+ <version>${maven-doxia-module-markdown.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+ </plugins>
+ </build>
+
+ <reporting>
+ </reporting>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-slider-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.swagger</groupId>
+ <artifactId>swagger-annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.jaxrs</groupId>
+ <artifactId>jackson-jaxrs-json-provider</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.jaxrs</groupId>
+ <artifactId>jackson-jaxrs-base</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-easymock</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.easymock</groupId>
+ <artifactId>easymock</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+ <profiles>
+
+ <profile>
+ <id>rat</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${apache-rat-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>check-licenses</id>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <excludes>
+ <exclude>**/*.json</exclude>
+ <exclude>**/THIRD-PARTY.properties</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ </profiles>
+
+</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java
new file mode 100644
index 0000000..654413c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.api;
+
+import javax.ws.rs.core.Response;
+
+import org.apache.hadoop.yarn.services.resource.Application;
+
+/**
+ * Apache Hadoop YARN Services REST API interface.
+ *
+ */
+public interface ApplicationApi {
+ Response createApplication(Application application);
+
+ Response getApplications(String state);
+
+ Response getApplication(String appName);
+
+ Response deleteApplication(String appName);
+
+ Response updateApplication(String appName, Application updateAppData);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
new file mode 100644
index 0000000..9645696
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -0,0 +1,1527 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.api.impl;
+
+import static org.apache.hadoop.yarn.services.utils.RestApiConstants.*;
+import static org.apache.hadoop.yarn.services.utils.RestApiErrorMessages.*;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+import javax.inject.Singleton;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.SerializationUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.services.api.ApplicationApi;
+import org.apache.hadoop.yarn.services.resource.Application;
+import org.apache.hadoop.yarn.services.resource.ApplicationState;
+import org.apache.hadoop.yarn.services.resource.ApplicationStatus;
+import org.apache.hadoop.yarn.services.resource.Artifact;
+import org.apache.hadoop.yarn.services.resource.Component;
+import org.apache.hadoop.yarn.services.resource.ConfigFile;
+import org.apache.hadoop.yarn.services.resource.Configuration;
+import org.apache.hadoop.yarn.services.resource.Container;
+import org.apache.hadoop.yarn.services.resource.ContainerState;
+import org.apache.hadoop.yarn.services.resource.Resource;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.StateValues;
+import org.apache.slider.client.SliderClient;
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.params.ActionCreateArgs;
+import org.apache.slider.common.params.ActionFlexArgs;
+import org.apache.slider.common.params.ActionFreezeArgs;
+import org.apache.slider.common.params.ActionListArgs;
+import org.apache.slider.common.params.ActionRegistryArgs;
+import org.apache.slider.common.params.ActionStatusArgs;
+import org.apache.slider.common.params.ActionThawArgs;
+import org.apache.slider.common.params.ComponentArgsDelegate;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.common.tools.SliderVersionInfo;
+import org.apache.slider.core.buildutils.BuildHelper;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.NotFoundException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.exceptions.UnknownApplicationInstanceException;
+import org.apache.slider.core.registry.docstore.ConfigFormat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonNull;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+
+@Singleton
+@Path(APPLICATIONS_API_RESOURCE_PATH)
+@Consumes({ MediaType.APPLICATION_JSON })
+@Produces({ MediaType.APPLICATION_JSON })
+public class ApplicationApiService implements ApplicationApi {
+ private static final Logger logger = LoggerFactory
+ .getLogger(ApplicationApiService.class);
+ private static org.apache.hadoop.conf.Configuration SLIDER_CONFIG;
+ private static UserGroupInformation SLIDER_USER;
+ private static SliderClient SLIDER_CLIENT;
+
+ static {
+ init();
+ }
+
+ // initialize all the common resources - order is important
+ protected static void init() {
+ SLIDER_CONFIG = getSliderClientConfiguration();
+ SLIDER_USER = getSliderUser();
+ SLIDER_CLIENT = createSliderClient();
+ }
+
+ @GET
+ @Path("/slider-version")
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Response getSliderVersion() {
+ logger.info("GET: getSliderVersion");
+
+ Map<String, Object> metadata = new HashMap<>();
+ BuildHelper.addBuildMetadata(metadata, "org.apache.hadoop.yarn.services");
+ String sliderVersion = metadata.toString();
+ logger.info("Slider version = {}", sliderVersion);
+ String hadoopVersion = SliderVersionInfo.getHadoopVersionString();
+ logger.info("Hadoop version = {}", hadoopVersion);
+ return Response.ok(
+ "{ \"slider_version\": \"" + sliderVersion
+ + "\", \"hadoop_version\": \"" + hadoopVersion + "\"}").build();
+ }
+
+ @POST
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Response createApplication(Application application) {
+ logger.info("POST: createApplication for app = {}", application);
+ ApplicationStatus applicationStatus = new ApplicationStatus();
+
+ Map<String, String> compNameArtifactIdMap = new HashMap<>();
+ // post payload validation
+ try {
+ validateApplicationPostPayload(application, compNameArtifactIdMap);
+ } catch (IllegalArgumentException e) {
+ applicationStatus.setDiagnostics(e.getMessage());
+ return Response.status(Status.BAD_REQUEST).entity(applicationStatus)
+ .build();
+ }
+ String applicationId = null;
+ try {
+ applicationId = createSliderApp(application, compNameArtifactIdMap);
+ applicationStatus.setState(ApplicationState.ACCEPTED);
+ } catch (SliderException se) {
+ logger.error("Create application failed", se);
+ if (se.getExitCode() == SliderExitCodes.EXIT_APPLICATION_IN_USE) {
+ applicationStatus.setDiagnostics(ERROR_APPLICATION_IN_USE);
+ return Response.status(Status.BAD_REQUEST).entity(applicationStatus)
+ .build();
+ } else {
+ applicationStatus.setDiagnostics(se.getMessage());
+ }
+ } catch (Exception e) {
+ logger.error("Create application failed", e);
+ applicationStatus.setDiagnostics(e.getMessage());
+ }
+
+ if (StringUtils.isNotEmpty(applicationId)) {
+ applicationStatus.setUri(CONTEXT_ROOT + APPLICATIONS_API_RESOURCE_PATH
+ + "/" + application.getName());
+ // 202 = ACCEPTED
+ return Response.status(HTTP_STATUS_CODE_ACCEPTED)
+ .entity(applicationStatus).build();
+ } else {
+ return Response.status(Status.INTERNAL_SERVER_ERROR)
+ .entity(applicationStatus).build();
+ }
+ }
+
+ @VisibleForTesting
+ protected void validateApplicationPostPayload(Application application,
+ Map<String, String> compNameArtifactIdMap) {
+ if (StringUtils.isEmpty(application.getName())) {
+ throw new IllegalArgumentException(ERROR_APPLICATION_NAME_INVALID);
+ }
+ if (!SliderUtils.isClusternameValid(application.getName())) {
+ throw new IllegalArgumentException(ERROR_APPLICATION_NAME_INVALID_FORMAT);
+ }
+
+ // If the application has no components do top-level checks
+ if (application.getComponents() == null) {
+ // artifact
+ if (application.getArtifact() == null) {
+ throw new IllegalArgumentException(ERROR_ARTIFACT_INVALID);
+ }
+ if (StringUtils.isEmpty(application.getArtifact().getId())) {
+ throw new IllegalArgumentException(ERROR_ARTIFACT_ID_INVALID);
+ }
+
+ // If artifact is of type APPLICATION, add a slider specific property
+ if (application.getArtifact().getType() == Artifact.TypeEnum.APPLICATION) {
+ if (application.getConfiguration() == null) {
+ application.setConfiguration(new Configuration());
+ }
+ addPropertyToConfiguration(application.getConfiguration(),
+ PROPERTY_COMPONENT_TYPE, COMPONENT_TYPE_EXTERNAL);
+ }
+ // resource
+ validateApplicationResource(application.getResource(), null, application
+ .getArtifact().getType());
+
+ // container size
+ if (application.getNumberOfContainers() == null) {
+ throw new IllegalArgumentException(ERROR_CONTAINERS_COUNT_INVALID);
+ }
+ } else {
+ // If the application has components, then run checks for each component.
+ // Let global values take effect if component level values are not
+ // provided.
+ Artifact globalArtifact = application.getArtifact();
+ Resource globalResource = application.getResource();
+ Long globalNumberOfContainers = application.getNumberOfContainers();
+ for (Component comp : application.getComponents()) {
+ // artifact
+ if (comp.getArtifact() == null) {
+ comp.setArtifact(globalArtifact);
+ }
+ // If still null raise validation exception
+ if (comp.getArtifact() == null) {
+ throw new IllegalArgumentException(String.format(
+ ERROR_ARTIFACT_FOR_COMP_INVALID, comp.getName()));
+ }
+ if (StringUtils.isEmpty(comp.getArtifact().getId())) {
+ throw new IllegalArgumentException(String.format(
+ ERROR_ARTIFACT_ID_FOR_COMP_INVALID, comp.getName()));
+ }
+
+ // If artifact is of type APPLICATION, add a slider specific property
+ if (comp.getArtifact().getType() == Artifact.TypeEnum.APPLICATION) {
+ if (comp.getConfiguration() == null) {
+ comp.setConfiguration(new Configuration());
+ }
+ addPropertyToConfiguration(comp.getConfiguration(),
+ PROPERTY_COMPONENT_TYPE, COMPONENT_TYPE_EXTERNAL);
+ compNameArtifactIdMap.put(comp.getName(), comp.getArtifact().getId());
+ comp.setName(comp.getArtifact().getId());
+ }
+
+ // resource
+ if (comp.getResource() == null) {
+ comp.setResource(globalResource);
+ }
+ validateApplicationResource(comp.getResource(), comp, comp
+ .getArtifact().getType());
+
+ // container count
+ if (comp.getNumberOfContainers() == null) {
+ comp.setNumberOfContainers(globalNumberOfContainers);
+ }
+ if (comp.getNumberOfContainers() == null) {
+ throw new IllegalArgumentException(String.format(
+ ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID, comp.getName()));
+ }
+ }
+ }
+
+ // If it is a simple app with no components, then create a default component
+ if (application.getComponents() == null) {
+ application.setComponents(getDefaultComponentAsList());
+ }
+
+ // Application lifetime if not specified, is set to unlimited lifetime
+ if (application.getLifetime() == null) {
+ application.setLifetime(DEFAULT_UNLIMITED_LIFETIME);
+ }
+ }
+
+ private void validateApplicationResource(Resource resource, Component comp,
+ Artifact.TypeEnum artifactType) {
+ // Only apps/components of type APPLICATION can skip resource requirement
+ if (resource == null && artifactType == Artifact.TypeEnum.APPLICATION) {
+ return;
+ }
+ if (resource == null) {
+ throw new IllegalArgumentException(comp == null ? ERROR_RESOURCE_INVALID
+ : String.format(ERROR_RESOURCE_FOR_COMP_INVALID, comp.getName()));
+ }
+ // One and only one of profile OR cpus & memory can be specified. Specifying
+ // both raises validation error.
+ if (StringUtils.isNotEmpty(resource.getProfile())
+ && (resource.getCpus() != null
+ || StringUtils.isNotEmpty(resource.getMemory()))) {
+ throw new IllegalArgumentException(
+ comp == null ? ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED
+ : String.format(
+ ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED,
+ comp.getName()));
+ }
+ // Currently resource profile is not supported yet, so we will raise
+ // validation error if only resource profile is specified
+ if (StringUtils.isNotEmpty(resource.getProfile())) {
+ throw new IllegalArgumentException(
+ ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET);
+ }
+
+ String memory = resource.getMemory();
+ Integer cpus = resource.getCpus();
+ if (StringUtils.isEmpty(memory)) {
+ throw new IllegalArgumentException(
+ comp == null ? ERROR_RESOURCE_MEMORY_INVALID : String.format(
+ ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID, comp.getName()));
+ }
+ if (cpus == null) {
+ throw new IllegalArgumentException(
+ comp == null ? ERROR_RESOURCE_CPUS_INVALID : String.format(
+ ERROR_RESOURCE_CPUS_FOR_COMP_INVALID, comp.getName()));
+ }
+ if (cpus <= 0) {
+ throw new IllegalArgumentException(
+ comp == null ? ERROR_RESOURCE_CPUS_INVALID_RANGE : String.format(
+ ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE, comp.getName()));
+ }
+ }
+
+ private String createSliderApp(Application application,
+ Map<String, String> compNameArtifactIdMap) throws IOException,
+ YarnException, InterruptedException {
+ final String appName = application.getName();
+ final String queueName = application.getQueue();
+
+ final ActionCreateArgs createArgs = new ActionCreateArgs();
+ addAppConfOptions(createArgs, application, compNameArtifactIdMap);
+ addResourceOptions(createArgs, application);
+ String metainfoJson = getMetainfoJson(application, compNameArtifactIdMap);
+
+ createArgs.appMetaInfoJson = metainfoJson;
+ if (queueName != null && queueName.trim().length() > 0) {
+ createArgs.queue = queueName.trim();
+ }
+
+ return invokeSliderClientRunnable(new SliderClientContextRunnable<String>() {
+ @Override
+ public String run(SliderClient sliderClient) throws YarnException,
+ IOException, InterruptedException {
+ sliderClient.actionCreate(appName, createArgs);
+ ApplicationId applicationId = sliderClient.applicationId;
+ if (applicationId != null) {
+ return applicationId.toString();
+ // return getApplicationIdString(applicationId);
+ }
+ return null;
+ }
+ });
+ }
+
+ private void addAppConfOptions(ActionCreateArgs createArgs,
+ Application application, Map<String, String> compNameArtifactIdMap) throws IOException {
+ List<String> appCompOptionTriples = createArgs.optionsDelegate.compOptTriples; // TODO: optionTuples instead of compOptTriples
+ logger.info("Initial appCompOptionTriples = {}",
+ Arrays.toString(appCompOptionTriples.toArray()));
+ List<String> appOptions = createArgs.optionsDelegate.optionTuples;
+ logger.info("Initial appOptions = {}",
+ Arrays.toString(appOptions.toArray()));
+ // TODO: Set Slider-AM memory and vcores here
+ // appCompOptionTriples.addAll(Arrays.asList(SLIDER_APPMASTER_COMPONENT_NAME,
+ // "", ""));
+
+ // Global configuration - for override purpose
+ // TODO: add it to yaml
+ Configuration globalConfig = null;
+ // Configuration globalConfig = (Configuration) SerializationUtils
+ // .clone(application.getConfiguration());
+
+ // TODO: Add the below into globalConfig
+ // if (application.getConfigurations() != null) {
+ // for (Entry<String, String> entry : application.getConfigurations()
+ // .entrySet()) {
+ // globalConf.addProperty(entry.getKey(), entry.getValue());
+ // }
+ // }
+
+ Set<String> uniqueGlobalPropertyCache = new HashSet<>();
+ if (application.getConfiguration() != null
+ && application.getConfiguration().getProperties() != null) {
+ for (Map.Entry<String, String> propEntry : application.getConfiguration()
+ .getProperties().entrySet()) {
+ addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache,
+ propEntry.getKey(), propEntry.getValue());
+ }
+ }
+ if (application.getComponents() != null) {
+ Map<String, String> placeholders = new HashMap<>();
+ for (Component comp : application.getComponents()) {
+ if (Boolean.TRUE.equals(comp.getUniqueComponentSupport())) {
+ for (int i = 1; i <= comp.getNumberOfContainers(); i++) {
+ placeholders.put(PLACEHOLDER_COMPONENT_ID, Integer.toString(i));
+ appCompOptionTriples.addAll(createAppConfigComponent(
+ comp.getName() + i, comp, comp.getName() + i, globalConfig,
+ placeholders, compNameArtifactIdMap));
+ }
+ } else {
+ appCompOptionTriples.addAll(createAppConfigComponent(comp.getName(),
+ comp, comp.getName(), globalConfig, null, compNameArtifactIdMap));
+ }
+ }
+ }
+
+ logger.info("Updated appCompOptionTriples = {}",
+ Arrays.toString(appCompOptionTriples.toArray()));
+ logger.info("Updated appOptions = {}",
+ Arrays.toString(appOptions.toArray()));
+ }
+
+ private void addOptionsIfNotPresent(List<String> options,
+ Set<String> uniqueGlobalPropertyCache, String key, String value) {
+ if (uniqueGlobalPropertyCache == null) {
+ options.addAll(Arrays.asList(key, value));
+ } else if (!uniqueGlobalPropertyCache.contains(key)) {
+ options.addAll(Arrays.asList(key, value));
+ uniqueGlobalPropertyCache.add(key);
+ }
+ }
+
+ private void addPropertyToConfiguration(Configuration conf, String key,
+ String value) {
+ if (conf == null) {
+ return;
+ }
+ if (conf.getProperties() == null) {
+ conf.setProperties(new HashMap<String, String>());
+ }
+ conf.getProperties().put(key, value);
+ }
+
+ private List<String> createAppConfigComponent(String compName,
+ Component component, String configPrefix, Configuration globalConf,
+ Map<String, String> placeholders,
+ Map<String, String> compNameArtifactIdMap) {
+ List<String> appConfOptTriples = new ArrayList<>();
+
+ if (component.getConfiguration() != null
+ && component.getConfiguration().getProperties() != null) {
+ for (Map.Entry<String, String> propEntry : component.getConfiguration()
+ .getProperties().entrySet()) {
+ appConfOptTriples.addAll(Arrays.asList(compName, propEntry.getKey(),
+ replacePlaceholders(propEntry.getValue(), placeholders)));
+ }
+ }
+
+ // If artifact is of type APPLICATION, then in the POST JSON there will
+ // be no component definition for that artifact. Hence it's corresponding id
+ // field is added. Every external APPLICATION has a unique id field.
+ List<String> convertedDeps = new ArrayList<>();
+ for (String dep : component.getDependencies()) {
+ if (compNameArtifactIdMap.containsKey(dep)) {
+ convertedDeps.add(compNameArtifactIdMap.get(dep));
+ } else {
+ convertedDeps.add(dep);
+ }
+ }
+ // If the DNS dependency property is set to true for a component, it means
+ // that it is ensured that DNS entry has been added for all the containers
+ // of this component, before moving on to the next component in the DAG.
+ if (hasPropertyWithValue(component, PROPERTY_DNS_DEPENDENCY, "true")) {
+ if (component.getArtifact().getType() == Artifact.TypeEnum.APPLICATION) {
+ convertedDeps.add(component.getArtifact().getId());
+ } else {
+ convertedDeps.add(compName);
+ }
+ }
+ if (convertedDeps.size() > 0) {
+ appConfOptTriples.addAll(Arrays.asList(compName, "requires",
+ StringUtils.join(convertedDeps, ",")));
+ }
+ return appConfOptTriples;
+ }
+
+ private String replacePlaceholders(String value,
+ Map<String, String> placeholders) {
+ if (StringUtils.isEmpty(value) || placeholders == null) {
+ return value;
+ }
+ for (Map.Entry<String, String> placeholder : placeholders.entrySet()) {
+ value = value.replaceAll(Pattern.quote(placeholder.getKey()),
+ placeholder.getValue());
+ }
+ return value;
+ }
+
+ private List<String> createAppConfigGlobal(Component component,
+ Configuration globalConf, Set<String> uniqueGlobalPropertyCache) {
+ List<String> appOptions = new ArrayList<>();
+ if (component.getConfiguration() != null
+ && component.getConfiguration().getProperties() != null) {
+ for (Map.Entry<String, String> propEntry : component.getConfiguration()
+ .getProperties().entrySet()) {
+ addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache,
+ propEntry.getKey(), propEntry.getValue());
+ }
+ }
+ return appOptions;
+ }
+
+ private void addResourceOptions(ActionCreateArgs createArgs,
+ Application application) throws IOException {
+ List<String> resCompOptionTriples = createArgs.optionsDelegate.resCompOptTriples;
+ logger.info("Initial resCompOptTriples = {}",
+ Arrays.toString(resCompOptionTriples.toArray()));
+ // TODO: Add any Slider AM resource specific props here like jvm.heapsize
+ // resCompOptionTriples.addAll(Arrays.asList(SLIDER_APPMASTER_COMPONENT_NAME,
+ // "", ""));
+
+ // Global resource - for override purpose
+ Resource globalResource = (Resource) SerializationUtils.clone(application
+ .getResource());
+ // Priority seeded with 1, expecting every new component will increase it by
+ // 1 making it ready for the next component to use.
+ if (application.getComponents() != null) {
+ int priority = 1;
+ for (Component comp : application.getComponents()) {
+ if (hasPropertyWithValue(comp, PROPERTY_COMPONENT_TYPE,
+ COMPONENT_TYPE_EXTERNAL)) {
+ continue;
+ }
+ if (Boolean.TRUE.equals(comp.getUniqueComponentSupport())) {
+ for (int i = 1; i <= comp.getNumberOfContainers(); i++) {
+ resCompOptionTriples.addAll(createResourcesComponent(comp.getName()
+ + i, comp, priority, 1, globalResource));
+ priority++;
+ }
+ } else {
+ resCompOptionTriples.addAll(createResourcesComponent(comp.getName(),
+ comp, priority, comp.getNumberOfContainers(), globalResource));
+ priority++;
+ }
+ }
+ }
+
+ logger.info("Updated resCompOptTriples = {}",
+ Arrays.toString(resCompOptionTriples.toArray()));
+ }
+
+ private boolean hasPropertyWithValue(Component comp, String key, String value) {
+ if (comp == null || key == null) {
+ return false;
+ }
+ if (comp.getConfiguration() == null
+ || comp.getConfiguration().getProperties() == null) {
+ return false;
+ }
+ Map<String, String> props = comp.getConfiguration().getProperties();
+ if (props.containsKey(key)) {
+ if (value == null) {
+ return props.get(key) == null;
+ } else {
+ if (value.equals(props.get(key))) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private List<String> createResourcesComponent(String compName,
+ Component component, int priority, long numInstances,
+ Resource globalResource) {
+ String memory = component.getResource() == null ? globalResource
+ .getMemory() : component.getResource().getMemory();
+ Integer cpus = component.getResource() == null ? globalResource.getCpus()
+ : component.getResource().getCpus();
+
+ List<String> resCompOptTriples = new ArrayList<String>();
+ resCompOptTriples.addAll(Arrays.asList(compName,
+ ResourceKeys.COMPONENT_PRIORITY, Integer.toString(priority)));
+ resCompOptTriples.addAll(Arrays.asList(compName,
+ ResourceKeys.COMPONENT_INSTANCES, Long.toString(numInstances)));
+ resCompOptTriples.addAll(Arrays.asList(compName, ResourceKeys.YARN_MEMORY,
+ memory));
+ resCompOptTriples.addAll(Arrays.asList(compName, ResourceKeys.YARN_CORES,
+ cpus.toString()));
+ if (component.getPlacementPolicy() != null) {
+ resCompOptTriples.addAll(Arrays.asList(compName,
+ ResourceKeys.COMPONENT_PLACEMENT_POLICY,
+ component.getPlacementPolicy().getLabel()));
+ }
+
+ return resCompOptTriples;
+ }
+
+ private String getMetainfoJson(Application application,
+ Map<String, String> compNameArtifactIdMap) throws SliderException,
+ IOException {
+ JsonObject rootObj = new JsonObject();
+ rootObj.addProperty("schemaVersion", METAINFO_SCHEMA_VERSION);
+ JsonObject applicationObj = new JsonObject();
+ rootObj.add("application", applicationObj);
+ applicationObj.addProperty("name", application.getName().toUpperCase());
+ JsonArray componentsArray = new JsonArray();
+ applicationObj.add("components", componentsArray);
+ JsonArray commandOrdersArray = new JsonArray();
+ applicationObj.add("commandOrders", commandOrdersArray);
+
+ JsonArray exportGroupsArray = new JsonArray();
+ applicationObj.add("exportGroups", exportGroupsArray);
+ // Use only one export group
+ JsonObject exportGroup = new JsonObject();
+ exportGroup.addProperty("name", EXPORT_GROUP_NAME);
+ exportGroupsArray.add(exportGroup);
+ JsonArray exportsArray = new JsonArray();
+ exportGroup.add("exports", exportsArray);
+
+ if (application.getComponents() != null) {
+
+ // Set exports at application level
+ Map<String, String> appQuicklinks = application.getQuicklinks();
+ Map<String, String> placeholders = new HashMap<>();
+ placeholders.put(PLACEHOLDER_APP_NAME, application.getName());
+ if (appQuicklinks != null) {
+ for (Map.Entry<String, String> quicklink : appQuicklinks.entrySet()) {
+ JsonObject export = new JsonObject();
+ export.addProperty("name", quicklink.getKey());
+ export.addProperty("value",
+ replacePlaceholders(quicklink.getValue(), placeholders));
+ exportsArray.add(export);
+ }
+ }
+
+ for (Component comp : application.getComponents()) {
+ JsonObject compObject = null;
+ if (!hasPropertyWithValue(comp, PROPERTY_COMPONENT_TYPE,
+ COMPONENT_TYPE_EXTERNAL)) {
+ if (Boolean.TRUE.equals(comp.getUniqueComponentSupport())) {
+ for (int i = 1; i <= comp.getNumberOfContainers(); i++) {
+ // we also need the capability to specify ports and mount points
+ // sometime
+ compObject = createMetainfoComponent(comp, application,
+ comp.getName() + i);
+ componentsArray.add(compObject);
+ }
+ } else {
+ compObject = createMetainfoComponent(comp, application,
+ comp.getName());
+ componentsArray.add(compObject);
+ }
+ }
+
+ // Translate dependencies into command orders
+ List<String> dependencies = comp.getDependencies();
+ if (dependencies != null && !dependencies.isEmpty()) {
+ JsonObject commandOrder = new JsonObject();
+ commandOrder.addProperty("command", comp.getName()
+ + COMMAND_ORDER_SUFFIX_START);
+ for (String dependency : dependencies) {
+ // If APPLICATION type artifact then map component name dependencies
+ // to artifact id
+ if (comp.getArtifact().getType() == Artifact.TypeEnum.APPLICATION) {
+ dependency = compNameArtifactIdMap.get(dependency);
+ }
+ commandOrder.addProperty("requires", dependency
+ + COMMAND_ORDER_SUFFIX_STARTED);
+ }
+ commandOrdersArray.add(commandOrder);
+ }
+
+ // Quicklinks need to be added as appExports and componentExports at the
+ // component level
+ List<String> compQuicklinks = comp.getQuicklinks();
+ if (compQuicklinks != null && !compQuicklinks.isEmpty()) {
+ if (MapUtils.isEmpty(appQuicklinks)) {
+ throw new SliderException(ERROR_QUICKLINKS_FOR_COMP_INVALID);
+ }
+ List<String> appExports = new ArrayList<>();
+ JsonArray compExportsArray = new JsonArray();
+ compObject.add("componentExports", compExportsArray);
+
+ for (String quicklink : compQuicklinks) {
+ appExports.add(EXPORT_GROUP_NAME + "-" + quicklink);
+
+ JsonObject compExport = new JsonObject();
+ compExport.addProperty("name", quicklink);
+ compExport.addProperty("value", appQuicklinks.get(quicklink));
+ compExportsArray.add(compExport);
+ }
+ compObject.addProperty("appExports",
+ StringUtils.join(appExports, ","));
+ // specify that there are published configs for this component
+ compObject.addProperty("publishConfig", "true");
+ }
+ }
+ }
+
+ String jsonString = new GsonBuilder().setPrettyPrinting().create()
+ .toJson(rootObj);
+ logger.info("Metainfo = \n{}", jsonString);
+ return jsonString;
+ }
+
+ private JsonObject createMetainfoComponent(Component comp,
+ Application application, String compName) {
+ JsonObject compObj = new JsonObject();
+ compObj.addProperty("name", compName);
+ // below is diff for each type
+ if (comp.getArtifact() != null && comp.getArtifact().getType() != null
+ && comp.getArtifact().getType() == Artifact.TypeEnum.DOCKER) {
+ compObj.addProperty("type", COMPONENT_TYPE_YARN_DOCKER);
+ JsonArray dockerContainerArray = new JsonArray();
+ compObj.add("dockerContainers", dockerContainerArray);
+ JsonObject dockerContainerObj = new JsonObject();
+ dockerContainerArray.add(dockerContainerObj);
+ dockerContainerObj.addProperty("name", compName.toLowerCase());
+ // if image not specified, then use global value
+ dockerContainerObj.addProperty("image",
+ comp.getArtifact().getId() == null ? application.getArtifact()
+ .getId() : comp.getArtifact().getId());
+ // If launch command not specified, then use global value. Resolve all
+ // placeholders.
+ Map<String, String> placeholders = new HashMap<>();
+ placeholders.put(PLACEHOLDER_APP_NAME, application.getName());
+ placeholders.put(PLACEHOLDER_APP_COMPONENT_NAME, compName);
+ dockerContainerObj.addProperty(
+ "startCommand",
+ comp.getLaunchCommand() == null ? replacePlaceholders(
+ application.getLaunchCommand(), placeholders)
+ : replacePlaceholders(comp.getLaunchCommand(), placeholders));
+ dockerContainerObj.addProperty("network", DEFAULT_NETWORK);
+ dockerContainerObj.addProperty("commandPath", DEFAULT_COMMAND_PATH);
+ // TODO: What to do with privContainer ?
+ dockerContainerObj.addProperty("runPrivilegedContainer",
+ comp.getRunPrivilegedContainer());
+ if (comp.getConfiguration() != null) {
+ List<ConfigFile> configFiles = comp.getConfiguration().getFiles();
+ if (configFiles != null && !configFiles.isEmpty()) {
+ JsonArray configFileArray = new JsonArray();
+ for (ConfigFile configFile : configFiles) {
+ JsonObject configFileObj = new JsonObject();
+ configFileObj.addProperty("type", configFile.getType().toString());
+ configFileObj.addProperty("fileName", configFile.getDestFile());
+ // TODO: add all properties which should include dictionaryName
+ configFileObj.addProperty("dictionaryName",
+ configFile.getSrcFile());
+ configFileArray.add(configFileObj);
+ }
+ dockerContainerObj.add("configFiles", configFileArray);
+ }
+ }
+ // we also need to specify artifact_management_service sometime
+ }
+ // we also need the capability to specify ports and mount points sometime
+ return compObj;
+ }
+
+ private static UserGroupInformation getSliderUser() {
+ if (SLIDER_USER != null) {
+ return SLIDER_USER;
+ }
+ UserGroupInformation sliderUser = null;
+ UserGroupInformation.setConfiguration(SLIDER_CONFIG);
+ String loggedInUser = getUserToRunAs();
+ try {
+ sliderUser = UserGroupInformation.getBestUGI(null, loggedInUser);
+ // TODO: Once plugged into RM process we should remove the previous call
+ // and replace it with getCurrentUser as commented below.
+ // sliderUser = UserGroupInformation.getCurrentUser();
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to create UGI (slider user)", e);
+ }
+ return sliderUser;
+ }
+
+ private <T> T invokeSliderClientRunnable(
+ final SliderClientContextRunnable<T> runnable)
+ throws IOException, InterruptedException, YarnException {
+ try {
+ T value = SLIDER_USER.doAs(new PrivilegedExceptionAction<T>() {
+ @Override
+ public T run() throws Exception {
+ return runnable.run(SLIDER_CLIENT);
+ }
+ });
+ return value;
+ } catch (UndeclaredThrowableException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof YarnException) {
+ YarnException ye = (YarnException) cause;
+ throw ye;
+ }
+ throw e;
+ }
+ }
+
+ protected static SliderClient createSliderClient() {
+ if (SLIDER_CLIENT != null) {
+ return SLIDER_CLIENT;
+ }
+ org.apache.hadoop.conf.Configuration sliderClientConfiguration = SLIDER_CONFIG;
+ SliderClient client = new SliderClient() {
+ @Override
+ public void init(org.apache.hadoop.conf.Configuration conf) {
+ super.init(conf);
+ try {
+ initHadoopBinding();
+ } catch (SliderException e) {
+ throw new RuntimeException(
+ "Unable to automatically init Hadoop binding", e);
+ } catch (IOException e) {
+ throw new RuntimeException(
+ "Unable to automatically init Hadoop binding", e);
+ }
+ }
+ };
+ try {
+ logger
+ .debug("Slider Client configuration: {}", sliderClientConfiguration);
+ sliderClientConfiguration = client.bindArgs(sliderClientConfiguration,
+ new String[] { "help" });
+ client.init(sliderClientConfiguration);
+ client.start();
+ } catch (Exception e) {
+ logger.error("Unable to create SliderClient", e);
+ throw new RuntimeException(e.getMessage(), e);
+ }
+ return client;
+ }
+
+ private static String getUserToRunAs() {
+ String user = System.getenv(PROPERTY_APP_RUNAS_USER);
+ if (StringUtils.isEmpty(user)) {
+ user = "root";
+ }
+ return user;
+ }
+
+ private static org.apache.hadoop.conf.Configuration getSliderClientConfiguration() {
+ if (SLIDER_CONFIG != null) {
+ return SLIDER_CONFIG;
+ }
+ YarnConfiguration yarnConfig = new YarnConfiguration();
+ logger.info("prop yarn.resourcemanager.address = {}",
+ yarnConfig.get("yarn.resourcemanager.address"));
+
+ return yarnConfig;
+ }
+
+ private interface SliderClientContextRunnable<T> {
+ T run(SliderClient sliderClient)
+ throws YarnException, IOException, InterruptedException;
+ }
+
+ @GET
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Response getApplications(@QueryParam("state") String state) {
+ logger.info("GET: getApplications with param state = {}", state);
+
+ // Get all applications in a specific state - lighter projection. For full
+ // detail, call getApplication on a specific app.
+ Set<String> applications;
+ try {
+ if (StringUtils.isNotEmpty(state)) {
+ ApplicationStatus appStatus = new ApplicationStatus();
+ try {
+ ApplicationState.valueOf(state);
+ } catch (IllegalArgumentException e) {
+ appStatus.setDiagnostics("Invalid value for param state - " + state);
+ return Response.status(Status.BAD_REQUEST).entity(appStatus).build();
+ }
+ applications = getSliderApplications(state);
+ } else {
+ applications = getSliderApplications(true);
+ }
+ } catch (Exception e) {
+ logger.error("Get applications failed", e);
+ return Response.status(Status.INTERNAL_SERVER_ERROR).build();
+ }
+
+ Set<Application> apps = new HashSet<Application>();
+ if (applications.size() > 0) {
+ try {
+ for (String app : applications) {
+ Application application = new Application();
+ // TODO: Need to get lifetime, launch-time and privileged container
+ // status from YARN
+ application.setLifetime(null);
+ application.setLaunchTime(new Date());
+ application.setName(app);
+ // Containers not required, setting to null to avoid empty list
+ application.setContainers(null);
+ apps.add(application);
+ }
+ } catch (Exception e) {
+ logger.error("Get applications failed", e);
+ return Response.status(Status.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+
+ return Response.ok().entity(apps).build();
+ }
+
+ @GET
+ @Path("/{app_name}")
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Response getApplication(@PathParam("app_name") String appName) {
+ logger.info("GET: getApplication for appName = {}", appName);
+
+ // app name validation
+ if (!SliderUtils.isClusternameValid(appName)) {
+ ApplicationStatus applicationStatus = new ApplicationStatus();
+ applicationStatus.setDiagnostics("Invalid application name");
+ applicationStatus.setCode(ERROR_CODE_APP_NAME_INVALID);
+ return Response.status(Status.NOT_FOUND).entity(applicationStatus)
+ .build();
+ }
+
+ // Check if app exists
+ try {
+ int livenessCheck = getSliderList(appName);
+ if (livenessCheck < 0) {
+ logger.info("Application not running");
+ ApplicationStatus applicationStatus = new ApplicationStatus();
+ applicationStatus.setDiagnostics(ERROR_APPLICATION_NOT_RUNNING);
+ applicationStatus.setCode(ERROR_CODE_APP_IS_NOT_RUNNING);
+ return Response.status(Status.NOT_FOUND).entity(applicationStatus)
+ .build();
+ }
+ } catch (UnknownApplicationInstanceException e) {
+ logger.error("Get application failed, application not found", e);
+ ApplicationStatus applicationStatus = new ApplicationStatus();
+ applicationStatus.setDiagnostics(ERROR_APPLICATION_DOES_NOT_EXIST);
+ applicationStatus.setCode(ERROR_CODE_APP_DOES_NOT_EXIST);
+ return Response.status(Status.NOT_FOUND).entity(applicationStatus)
+ .build();
+ } catch (Exception e) {
+ logger.error("Get application failed, application not running", e);
+ ApplicationStatus applicationStatus = new ApplicationStatus();
+ applicationStatus.setDiagnostics(ERROR_APPLICATION_NOT_RUNNING);
+ applicationStatus.setCode(ERROR_CODE_APP_IS_NOT_RUNNING);
+ return Response.status(Status.NOT_FOUND).entity(applicationStatus)
+ .build();
+ }
+
+ Application app = new Application();
+ app.setName(appName);
+ app.setUri(CONTEXT_ROOT + APPLICATIONS_API_RESOURCE_PATH + "/"
+ + appName);
+ // TODO: add status
+ app.setState(ApplicationState.ACCEPTED);
+ JsonObject appStatus = null;
+ JsonObject appRegistryDocker = null;
+ JsonObject appRegistryQuicklinks = null;
+ try {
+ appStatus = getSliderApplicationStatus(appName);
+ appRegistryDocker = getSliderApplicationRegistry(appName, "docker");
+ appRegistryQuicklinks = getSliderApplicationRegistry(appName,
+ "quicklinks");
+ return populateAppData(app, appStatus, appRegistryDocker,
+ appRegistryQuicklinks);
+ } catch (BadClusterStateException | NotFoundException e) {
+ logger.error(
+ "Get application failed, application not in running state yet", e);
+ ApplicationStatus applicationStatus = new ApplicationStatus();
+ applicationStatus.setDiagnostics("Application not running yet");
+ applicationStatus.setCode(ERROR_CODE_APP_SUBMITTED_BUT_NOT_RUNNING_YET);
+ return Response.status(Status.NOT_FOUND).entity(applicationStatus)
+ .build();
+ } catch (Exception e) {
+ logger.error("Get application failed", e);
+ ApplicationStatus applicationStatus = new ApplicationStatus();
+ applicationStatus.setDiagnostics("Failed to retrieve application: "
+ + e.getMessage());
+ return Response.status(Status.INTERNAL_SERVER_ERROR)
+ .entity(applicationStatus).build();
+ }
+ }
+
+ private Response populateAppData(Application app, JsonObject appStatus,
+ JsonObject appRegistryDocker, JsonObject appRegistryQuicklinks) {
+ String appName = jsonGetAsString(appStatus, "name");
+ Long totalNumberOfRunningContainers = 0L;
+ Long totalExpectedNumberOfRunningContainers = 0L;
+ Long totalNumberOfIpAssignedContainers = 0L;
+
+ // info
+ JsonObject applicationInfo = jsonGetAsObject(appStatus, "info");
+ if (applicationInfo != null) {
+ String applicationId = jsonGetAsString(applicationInfo, "info.am.app.id");
+ if (applicationId != null) {
+ app.setId(applicationId);
+ }
+ }
+
+ // state
+ String appState = jsonGetAsString(appStatus, "state");
+ switch (Integer.parseInt(appState)) {
+ case StateValues.STATE_LIVE:
+ app.setState(ApplicationState.STARTED);
+ break;
+ case StateValues.STATE_CREATED:
+ case StateValues.STATE_INCOMPLETE:
+ case StateValues.STATE_SUBMITTED:
+ app.setState(ApplicationState.ACCEPTED);
+ return Response.ok(app).build();
+ case StateValues.STATE_DESTROYED:
+ case StateValues.STATE_STOPPED:
+ app.setState(ApplicationState.STOPPED);
+ return Response.ok(app).build();
+ default:
+ break;
+ }
+
+ // start time
+ app.setLaunchTime(appStatus.get("createTime") == null ? null
+ : new Date(appStatus.get("createTime").getAsLong()));
+
+ // lifetime - set it to unlimited for now
+ // TODO: Once YARN-3813 and YARN-4205 are available - get it from YARN
+ app.setLifetime(DEFAULT_UNLIMITED_LIFETIME);
+
+ // Quicklinks
+ Map<String, String> appQuicklinks = new HashMap<>();
+ for (Map.Entry<String, JsonElement> quicklink : appRegistryQuicklinks
+ .entrySet()) {
+ appQuicklinks.put(quicklink.getKey(), quicklink.getValue() == null ? null
+ : quicklink.getValue().getAsString());
+ }
+ if (!appQuicklinks.isEmpty()) {
+ app.setQuicklinks(appQuicklinks);
+ }
+
+ ArrayList<String> componentNames = new ArrayList<>();
+
+ // status.live
+ JsonObject applicationStatus = jsonGetAsObject(appStatus, "status");
+ // roles
+ JsonObject applicationRoles = jsonGetAsObject(appStatus, "roles");
+ // statistics
+ JsonObject applicationStatistics = jsonGetAsObject(appStatus, "statistics");
+ if (applicationRoles == null) {
+ // initialize to empty object to avoid too many null checks
+ applicationRoles = new JsonObject();
+ }
+ if (applicationStatus != null) {
+ JsonObject applicationLive = jsonGetAsObject(applicationStatus, "live");
+ if (applicationLive != null) {
+ for (Entry<String, JsonElement> entry : applicationLive.entrySet()) {
+ if (entry.getKey().equals(SLIDER_APPMASTER_COMPONENT_NAME)) {
+ continue;
+ }
+ componentNames.add(entry.getKey());
+ JsonObject componentRole = applicationRoles.get(entry.getKey()) == null ? new JsonObject()
+ : applicationRoles.get(entry.getKey()).getAsJsonObject();
+ JsonObject liveContainers = entry.getValue().getAsJsonObject();
+ if (liveContainers != null) {
+ for (Map.Entry<String, JsonElement> liveContainerEntry : liveContainers
+ .entrySet()) {
+ String containerId = liveContainerEntry.getKey();
+ Container container = new Container();
+ container.setId(containerId);
+ JsonObject liveContainer = (JsonObject) liveContainerEntry
+ .getValue();
+ container
+ .setLaunchTime(liveContainer.get("startTime") == null ? null
+ : new Date(liveContainer.get("startTime").getAsLong()));
+ container
+ .setComponentName(jsonGetAsString(liveContainer, "role"));
+ container.setIp(jsonGetAsString(liveContainer, "ip"));
+ // If ip is non-null increment count
+ if (container.getIp() != null) {
+ totalNumberOfIpAssignedContainers++;
+ }
+ container.setHostname(jsonGetAsString(liveContainer, "hostname"));
+ container.setState(ContainerState.INIT);
+ if (StringUtils.isNotEmpty(container.getIp())
+ && StringUtils.isNotEmpty(container.getHostname())) {
+ container.setState(ContainerState.READY);
+ }
+ container.setBareHost(jsonGetAsString(liveContainer, "host"));
+ container.setUri(CONTEXT_ROOT + APPLICATIONS_API_RESOURCE_PATH
+ + "/" + appName + CONTAINERS_API_RESOURCE_PATH + "/"
+ + containerId);
+ Resource resource = new Resource();
+ resource.setCpus(jsonGetAsInt(componentRole, "yarn.vcores"));
+ resource.setMemory(jsonGetAsString(componentRole, "yarn.memory"));
+ container.setResource(resource);
+ // TODO: add container property - for response only?
+ app.addContainer(container);
+ }
+ }
+ }
+ }
+ }
+
+ // application info
+ if (applicationRoles != null && !componentNames.isEmpty()) {
+ JsonObject applicationRole = jsonGetAsObject(applicationRoles,
+ componentNames.get(0));
+ if (applicationRole != null) {
+ Artifact artifact = new Artifact();
+ // how to get artifact id - docker image name??
+ artifact.setId(null);
+ }
+ }
+
+ // actual and expected number of containers
+ if (applicationStatistics != null) {
+ for (Entry<String, JsonElement> entry : applicationStatistics.entrySet()) {
+ if (entry.getKey().equals(SLIDER_APPMASTER_COMPONENT_NAME)) {
+ continue;
+ }
+ JsonObject containerStats = (JsonObject) entry.getValue();
+ totalNumberOfRunningContainers += jsonGetAsInt(containerStats,
+ "containers.live");
+ totalExpectedNumberOfRunningContainers += jsonGetAsInt(containerStats,
+ "containers.desired");
+ }
+ app.setNumberOfContainers(totalExpectedNumberOfRunningContainers);
+ app.setNumberOfRunningContainers(totalNumberOfRunningContainers);
+ }
+
+ // If all containers of the app has IP assigned, then according to the REST
+ // API it is considered to be READY. Note, application readiness from
+ // end-users point of view, is out of scope of the REST API. Also, this
+ // readiness has nothing to do with readiness-check defined at the component
+ // level (which is used for dependency resolution of component DAG).
+ if (totalNumberOfIpAssignedContainers == totalExpectedNumberOfRunningContainers) {
+ app.setState(ApplicationState.READY);
+ }
+ logger.info("Application = {}", app);
+ return Response.ok(app).build();
+ }
+
+ private String jsonGetAsString(JsonObject object, String key) {
+ return object.get(key) == null ? null : object.get(key).getAsString();
+ }
+
+ private Integer jsonGetAsInt(JsonObject object, String key) {
+ return object.get(key) == null ? null
+ : object.get(key).isJsonNull() ? null : object.get(key).getAsInt();
+ }
+
+ private JsonObject jsonGetAsObject(JsonObject object, String key) {
+ return object.get(key) == null ? null : object.get(key).getAsJsonObject();
+ }
+
+ private JsonObject getSliderApplicationStatus(final String appName)
+ throws IOException, YarnException, InterruptedException {
+ final File appStatusOutputFile = File.createTempFile("status_", ".json");
+ final ActionStatusArgs statusArgs = new ActionStatusArgs();
+ statusArgs.output = appStatusOutputFile.getAbsolutePath();
+
+ return invokeSliderClientRunnable(new SliderClientContextRunnable<JsonObject>() {
+ @Override
+ public JsonObject run(SliderClient sliderClient) throws YarnException,
+ IOException, InterruptedException {
+ sliderClient.actionStatus(appName, statusArgs);
+ JsonParser parser = new JsonParser();
+ FileReader reader = null;
+ JsonElement statusElement = null;
+ try {
+ reader = new FileReader(appStatusOutputFile);
+ statusElement = parser.parse(reader);
+ } finally {
+ if (reader != null) {
+ reader.close();
+ }
+ appStatusOutputFile.delete();
+ }
+ return (statusElement == null || statusElement instanceof JsonNull) ?
+ new JsonObject() : (JsonObject) statusElement;
+ }
+ });
+ }
+
+ private JsonObject getSliderApplicationRegistry(final String appName,
+ final String registryName) throws IOException, YarnException,
+ InterruptedException {
+ final File appRegistryOutputFile = File
+ .createTempFile("registry_", ".json");
+ final ActionRegistryArgs registryArgs = new ActionRegistryArgs();
+ registryArgs.out = appRegistryOutputFile;
+ registryArgs.name = appName;
+ registryArgs.getConf = registryName;
+ registryArgs.format = ConfigFormat.JSON.toString();
+
+ return invokeSliderClientRunnable(new SliderClientContextRunnable<JsonObject>() {
+ @Override
+ public JsonObject run(SliderClient sliderClient) throws YarnException,
+ IOException, InterruptedException {
+ sliderClient.actionRegistry(registryArgs);
+ JsonParser parser = new JsonParser();
+ FileReader reader = null;
+ JsonElement registryElement = null;
+ try {
+ reader = new FileReader(appRegistryOutputFile);
+ registryElement = parser.parse(reader);
+ } catch (Throwable t) {
+ logger.error("Error reading file {}", appRegistryOutputFile);
+ } finally {
+ if (reader != null) {
+ reader.close();
+ }
+ appRegistryOutputFile.delete();
+ }
+ return (registryElement == null || registryElement instanceof JsonNull) ?
+ new JsonObject() : (JsonObject) registryElement;
+ }
+ });
+ }
+
+ private Integer getSliderList(final String appName)
+ throws IOException, YarnException, InterruptedException {
+ return getSliderList(appName, true);
+ }
+
+ private Integer getSliderList(final String appName, final boolean liveOnly)
+ throws IOException, YarnException, InterruptedException {
+ return invokeSliderClientRunnable(new SliderClientContextRunnable<Integer>() {
+ @Override
+ public Integer run(SliderClient sliderClient) throws YarnException,
+ IOException, InterruptedException {
+ int status = 0;
+ if (liveOnly) {
+ status = sliderClient.actionList(appName);
+ } else {
+ ActionListArgs listArgs = new ActionListArgs();
+ status = sliderClient.actionList(appName, listArgs);
+ }
+ return status;
+ }
+ });
+ }
+
+ private Set<String> getSliderApplications(final String state)
+ throws IOException, YarnException, InterruptedException {
+ return getSliderApplications(false, state);
+ }
+
+ private Set<String> getSliderApplications(final boolean liveOnly)
+ throws IOException, YarnException, InterruptedException {
+ return getSliderApplications(liveOnly, null);
+ }
+
+ private Set<String> getSliderApplications(final boolean liveOnly,
+ final String state) throws IOException, YarnException,
+ InterruptedException {
+ return invokeSliderClientRunnable(new SliderClientContextRunnable<Set<String>>() {
+ @Override
+ public Set<String> run(SliderClient sliderClient) throws YarnException,
+ IOException, InterruptedException {
+ Set<String> apps;
+ ActionListArgs listArgs = new ActionListArgs();
+ if (liveOnly) {
+ apps = sliderClient.getApplicationList(null);
+ } else if (StringUtils.isNotEmpty(state)) {
+ listArgs.state = state;
+ apps = sliderClient.getApplicationList(null, listArgs);
+ } else {
+ apps = sliderClient.getApplicationList(null, listArgs);
+ }
+ return apps;
+ }
+ });
+ }
+
+ @DELETE
+ @Path("/{app_name}")
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Response deleteApplication(@PathParam("app_name") String appName) {
+ logger.info("DELETE: deleteApplication for appName = {}", appName);
+
+ try {
+ Response stopResponse = stopSliderApplication(appName);
+ if (stopResponse.getStatus() == Status.INTERNAL_SERVER_ERROR
+ .getStatusCode()) {
+ return Response.status(Status.NOT_FOUND).build();
+ }
+ } catch (UnknownApplicationInstanceException e) {
+ logger.error("Application does not exist", e);
+ return Response.status(Status.NOT_FOUND).build();
+ } catch (Exception e) {
+ logger.error("Delete application failed", e);
+ return Response.status(Status.INTERNAL_SERVER_ERROR).build();
+ }
+
+ // Although slider client stop returns immediately, it usually takes a
+ // little longer for it to stop from YARN point of view. Slider destroy
+ // fails if the application is not completely stopped. Hence the need to
+ // call destroy in a controlled loop few times (only if exit code is
+ // EXIT_APPLICATION_IN_USE), before giving up.
+ boolean keepTrying = true;
+ int maxDeleteAttempt = 5;
+ int deleteAttempt = 0;
+ while (keepTrying && deleteAttempt < maxDeleteAttempt) {
+ try {
+ destroySliderApplication(appName);
+ keepTrying = false;
+ } catch (SliderException e) {
+ logger.error("Delete application threw exception", e);
+ if (e.getExitCode() == SliderExitCodes.EXIT_APPLICATION_IN_USE) {
+ deleteAttempt++;
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e1) {
+ }
+ } else {
+ return Response.status(Status.INTERNAL_SERVER_ERROR).build();
+ }
+ } catch (Exception e) {
+ logger.error("Delete application failed", e);
+ return Response.status(Status.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+ return Response.status(Status.NO_CONTENT).build();
+ }
+
+ private Response stopSliderApplication(final String appName)
+ throws IOException, YarnException, InterruptedException {
+ return invokeSliderClientRunnable(new SliderClientContextRunnable<Response>() {
+ @Override
+ public Response run(SliderClient sliderClient) throws YarnException,
+ IOException, InterruptedException {
+ ActionFreezeArgs freezeArgs = new ActionFreezeArgs();
+ int returnCode = sliderClient.actionFreeze(appName, freezeArgs);
+ if (returnCode == 0) {
+ logger.info("Successfully stopped application {}", appName);
+ return Response.status(Status.NO_CONTENT).build();
+ } else {
+ logger.error("Stop of application {} failed with return code ",
+ appName, returnCode);
+ ApplicationStatus applicationStatus = new ApplicationStatus();
+ applicationStatus.setDiagnostics("Stop of application " + appName
+ + " failed");
+ return Response.status(Status.INTERNAL_SERVER_ERROR)
+ .entity(applicationStatus).build();
+ }
+ }
+ });
+ }
+
+ private Response startSliderApplication(final String appName)
+ throws IOException, YarnException, InterruptedException {
+ return invokeSliderClientRunnable(new SliderClientContextRunnable<Response>() {
+ @Override
+ public Response run(SliderClient sliderClient) throws YarnException,
+ IOException, InterruptedException {
+ ActionThawArgs thawArgs = new ActionThawArgs();
+ int returnCode = sliderClient.actionThaw(appName, thawArgs);
+ if (returnCode == 0) {
+ logger.info("Successfully started application {}", appName);
+ ApplicationStatus applicationStatus = new ApplicationStatus();
+ applicationStatus.setState(ApplicationState.ACCEPTED);
+ applicationStatus.setUri(CONTEXT_ROOT
+ + APPLICATIONS_API_RESOURCE_PATH + "/" + appName);
+ // 202 = ACCEPTED
+ return Response.status(HTTP_STATUS_CODE_ACCEPTED)
+ .entity(applicationStatus).build();
+ } else {
+ logger.error("Start of application {} failed with returnCode ",
+ appName, returnCode);
+ ApplicationStatus applicationStatus = new ApplicationStatus();
+ applicationStatus.setDiagnostics("Start of application " + appName
+ + " failed");
+ return Response.status(Status.INTERNAL_SERVER_ERROR)
+ .entity(applicationStatus).build();
+ }
+ }
+ });
+ }
+
+ private Void destroySliderApplication(final String appName)
+ throws IOException, YarnException, InterruptedException {
+ return invokeSliderClientRunnable(new SliderClientContextRunnable<Void>() {
+ @Override
+ public Void run(SliderClient sliderClient) throws YarnException,
+ IOException, InterruptedException {
+ sliderClient.actionDestroy(appName);
+ return null;
+ }
+ });
+ }
+
+ @PUT
+ @Path("/{app_name}")
+ @Consumes({ MediaType.APPLICATION_JSON })
+ @Produces({ MediaType.APPLICATION_JSON })
+ public Response updateApplication(@PathParam("app_name") String appName,
+ Application updateAppData) {
+ logger.info("PUT: updateApplication for app = {} with data = {}", appName,
+ updateAppData);
+
+ // Ignore the app name provided in updateAppData and always use appName
+ // path param
+ updateAppData.setName(appName);
+
+ // Adding support for stop and start
+ // For STOP the app should be running. If already stopped then this
+ // operation will be a no-op. For START it should be in stopped state.
+ // If already running then this operation will be a no-op.
+
+ // Check if app exists in any state
+ try {
+ int appsFound = getSliderList(appName, false);
+ if (appsFound < 0) {
+ return Response.status(Status.NOT_FOUND).build();
+ }
+ } catch (Exception e) {
+ logger.error("Update application failed", e);
+ return Response.status(Status.NOT_FOUND).build();
+ }
+
+ // If a STOP is requested
+ if (updateAppData.getState() != null
+ && updateAppData.getState() == ApplicationState.STOPPED) {
+ try {
+ int livenessCheck = getSliderList(appName);
+ if (livenessCheck == 0) {
+ return stopSliderApplication(appName);
+ } else {
+ logger.info("Application {} is already stopped", appName);
+ ApplicationStatus applicationStatus = new ApplicationStatus();
+ applicationStatus.setDiagnostics("Application " + appName
+ + " is already stopped");
+ return Response.status(Status.BAD_REQUEST).entity(applicationStatus)
+ .build();
+ }
+ } catch (Exception e) {
+ logger.error("Stop application failed", e);
+ return Response.status(Status.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+
+ // If a START is requested
+ if (updateAppData.getState() != null
+ && updateAppData.getState() == ApplicationState.STARTED) {
+ try {
+ int livenessCheck = getSliderList(appName);
+ if (livenessCheck != 0) {
+ return startSliderApplication(appName);
+ } else {
+ logger.info("Application {} is already running", appName);
+ ApplicationStatus applicationStatus = new ApplicationStatus();
+ applicationStatus.setDiagnostics("Application " + appName
+ + " is already running");
+ applicationStatus.setUri(CONTEXT_ROOT
+ + APPLICATIONS_API_RESOURCE_PATH + "/" + appName);
+ return Response.status(Status.BAD_REQUEST).entity(applicationStatus)
+ .build();
+ }
+ } catch (Exception e) {
+ logger.error("Start application failed", e);
+ return Response.status(Status.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+
+ // If no of instances specified then treat it as a flex
+ if (updateAppData.getNumberOfContainers() != null
+ && updateAppData.getComponents() == null) {
+ updateAppData.setComponents(getDefaultComponentAsList());
+ }
+
+ // At this point if there are components then it is a flex
+ if (updateAppData.getComponents() != null) {
+ try {
+ int livenessCheck = getSliderList(appName);
+ if (livenessCheck == 0) {
+ flexSliderApplication(appName, updateAppData);
+ }
+ return Response.status(Status.NO_CONTENT).build();
+ } catch (Exception e) {
+ logger.error("Update application failed", e);
+ return Response.status(Status.INTERNAL_SERVER_ERROR).build();
+ }
+ }
+
+ // If new lifetime value specified then update it
+ if (updateAppData.getLifetime() != null) {
+ // TODO: Once YARN-3813 and YARN-4205 are available
+ }
+
+ // If nothing happens consider it a no-op
+ return Response.status(Status.NO_CONTENT).build();
+ }
+
+ private List<Component> getDefaultComponentAsList() {
+ Component comp = new Component();
+ comp.setName(DEFAULT_COMPONENT_NAME);
+ List<Component> comps = new ArrayList<>();
+ comps.add(comp);
+ return comps;
+ }
+
+ private Void flexSliderApplication(final String appName,
+ final Application updateAppData) throws IOException, YarnException,
+ InterruptedException {
+ return invokeSliderClientRunnable(new SliderClientContextRunnable<Void>() {
+ @Override
+ public Void run(SliderClient sliderClient) throws YarnException,
+ IOException, InterruptedException {
+ ActionFlexArgs flexArgs = new ActionFlexArgs();
+ ComponentArgsDelegate compDelegate = new ComponentArgsDelegate();
+ Long globalNumberOfContainers = updateAppData.getNumberOfContainers();
+ for (Component comp : updateAppData.getComponents()) {
+ Long noOfContainers = comp.getNumberOfContainers() == null
+ ? globalNumberOfContainers : comp.getNumberOfContainers();
+ if (noOfContainers != null) {
+ compDelegate.componentTuples.addAll(
+ Arrays.asList(comp.getName(), String.valueOf(noOfContainers)));
+ }
+ }
+ if (!compDelegate.componentTuples.isEmpty()) {
+ flexArgs.componentDelegate = compDelegate;
+ sliderClient.actionFlex(appName, flexArgs);
+ }
+ return null;
+ }
+ });
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/51] [abbrv] hadoop git commit: YARN-5690. Integrate native
services modules into maven build. Contributed by Billie Rinaldi
Posted by ji...@apache.org.
YARN-5690. Integrate native services modules into maven build. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6c857ea7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6c857ea7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6c857ea7
Branch: refs/heads/yarn-native-services
Commit: 6c857ea778f4a04926213910438cd86077c5f5ca
Parents: 31848c7
Author: Gour Saha <go...@apache.org>
Authored: Thu Oct 27 08:50:36 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../resources/assemblies/hadoop-yarn-dist.xml | 26 ++++++++
.../assemblies/hadoop-yarn-services-api.xml | 36 +++++++++++
.../assemblies/hadoop-yarn-slider-dist.xml | 30 +++++++++
hadoop-project/pom.xml | 6 ++
hadoop-yarn-project/hadoop-yarn/bin/yarn | 30 +++++++++
.../hadoop-yarn-services-api/pom.xml | 44 +++++++++++--
.../conf/slideram-log4j.properties | 68 ++++++++++++++++++++
.../hadoop-yarn-slider-core/pom.xml | 38 +++++++++++
.../org/apache/slider/client/SliderClient.java | 29 +++++++--
.../org/apache/slider/common/SliderKeys.java | 2 +-
.../apache/slider/common/tools/SliderUtils.java | 48 ++++++++++----
.../providers/agent/AgentClientProvider.java | 3 +-
.../slideram/SliderAMClientProvider.java | 15 +++--
.../TestPublishedConfigurationOutputter.java | 10 ++-
14 files changed, 343 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index c3f459c..552087c 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -86,6 +86,32 @@
</includes>
</fileSet>
<fileSet>
+ <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target</directory>
+ <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
+ <includes>
+ <include>*-sources.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf</directory>
+ <outputDirectory>etc/hadoop</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target/hadoop-yarn-slider-core-${project.version}</directory>
+ <outputDirectory>/share/hadoop/${hadoop.component}/lib/slider</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target</directory>
+ <outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
+ <includes>
+ <include>*-sources.jar</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target/hadoop-yarn-services-api-${project.version}</directory>
+ <outputDirectory>/share/hadoop/${hadoop.component}/lib/services-api</outputDirectory>
+ </fileSet>
+ <fileSet>
<directory>hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/target</directory>
<outputDirectory>/share/hadoop/${hadoop.component}/sources</outputDirectory>
<includes>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
new file mode 100644
index 0000000..589f724
--- /dev/null
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
@@ -0,0 +1,36 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.01
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <id>hadoop-yarn-services-api-dist</id>
+ <formats>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ <includes>
+ <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base</include>
+ <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider</include>
+ <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations</include>
+ <include>io.swagger:swagger-annotations</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml
new file mode 100644
index 0000000..5de45a9
--- /dev/null
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml
@@ -0,0 +1,30 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.01
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ <id>hadoop-yarn-slider-dist</id>
+ <formats>
+ <format>dir</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <dependencySets>
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ </dependencySet>
+ </dependencySets>
+</assembly>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index a633873..409dd93 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -385,6 +385,12 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-slider-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 804fd1a..26d54b8 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -47,7 +47,9 @@ function hadoop_usage
hadoop_add_subcommand "resourcemanager" "run the ResourceManager"
hadoop_add_subcommand "rmadmin" "admin tools"
hadoop_add_subcommand "scmadmin" "SharedCacheManager admin tools"
+ hadoop_add_subcommand "services-api" "run slider services api"
hadoop_add_subcommand "sharedcachemanager" "run the SharedCacheManager daemon"
+ hadoop_add_subcommand "slider" "run a slider app"
hadoop_add_subcommand "timelinereader" "run the timeline reader server"
hadoop_add_subcommand "timelineserver" "run the timeline server"
hadoop_add_subcommand "top" "view cluster information"
@@ -142,10 +144,38 @@ function yarncmd_case
scmadmin)
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
;;
+ services-api)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
+ hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services-api"'/*'
+ HADOOP_CLASSNAME='org.apache.hadoop.yarn.services.webapp.ApplicationApiWebApp'
+ hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS} \
+-Dslider.libdir=${HADOOP_YARN_HOME}/${YARN_DIR},\
+${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
+${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
+${HADOOP_HDFS_HOME}/${HDFS_DIR},\
+${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
+${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
+${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
+ ;;
sharedcachemanager)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager'
;;
+ slider)
+ hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
+ HADOOP_CLASSNAME='org.apache.slider.Slider'
+ hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS} \
+-Dslider.libdir=${HADOOP_YARN_HOME}/${YARN_DIR},\
+${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
+${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
+${HADOOP_HDFS_HOME}/${HDFS_DIR},\
+${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
+${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
+${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
+ ;;
timelinereader)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer'
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
index 78b7855..c198f28 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -159,12 +159,6 @@
<dependency>
<groupId>com.fasterxml.jackson.jaxrs</groupId>
<artifactId>jackson-jaxrs-json-provider</artifactId>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.jaxrs</groupId>
- <artifactId>jackson-jaxrs-base</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
@@ -192,6 +186,44 @@
</dependencies>
<profiles>
+ <profile>
+ <id>dist</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-assemblies</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <id>dist</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <appendAssemblyId>false</appendAssemblyId>
+ <attach>false</attach>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ <descriptorRefs>
+ <descriptorRef>hadoop-yarn-services-api</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
<profile>
<id>rat</id>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties
new file mode 100644
index 0000000..333859e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is the log4j configuration for Slider Application Master
+
+# Log rotation based on size (100KB) with a max of 10 backup files
+log4j.rootLogger=INFO, amlog
+log4j.threshhold=ALL
+log4j.appender.amlog=org.apache.log4j.RollingFileAppender
+log4j.appender.amlog.layout=org.apache.log4j.PatternLayout
+log4j.appender.amlog.File=${LOG_DIR}/slider.log
+log4j.appender.amlog.MaxFileSize=1MB
+log4j.appender.amlog.MaxBackupIndex=10
+
+# log layout skips stack-trace creation operations by avoiding line numbers and method
+log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+# debug edition is much more expensive
+#log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+# configure stderr
+# set the conversion pattern of stderr
+# Print the date in ISO 8601 format
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.Target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess
+
+# for debugging Slider
+#log4j.logger.org.apache.slider=DEBUG
+
+# uncomment to debug service lifecycle issues
+#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+
+# uncomment for YARN operations
+#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+
+# uncomment this to debug security problems
+#log4j.logger.org.apache.hadoop.security=DEBUG
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs=WARN
+log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR
+
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.curator.framework.state=ERROR
+log4j.logger.org.apache.curator.framework.imps=WARN
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index d778f44..a2c67c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -332,6 +332,44 @@
<profiles>
<profile>
+ <id>dist</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-assemblies</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ <executions>
+ <execution>
+ <id>dist</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <appendAssemblyId>false</appendAssemblyId>
+ <attach>false</attach>
+ <finalName>${project.artifactId}-${project.version}</finalName>
+ <descriptorRefs>
+ <descriptorRef>hadoop-yarn-slider-dist</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
<id>compile-protobuf</id>
<build>
<plugins>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 4b546cd..d1f88c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -2162,6 +2162,24 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
new File(confDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME);
hasServerLog4jProperties = log4jserver.isFile();
}
+ if (!hasServerLog4jProperties) {
+ // check for log4j properties in hadoop conf dir
+ String hadoopConfDir = System.getenv(ApplicationConstants.Environment
+ .HADOOP_CONF_DIR.name());
+ if (hadoopConfDir != null) {
+ File localFile = new File(hadoopConfDir, SliderKeys
+ .LOG4J_SERVER_PROP_FILENAME);
+ if (localFile.exists()) {
+ Path localFilePath = createLocalPath(localFile);
+ remoteConfPath = new Path(clusterDirectory,
+ SliderKeys.SUBMITTED_CONF_DIR);
+ Path remoteFilePath = new Path(remoteConfPath, SliderKeys
+ .LOG4J_SERVER_PROP_FILENAME);
+ copy(config, localFilePath, remoteFilePath);
+ hasServerLog4jProperties = true;
+ }
+ }
+ }
// the assumption here is that minimr cluster => this is a test run
// and the classpath can look after itself
@@ -2300,7 +2318,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// enable asserts
commandLine.enableJavaAssertions();
- // if the conf dir has a log4j-server.properties, switch to that
+ // if the conf dir has a slideram-log4j.properties, switch to that
if (hasServerLog4jProperties) {
commandLine.sysprop(SYSPROP_LOG4J_CONFIGURATION, LOG4J_SERVER_PROP_FILENAME);
commandLine.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
@@ -4471,14 +4489,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return EXIT_SUCCESS;
}
- String libDir = System.getProperty(SliderKeys.PROPERTY_LIB_DIR);
- if (isSet(libDir)) {
- File srcFolder = new File(libDir);
+ String[] libDirs = SliderUtils.getLibDirs();
+ if (libDirs.length > 0) {
File tempLibTarGzipFile = File.createTempFile(
SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME + "_",
SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT);
- // copy all jars except slider-core-<version>.jar
- tarGzipFolder(srcFolder, tempLibTarGzipFile, createJarFilter());
+ // copy all jars
+ tarGzipFolder(libDirs, tempLibTarGzipFile, createJarFilter());
log.info("Uploading dependency for AM (version {}) from {} to {}",
version, tempLibTarGzipFile.toURI(), dependencyLibTarGzip.toUri());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 1484ee3..3d25d33 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -182,7 +182,7 @@ public interface SliderKeys extends SliderXmlConfKeys {
/**
* Slider AM log4j file name : {@value}
*/
- String LOG4J_SERVER_PROP_FILENAME = "log4j-server.properties";
+ String LOG4J_SERVER_PROP_FILENAME = "slideram-log4j.properties";
/**
* Standard log4j file name : {@value}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index f773982..713cd02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -25,6 +25,7 @@ import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
import org.apache.commons.io.output.ByteArrayOutputStream;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -1399,6 +1400,22 @@ public final class SliderUtils {
}
}
+ public static String getLibDir() {
+ String[] libDirs = getLibDirs();
+ if (libDirs == null || libDirs.length == 0) {
+ return null;
+ }
+ return libDirs[0];
+ }
+
+ public static String[] getLibDirs() {
+ String libDirStr = System.getProperty(SliderKeys.PROPERTY_LIB_DIR);
+ if (isUnset(libDirStr)) {
+ return ArrayUtils.EMPTY_STRING_ARRAY;
+ }
+ return StringUtils.split(libDirStr, ',');
+ }
+
/**
* Submit a JAR containing a specific class and map it
* @param providerResources provider map to build up
@@ -1962,31 +1979,34 @@ public final class SliderUtils {
/**
* Given a source folder create a tar.gz file
*
- * @param srcFolder
+ * @param libDirs
* @param tarGzipFile
*
* @throws IOException
*/
- public static void tarGzipFolder(File srcFolder, File tarGzipFile,
+ public static void tarGzipFolder(String[] libDirs, File tarGzipFile,
FilenameFilter filter) throws IOException {
- log.info("Tar-gzipping folder {} to {}", srcFolder.getAbsolutePath(),
+ log.info("Tar-gzipping folders {} to {}", libDirs,
tarGzipFile.getAbsolutePath());
- List<String> files = new ArrayList<>();
- generateFileList(files, srcFolder, srcFolder, true, filter);
try(TarArchiveOutputStream taos =
new TarArchiveOutputStream(new GZIPOutputStream(
new BufferedOutputStream(new FileOutputStream(tarGzipFile))))) {
- for (String file : files) {
- File srcFile = new File(srcFolder, file);
- TarArchiveEntry tarEntry = new TarArchiveEntry(
- srcFile, file);
- taos.putArchiveEntry(tarEntry);
- try(FileInputStream in = new FileInputStream(srcFile)) {
- org.apache.commons.io.IOUtils.copy(in, taos);
+ for (String libDir : libDirs) {
+ File srcFolder = new File(libDir);
+ List<String> files = new ArrayList<>();
+ generateFileList(files, srcFolder, srcFolder, true, filter);
+ for (String file : files) {
+ File srcFile = new File(srcFolder, file);
+ TarArchiveEntry tarEntry = new TarArchiveEntry(
+ srcFile, file);
+ taos.putArchiveEntry(tarEntry);
+ try(FileInputStream in = new FileInputStream(srcFile)) {
+ org.apache.commons.io.IOUtils.copy(in, taos);
+ }
+ taos.flush();
+ taos.closeArchiveEntry();
}
- taos.flush();
- taos.closeArchiveEntry();
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index fdc5be1..7ca469f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -562,8 +562,7 @@ public class AgentClientProvider extends AbstractClientProvider
}
private void expandAgentTar(File agentPkgDir) throws IOException {
- String libDirProp =
- System.getProperty(PROPERTY_LIB_DIR);
+ String libDirProp = SliderUtils.getLibDir();
File tarFile = new File(libDirProp, AGENT_TAR);
expandTar(tarFile, agentPkgDir);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
index e5430f2..b58d3aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
@@ -184,19 +184,20 @@ public class SliderAMClientProvider extends AbstractClientProvider
libdir,
miniClusterTestRun);
- String libDirProp =
- System.getProperty(SliderKeys.PROPERTY_LIB_DIR);
log.info("Loading all dependencies for AM.");
// If slider.tar.gz is available in hdfs use it, else upload all jars
Path dependencyLibTarGzip = fileSystem.getDependencyTarGzip();
if (fileSystem.isFile(dependencyLibTarGzip)) {
SliderUtils.putAmTarGzipAndUpdate(providerResources, fileSystem);
} else {
- ProviderUtils.addAllDependencyJars(providerResources,
- fileSystem,
- tempPath,
- libdir,
- libDirProp);
+ for (String libDirProp : SliderUtils.getLibDirs()) {
+ ProviderUtils.addAllDependencyJars(providerResources,
+ fileSystem,
+ tempPath,
+ libdir,
+ libDirProp);
+
+ }
}
addKeytabResourceIfNecessary(fileSystem,
instanceDescription,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c857ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
index 3706182..56b84e0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
@@ -82,16 +82,14 @@ public class TestPublishedConfigurationOutputter {
String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
"");
- assert output.contains(
- "<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>");
+ assert output.contains("<name>key1</name><value>val1</value>");
File file = tmpDir.newFile();
configurationOutputter.save(file);
assert FileUtils.readFileToString(file, Charsets.UTF_8)
.replaceAll("( |\\r|\\n)", "")
- .contains(
- "<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>");
+ .contains("<name>key1</name><value>val1</value>");
}
@Test
@@ -103,14 +101,14 @@ public class TestPublishedConfigurationOutputter {
String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
"");
- assert output.contains("<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>");
+ assert output.contains("<name>key1</name><value>val1</value>");
File file = tmpDir.newFile();
configurationOutputter.save(file);
assert FileUtils.readFileToString(file, Charsets.UTF_8)
.replaceAll("( |\\r|\\n)", "")
- .contains( "<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>");
+ .contains("<name>key1</name><value>val1</value>");
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/51] [abbrv] hadoop git commit: YARN-5958. Fix ASF license
warnings for slider core module. Contributed by Billie Rinaldi
Posted by ji...@apache.org.
YARN-5958. Fix ASF license warnings for slider core module. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c1459aa6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c1459aa6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c1459aa6
Branch: refs/heads/yarn-native-services
Commit: c1459aa6db4fc1bd855039e840eface526091138
Parents: 03a3258
Author: Gour Saha <go...@apache.org>
Authored: Thu Dec 1 17:45:44 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../hadoop-yarn-slider-core/pom.xml | 61 +++++---------------
.../src/license/THIRD-PARTY.properties | 33 -----------
2 files changed, 14 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1459aa6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index 66e9ee9..10cf6b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -78,6 +78,20 @@
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>**/*.json</exclude>
+ <exclude>src/main/resources/webapps/slideram/.keep</exclude>
+ <!-- protobuf generated classes -->
+ <exclude>src/main/java/org/apache/slider/api/proto/Messages.java</exclude>
+ <exclude>src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+
</plugins>
</build>
<dependencies>
@@ -384,53 +398,6 @@
</build>
</profile>
-
- <profile>
- <id>rat</id>
- <build>
- <plugins>
-
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <executions>
- <execution>
- <id>check-licenses</id>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <excludes>
- <exclude>**/*.json</exclude>
- <exclude>src/test/python/agent.ini</exclude>
- <exclude>src/test/python/version</exclude>
- <exclude>**/THIRD-PARTY.properties</exclude>
- <exclude>src/main/resources/webapps/slideram/.keep</exclude>
- <exclude>src/main/resources/webapps/slideragent/.keep</exclude>
- <exclude>src/main/resources/webapps/static/yarn.dt.plugins.js</exclude>
- <!-- jQuery DataTables files (BSD license) -->
- <exclude>src/main/resources/webapps/static/dt-1.9.4/**</exclude>
- <!-- jQuery (MIT license) -->
- <exclude>src/main/resources/webapps/static/jquery/jquery-1.8.2.min.js</exclude>
- <!-- jQuery UI (MIT license) -->
- <exclude>src/main/resources/webapps/static/jquery/jquery-ui-1.9.1.custom.min.js</exclude>
- <exclude>src/main/resources/webapps/static/jquery/themes-1.9.1/base/jquery-ui.css</exclude>
- <!-- jQuery jsTree (MIT license) -->
- <exclude>src/main/resources/webapps/static/jt/jquery.jstree.js</exclude>
- <!-- protobuf generated classes -->
- <exclude>src/main/java/org/apache/slider/api/proto/Messages.java</exclude>
- <exclude>src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java</exclude>
- <exclude>src/test/app_packages/test_am_config/resources/test.template</exclude>
- <exclude>src/test/app_packages/test_am_config/test_archive/testfile</exclude>
- </excludes>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
-
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c1459aa6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/license/THIRD-PARTY.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/license/THIRD-PARTY.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/license/THIRD-PARTY.properties
deleted file mode 100644
index 1abd56e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/license/THIRD-PARTY.properties
+++ /dev/null
@@ -1,33 +0,0 @@
-# Generated by org.codehaus.mojo.license.AddThirdPartyMojo
-#-------------------------------------------------------------------------------
-# Already used licenses in project :
-# - Apache License
-# - BSD
-# - CDDL + GPLv2 with classpath exception
-# - CDDL 1.1
-# - CDDL License
-# - CDDL+GPL
-# - Common Public License Version 1.0
-# - Eclipse Public License - Version 1.0
-# - GNU Lesser General Public License (LGPL), Version 2.1
-# - GNU Lesser General Public License, Version 2.1
-# - GPL2 w/ CPE
-# - MIT License
-# - MPL 1.1
-# - New BSD License
-# - Public Domain
-# - Revised BSD
-# - The Apache Software License, Version 2.0
-# - The BSD 3-Clause License
-# - The BSD License
-# - The MIT License
-#-------------------------------------------------------------------------------
-# Please fill the missing licenses for dependencies :
-#
-#
-#Thu Oct 15 16:45:02 EDT 2015
-commons-beanutils--commons-beanutils--1.7.0=The Apache Software License, Version 2.0
-javax.servlet--servlet-api--2.5=CDDL License
-javax.servlet.jsp--jsp-api--2.1=CDDL License
-org.apache.zookeeper--zookeeper--3.4.6=The Apache Software License, Version 2.0
-org.codehaus.jettison--jettison--1.1=The Apache Software License, Version 2.0
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/51] [abbrv] hadoop git commit: YARN-5675. Swagger definition for
YARN service API. Contributed by Gour Saha
Posted by ji...@apache.org.
YARN-5675. Swagger definition for YARN service API. Contributed by Gour Saha
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e2cfd761
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e2cfd761
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e2cfd761
Branch: refs/heads/yarn-native-services
Commit: e2cfd76147fbd86e82471ea0b8f93cb7b8fb6445
Parents: 0771075
Author: Jian He <ji...@apache.org>
Authored: Wed Oct 12 13:27:53 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
...RN-Simplified-V1-API-Layer-For-Services.yaml | 416 +++++++++++++++++++
1 file changed, 416 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2cfd761/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
new file mode 100644
index 0000000..6169fcd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -0,0 +1,416 @@
+# Hadoop YARN REST APIs for services v1 spec in YAML
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+swagger: '2.0'
+info:
+ title: "[YARN-4793] Simplified API layer for services and beyond"
+ description: |
+ Bringing a new service on YARN today is not a simple experience. The APIs of existing frameworks are either too low level (native YARN), require writing new code (for frameworks with programmatic APIs) or writing a complex spec (for declarative frameworks). In addition to building critical building blocks inside YARN (as part of other efforts at link:https://issues.apache.org/jira/browse/YARN-4692[YARN-4692]), there is a need for simplifying the user facing story for building services. Experience of projects like Apache Slider running real-life services like HBase, Storm, Accumulo, Solr etc, gives us some very good insights on how simplified APIs for services should look like.
+
+
+ To this end, we should look at a new simple-services API layer backed by REST interfaces. This API can be used to create and manage the lifecycle of YARN services. Services here can range from simple single-component apps to complex multi-component assemblies needing orchestration.
+
+
+ We should also look at making this a unified REST based entry point for other important features like resource-profile management (link:https://issues.apache.org/jira/browse/YARN-3926[YARN-3926]), package-definitions' lifecycle-management and service-discovery (link:https://issues.apache.org/jira/browse/YARN-913[YARN-913]/link:https://issues.apache.org/jira/browse/YARN-4757[YARN-4757]). We also need to flesh out its relation to our present much lower level REST APIs (link:https://issues.apache.org/jira/browse/YARN-1695[YARN-1695]) in YARN for application-submission and management.
+
+
+ This document spotlights on this specification. In most of the cases, the application owner will not be forced to make any changes to their application. This is primarily true if the application is packaged with containerization technologies like docker. Irrespective of how complex the application is, there will be hooks provided at appropriate layers to allow pluggable and customizable application behavior.
+
+ version: "1.0.0"
+ license:
+ name: Apache 2.0
+ url: http://www.apache.org/licenses/LICENSE-2.0.html
+# the domain of the service
+host: host.mycompany.com
+# array of all schemes that your API supports
+schemes:
+ - http
+ - https
+# will be prefixed to all paths
+basePath: /services/v1/
+consumes:
+ - application/json
+produces:
+ - application/json
+paths:
+ /applications:
+ get:
+ summary: List of applications/services running in the cluster
+ description: Get a list of all currently running applications (response includes a minimal projection of the application info). For more details do a GET on a specific application name.
+ responses:
+ 200:
+ description: An array of applications
+ schema:
+ type: array
+ items:
+ $ref: '#/definitions/Application'
+ default:
+ description: Unexpected error
+ schema:
+ $ref: '#/definitions/ApplicationStatus'
+ post:
+ summary: Create an application/service
+ description: Create an application. The request JSON is an Application object with details required for creation. If the request is successful it returns 202 Accepted. A success of this API only confirms success in submission of the application creation request. There is no guarantee that the application will actually reach a RUNNING state. Resource availability and several other factors determines if the application will be deployed in the cluster. It is expected that clients would subsequently call the GET API to get details of the application and determine its state.
+ parameters:
+ - name: Application
+ in: body
+ description: Application request object
+ required: true
+ schema:
+ $ref: '#/definitions/Application'
+ responses:
+ 202:
+ description: Request accepted
+ default:
+ description: Unexpected error
+ schema:
+ $ref: '#/definitions/ApplicationStatus'
+
+ /applications/{app_name}:
+ get:
+ summary: Get an application/service details
+ description: Return the details (including containers) of a running application
+ parameters:
+ - name: app_name
+ in: path
+ description: Application name
+ required: true
+ type: string
+ responses:
+ 200:
+ description: An application object
+ schema:
+ type: object
+ items:
+ $ref: '#/definitions/Application'
+ examples:
+ app_name: logsearch
+ artifact:
+ id: logsearch:latest
+ type: docker
+ 404:
+ description: Application does not exist
+ default:
+ description: Unexpected error
+ schema:
+ $ref: '#/definitions/ApplicationStatus'
+ put:
+ summary: Update an application/service or upgrade the binary version of the components of a running application
+ description: Update the runtime properties of an application. As of now, only update of lifetime and number of instances (flexing) of the components of an application is supported. The PUT operation is also used to orchestrate an upgrade of the application containers to a newer version of their artifacts.
+ parameters:
+ - name: app_name
+ in: path
+ description: Application name
+ required: true
+ type: string
+ responses:
+ 204:
+ description: Update or upgrade was successful
+ 404:
+ description: Application does not exist
+ default:
+ description: Unexpected error
+ schema:
+ $ref: '#/definitions/ApplicationStatus'
+ delete:
+ summary: Destroy application/service
+ description: Destroy an application and release all resources. This API might have to return JSON data providing location of logs, etc. Not finalized yet.
+ parameters:
+ - name: app_name
+ in: path
+ description: Application name
+ required: true
+ type: string
+ responses:
+ 204:
+ description: Destroy was successful
+ 404:
+ description: Application does not exist
+ default:
+ description: Unexpected error
+ schema:
+ $ref: '#/definitions/ApplicationStatus'
+
+definitions:
+ Application:
+ description: An Application resource has the following attributes.
+ required:
+ - name
+ properties:
+ name:
+ type: string
+ description: A unique application name.
+ id:
+ type: string
+ description: A unique application id.
+ artifact:
+ description: Artifact of single-component applications. Mandatory if components attribute is not specified.
+ $ref: '#/definitions/Artifact'
+ resource:
+ description: Resource of single-component applications or the global default for multi-component applications. Mandatory if it is a single-component application and if cpus and memory are not specified at the Application level.
+ $ref: '#/definitions/Resource'
+ launch_command:
+ type: string
+ description: The custom launch command of an application component (optional). If not specified for applications with docker images say, it will default to the default start command of the image. If there is a single component in this application, you can specify this without the need to have a 'components' section.
+ launch_time:
+ type: string
+ format: date
+ description: The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.
+ number_of_containers:
+ type: integer
+ format: int64
+ description: Number of containers for each app-component in the application. Each app-component can further override this app-level global default.
+ number_of_running_containers:
+ type: integer
+ format: int64
+ description: In get response this provides the total number of running containers for this application (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.
+ lifetime:
+ type: integer
+ format: int64
+ description: Life time (in seconds) of the application from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.
+ placement_policy:
+ description: Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the app owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to application owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.
+ $ref: '#/definitions/PlacementPolicy'
+ components:
+ description: Components of an application.
+ type: array
+ items:
+ $ref: '#/definitions/Component'
+ configuration:
+ description: Config properties of an application. Configurations provided at the application/global level are available to all the components. Specific properties can be overridden at the component level.
+ $ref: '#/definitions/Configuration'
+ containers:
+ description: Containers of a started application. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started application.
+ type: array
+ items:
+ $ref: '#/definitions/Container'
+ state:
+ description: State of the application. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started application.
+ $ref: '#/definitions/ApplicationState'
+ quicklinks:
+ type: object
+ description: A blob of key-value pairs of quicklinks to be exported for an application.
+ additionalProperties:
+ type: string
+ queue:
+ type: string
+ description: The YARN queue that this application should be submitted to.
+ Resource:
+ description:
+ Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or application. The resource specified at the app (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are exepected. It raises a validation exception otherwise.
+ properties:
+ profile:
+ type: string
+ description: Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc.
+ cpus:
+ type: integer
+ format: int32
+ description: Amount of vcores allocated to each container (optional but overrides cpus in profile if specified).
+ memory:
+ type: string
+ description: Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.
+ PlacementPolicy:
+ description: Placement policy of an instance of an application. This feature is in the works in YARN-4902.
+ properties:
+ label:
+ type: string
+ description: Assigns an app to a named partition of the cluster where the application desires to run (optional). If not specified all apps are submitted to a default label of the app owner. One or more labels can be setup for each application owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.
+ Artifact:
+ description: Artifact of an application component.
+ required:
+ - id
+ properties:
+ id:
+ type: string
+ description: Artifact id. Examples are package location uri for tarball based apps, image name for docker, etc.
+ type:
+ type: string
+ description: Artifact type, like docker, tarball, etc. (optional).
+ enum:
+ - docker
+ - tarball
+ - application
+ default: docker
+ uri:
+ type: string
+ description: Artifact location to support multiple artifact stores (optional).
+ Component:
+ description: One or more components of the application. If the application is HBase say, then the component can be a simple role like master or regionserver. If the application is a complex business webapp then a component can be other applications say Kafka or Storm. Thereby it opens up the support for complex and nested applications.
+ required:
+ - name
+ properties:
+ name:
+ type: string
+ description: Name of the application component (mandatory).
+ dependencies:
+ type: array
+ items:
+ type: string
+ description: An array of application components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of an application should be represented as a DAG.
+ readiness_check:
+ description: Readiness check for this app-component.
+ $ref: '#/definitions/ReadinessCheck'
+ artifact:
+ description: Artifact of the component (optional). If not specified, the application level global artifact takes effect.
+ $ref: '#/definitions/Artifact'
+ launch_command:
+ type: string
+ description: The custom launch command of this component (optional). When specified at the component level, it overrides the value specified at the global level (if any).
+ resource:
+ description: Resource of this component (optional). If not specified, the application level global resource takes effect.
+ $ref: '#/definitions/Resource'
+ number_of_containers:
+ type: integer
+ format: int64
+ description: Number of containers for this app-component (optional). If not specified, the application level global number_of_containers takes effect.
+ unique_component_support:
+ type: boolean
+ description: Certain applications need to define multiple components using the same artifact and resource profile, differing only in configurations. In such cases, this field helps app owners to avoid creating multiple component definitions with repeated information. The number_of_containers field dictates the initial number of components created. Component names typically differ with a trailing id, but assumptions should not be made on that, as the algorithm can change at any time. Configurations section will be able to use placeholders like ${USER}, ${CLUSTER_NAME} and ${COMPONENT_NAME} to be replaced at runtime with user the app is submitted as, application name and application component name respectively. Launch command can use placeholders like ${APP_COMPONENT_NAME} and ${APP_NAME} to get its component name and app name respectively at runtime. The best part of this feature is that when the component is flexed up, entirely new components (with new trailing ids) are crea
ted.
+ run_privileged_container:
+ type: boolean
+ description: Run all containers of this component in privileged mode (YARN-4262).
+ placement_policy:
+ description: Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the app level placement_policy takes effect. Refer to the description at the global level for more details.
+ $ref: '#/definitions/PlacementPolicy'
+ configuration:
+ description: Config properties for this app-component.
+ $ref: '#/definitions/Configuration'
+ quicklinks:
+ type: array
+ items:
+ type: string
+ description: A list of quicklink keys defined at the application level, and to be resolved by this component.
+ ReadinessCheck:
+ description: A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every application is different. Hence the need for a simple interface, with scope to support advanced usecases.
+ required:
+ - uri
+ properties:
+ type:
+ type: string
+ description: E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content).
+ enum:
+ - http
+ uri:
+ type: string
+ description: Fully qualified REST uri endpoint.
+ artifact:
+ description: Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits application owners who can run applications without any packaging modifications. Note, artifacts of type docker only is supported for now.
+ $ref: '#/definitions/Artifact'
+ Configuration:
+ description: Set of configuration properties that can be injected into the application components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.
+ properties:
+ properties:
+ type: object
+ description: A blob of key-value pairs of common application properties.
+ additionalProperties:
+ type: string
+ env:
+ type: object
+ description: A blob of key-value pairs which will be appended to the default system properties and handed off to the application at start time. All placeholder references to properties will be substituted before injection.
+ additionalProperties:
+ type: string
+ files:
+ description: Array of list of files that needs to be created and made available as volumes in the application component containers.
+ type: array
+ items:
+ $ref: '#/definitions/ConfigFile'
+ ConfigFile:
+ description: A config file that needs to be created and made available as a volume in an application component container.
+ properties:
+ type:
+ type: string
+ description: Config file in the standard format like xml, properties, json, yaml, template.
+ enum:
+ - xml
+ - properties
+ - json
+ - yaml
+ - template
+ - env
+ - hadoop_xml
+ dest_file:
+ type: string
+ description: The absolute path that this configuration file should be mounted as, in the application container.
+ src_file:
+ type: string
+ description: Required for type template. This provides the source location of the template which needs to be mounted as dest_file post property substitutions. Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, etc.
+ props:
+ type: object
+ description: A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If the type is template then the attribute src_file is mandatory and the src_file content is dumped to dest_file post property substitutions.
+ Container:
+ description: An instance of a running application container.
+ properties:
+ id:
+ type: string
+ description: Unique container id of a running application, e.g. container_e3751_1458061340047_0008_01_000002.
+ launch_time:
+ type: string
+ format: date
+ description: The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.
+ ip:
+ type: string
+ description: IP address of a running container, e.g. 172.31.42.141. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.
+ hostname:
+ type: string
+ description: Fully qualified hostname of a running container, e.g. ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.
+ bare_host:
+ type: string
+ description: The bare node or host in which the container is running, e.g. cn008.example.com.
+ state:
+ description: State of the container of an application.
+ $ref: '#/definitions/ContainerState'
+ component_name:
+ type: string
+ description: Name of the component that this container instance belongs to.
+ resource:
+ description: Resource used for this container.
+ $ref: '#/definitions/Resource'
+ ApplicationState:
+ description: The current state of an application.
+ properties:
+ state:
+ type: string
+ description: enum of the state of the application
+ enum:
+ - accepted
+ - started
+ - ready
+ - stopped
+ - failed
+ ContainerState:
+ description: The current state of the container of an application.
+ properties:
+ state:
+ type: string
+ description: enum of the state of the container
+ enum:
+ - init
+ - ready
+ ApplicationStatus:
+ description: The current status of a submitted application, returned as a response to the GET API.
+ properties:
+ diagnostics:
+ type: string
+ description: Diagnostic information (if any) for the reason of the current state of the application. It typically has a non-null value, if the application is in a non-running state.
+ state:
+ description: Application state.
+ $ref: '#/definitions/ApplicationState'
+ code:
+ type: integer
+ format: int32
+ description: An error code specific to a scenario which app owners should be able to use to understand the failure in addition to the diagnostic information.
+
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/51] [abbrv] hadoop git commit: YARN-5909. Remove agent related
code in slider AM. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
deleted file mode 100644
index 0150079..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.apache.slider.providers.agent.State;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.Map;
-
-/** Data model for agent to send heartbeat to ambari and/or app master. */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class Register {
- private int responseId = -1;
- private long timestamp;
- private String label;
- private int currentPingPort;
- private HostInfo hardwareProfile;
- private String publicHostname;
- private String tags;
- private AgentEnv agentEnv;
- private String agentVersion;
- private State actualState;
- private State expectedState;
- private Map<String, String> allocatedPorts;
- private Map<String, String> logFolders;
- private String pkg;
- private String appVersion;
-
- @JsonProperty("responseId")
- public int getResponseId() {
- return responseId;
- }
-
- @JsonProperty("responseId")
- public void setResponseId(int responseId) {
- this.responseId = responseId;
- }
-
- public long getTimestamp() {
- return timestamp;
- }
-
- public void setTimestamp(long timestamp) {
- this.timestamp = timestamp;
- }
-
- public String getLabel() {
- return label;
- }
-
- public void setLabel(String label) {
- this.label = label;
- }
-
- public String getTags() {
- return tags;
- }
-
- public void setTags(String tags) {
- this.tags = tags;
- }
-
- public HostInfo getHardwareProfile() {
- return hardwareProfile;
- }
-
- public void setHardwareProfile(HostInfo hardwareProfile) {
- this.hardwareProfile = hardwareProfile;
- }
-
- public String getPublicHostname() {
- return publicHostname;
- }
-
- public void setPublicHostname(String name) {
- this.publicHostname = name;
- }
-
- public AgentEnv getAgentEnv() {
- return agentEnv;
- }
-
- public void setAgentEnv(AgentEnv env) {
- this.agentEnv = env;
- }
-
- public String getAgentVersion() {
- return agentVersion;
- }
-
- public void setAgentVersion(String agentVersion) {
- this.agentVersion = agentVersion;
- }
-
- public int getCurrentPingPort() {
- return currentPingPort;
- }
-
- public void setCurrentPingPort(int currentPingPort) {
- this.currentPingPort = currentPingPort;
- }
-
- public State getActualState() {
- return actualState;
- }
-
- public void setActualState(State actualState) {
- this.actualState = actualState;
- }
-
- public State getExpectedState() {
- return expectedState;
- }
-
- public void setExpectedState(State expectedState) {
- this.expectedState = expectedState;
- }
-
- /** @return the allocated ports, or <code>null</code> if none are present */
- @JsonProperty("allocatedPorts")
- public Map<String, String> getAllocatedPorts() {
- return allocatedPorts;
- }
-
- /** @param ports allocated ports */
- @JsonProperty("allocatedPorts")
- public void setAllocatedPorts(Map<String, String> ports) {
- this.allocatedPorts = ports;
- }
-
- /** @return the log folders, or <code>null</code> if none are present */
- @JsonProperty("logFolders")
- public Map<String, String> getLogFolders() {
- return logFolders;
- }
-
- /** @param logFolders assigned log folders */
- @JsonProperty("logFolders")
- public void setLogFolders(Map<String, String> logFolders) {
- this.logFolders = logFolders;
- }
-
- public String getPkg() {
- return pkg;
- }
-
- public void setPkg(String pkg) {
- this.pkg = pkg;
- }
-
- @JsonProperty("appVersion")
- public String getAppVersion() {
- return appVersion;
- }
-
- @JsonProperty("appVersion")
- public void setAppVersion(String appVersion) {
- this.appVersion = appVersion;
- }
-
- @Override
- public String toString() {
- String ret = "responseId=" + responseId + "\n" +
- "timestamp=" + timestamp + "\n" +
- "label=" + label + "\n" +
- "hostname=" + publicHostname + "\n" +
- "expectedState=" + expectedState + "\n" +
- "actualState=" + actualState + "\n" +
- "appVersion=" + appVersion + "\n";
-
- if (hardwareProfile != null) {
- ret = ret + "hardwareprofile=" + this.hardwareProfile.toString();
- }
- return ret;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationCommand.java
deleted file mode 100644
index 4b87dd2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationCommand.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-/**
- *
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class RegistrationCommand {
-
- private String command;
-
- public String getCommand() {
- return command;
- }
-
- public void setCommand(String command) {
- this.command = command;
- }
-
- public RegistrationCommand(String command) {
-
- this.command = command;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java
deleted file mode 100644
index 80b7a5e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.List;
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class RegistrationResponse {
-
- @JsonProperty("response")
- private RegistrationStatus response;
-
- /**
- * exitstatus is a code of error which was rised on server side. exitstatus
- * = 0 (OK - Default) exitstatus = 1 (Registration failed because different
- * version of agent and server)
- */
- @JsonProperty("exitstatus")
- private int exitstatus;
-
- /** log - message, which will be printed to agents log */
- @JsonProperty("log")
- private String log;
-
- /** tags - tags associated with the container */
- @JsonProperty("tags")
- private String tags;
-
- @JsonProperty("package")
- private String pkg;
-
- //Response id to start with, usually zero.
- @JsonProperty("responseId")
- private long responseId;
-
- @JsonProperty("statusCommands")
- private List<StatusCommand> statusCommands = null;
-
- public RegistrationResponse() {
- }
-
- public RegistrationStatus getResponse() {
- return response;
- }
-
- public void setResponse(RegistrationStatus response) {
- this.response = response;
- }
-
- public int getExitstatus() {
- return exitstatus;
- }
-
- public void setExitstatus(int exitstatus) {
- this.exitstatus = exitstatus;
- }
-
- public RegistrationStatus getResponseStatus() {
- return response;
- }
-
- public void setResponseStatus(RegistrationStatus response) {
- this.response = response;
- }
-
- public List<StatusCommand> getStatusCommands() {
- return statusCommands;
- }
-
- public void setStatusCommands(List<StatusCommand> statusCommands) {
- this.statusCommands = statusCommands;
- }
-
- public long getResponseId() {
- return responseId;
- }
-
- public void setResponseId(long responseId) {
- this.responseId = responseId;
- }
-
- public String getTags() {
- return tags;
- }
-
- public void setTags(String tags) {
- this.tags = tags;
- }
-
- public String getLog() {
- return log;
- }
-
- public void setLog(String log) {
- this.log = log;
- }
-
- public String getPkg() {
- return pkg;
- }
-
- public void setPkg(String pkg) {
- this.pkg = pkg;
- }
-
- @Override
- public String toString() {
- return "RegistrationResponse{" +
- "response=" + response +
- ", responseId=" + responseId +
- ", statusCommands=" + statusCommands +
- '}';
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationStatus.java
deleted file mode 100644
index 8374710..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationStatus.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-public enum RegistrationStatus {
- OK,
- FAILED
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java
deleted file mode 100644
index 5b205b5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Command to report the status of a list of services in roles.
- */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class StatusCommand {
- public static String STATUS_COMMAND = "STATUS";
- public static String GET_CONFIG_COMMAND = "GET_CONFIG";
-
- AgentCommandType agentCommandType;
-
- private String clusterName;
- private String serviceName;
- private String componentName;
- private Map<String, Map<String, String>> configurations;
- private Map<String, String> commandParams = new HashMap<String, String>();
- private Map<String, String> hostLevelParams = new HashMap<String, String>();
- private String roleCommand;
- private boolean yarnDockerMode;
-
- public StatusCommand() {
- this.agentCommandType = AgentCommandType.STATUS_COMMAND;
- }
-
- @JsonProperty("clusterName")
- public String getClusterName() {
- return clusterName;
- }
-
- @JsonProperty("clusterName")
- public void setClusterName(String clusterName) {
- this.clusterName = clusterName;
- }
-
- @JsonProperty("serviceName")
- public String getServiceName() {
- return serviceName;
- }
-
- @JsonProperty("serviceName")
- public void setServiceName(String serviceName) {
- this.serviceName = serviceName;
- }
-
- @JsonProperty("componentName")
- public String getComponentName() {
- return componentName;
- }
-
- @JsonProperty("componentName")
- public void setComponentName(String componentName) {
- this.componentName = componentName;
- }
-
- @JsonProperty("configurations")
- public Map<String, Map<String, String>> getConfigurations() {
- return configurations;
- }
-
- @JsonProperty("configurations")
- public void setConfigurations(Map<String, Map<String, String>> configurations) {
- this.configurations = configurations;
- }
-
- @JsonProperty("hostLevelParams")
- public Map<String, String> getHostLevelParams() {
- return hostLevelParams;
- }
-
- @JsonProperty("hostLevelParams")
- public void setHostLevelParams(Map<String, String> params) {
- this.hostLevelParams = params;
- }
-
- @JsonProperty("commandParams")
- public Map<String, String> getCommandParams() {
- return commandParams;
- }
-
- @JsonProperty("commandParams")
- public void setCommandParams(Map<String, String> commandParams) {
- this.commandParams = commandParams;
- }
-
- @JsonProperty("commandType")
- public AgentCommandType getCommandType() {
- return agentCommandType;
- }
-
- @JsonProperty("commandType")
- public void setCommandType(AgentCommandType commandType) {
- this.agentCommandType = commandType;
- }
-
- @JsonProperty("roleCommand")
- public String getRoleCommand() {
- return roleCommand;
- }
-
- @JsonProperty("roleCommand")
- public void setRoleCommand(String roleCommand) {
- this.roleCommand = roleCommand;
- }
-
- @JsonProperty("yarnDockerMode")
- public boolean isYarnDockerMode() {
- return yarnDockerMode;
- }
-
- @JsonProperty("yarnDockerMode")
- public void setYarnDockerMode(boolean yarnDockerMode) {
- this.yarnDockerMode = yarnDockerMode;
- }
-
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder();
- builder.append("StatusCommand [agentCommandType=").append(agentCommandType)
- .append(", clusterName=").append(clusterName).append(", serviceName=")
- .append(serviceName).append(", componentName=").append(componentName)
- .append(", configurations=").append(configurations)
- .append(", commandParams=").append(commandParams)
- .append(", hostLevelParams=").append(hostLevelParams)
- .append(", roleCommand=").append(roleCommand).append("]");
- return builder.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
deleted file mode 100644
index 0bea8fa..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.tools.TestUtility;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Set;
-
-/**
- *
- */
-public class TestAgentClientProvider {
- protected static final Logger log =
- LoggerFactory.getLogger(TestAgentClientProvider.class);
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
- @Test
- public void testGetApplicationTags() throws Exception {
- Configuration configuration = new Configuration();
- FileSystem fs = FileSystem.getLocal(configuration);
- SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration);
-
- AgentClientProvider provider = new AgentClientProvider(null);
- String zipFileName = TestUtility.createAppPackage(
- folder,
- "testpkg",
- "test.zip",
- "target/test-classes/org/apache/slider/common/tools/test");
- Set<String> tags = provider.getApplicationTags(sliderFileSystem, zipFileName);
- assert tags != null;
- assert !tags.isEmpty();
- assert tags.contains("Name: STORM");
- assert tags.contains("Description: Apache Hadoop Stream processing framework");
- assert tags.contains("Version: 0.9.1.2.1");
-
- }
-
- @Test
- public void testValidateInstanceDefinition() throws Exception {
- AgentClientProvider provider = new AgentClientProvider(null);
- AggregateConf instanceDefinition = new AggregateConf();
-
- try {
- provider.validateInstanceDefinition(instanceDefinition, null);
- Assert.assertFalse("Should fail with BadConfigException", true);
- } catch (BadConfigException e) {
- log.info(e.toString());
- Assert.assertTrue(e.getMessage().contains("Application definition must be provided"));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentLaunchParameter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentLaunchParameter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentLaunchParameter.java
deleted file mode 100644
index ec62b54..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentLaunchParameter.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- *
- */
-public class TestAgentLaunchParameter {
- protected static final Logger log =
- LoggerFactory.getLogger(TestAgentLaunchParameter.class);
-
- @Test
- public void testTestAgentLaunchParameter() throws Exception {
- AgentLaunchParameter alp = new AgentLaunchParameter("");
- Assert.assertEquals("", alp.getNextLaunchParameter("abc"));
- Assert.assertEquals("", alp.getNextLaunchParameter("HBASE_MASTER"));
-
- alp = new AgentLaunchParameter("a:1:2:3|b:5:6:NONE");
- Assert.assertEquals("1", alp.getNextLaunchParameter("a"));
- Assert.assertEquals("2", alp.getNextLaunchParameter("a"));
- Assert.assertEquals("3", alp.getNextLaunchParameter("a"));
- Assert.assertEquals("3", alp.getNextLaunchParameter("a"));
-
- Assert.assertEquals("5", alp.getNextLaunchParameter("b"));
- Assert.assertEquals("6", alp.getNextLaunchParameter("b"));
- Assert.assertEquals("", alp.getNextLaunchParameter("b"));
- Assert.assertEquals("", alp.getNextLaunchParameter("b"));
- Assert.assertEquals("", alp.getNextLaunchParameter("c"));
-
- alp = new AgentLaunchParameter("|a:1:3|b::5:NONE:");
- Assert.assertEquals("1", alp.getNextLaunchParameter("a"));
- Assert.assertEquals("3", alp.getNextLaunchParameter("a"));
- Assert.assertEquals("3", alp.getNextLaunchParameter("a"));
-
- Assert.assertEquals("", alp.getNextLaunchParameter("b"));
- Assert.assertEquals("5", alp.getNextLaunchParameter("b"));
- Assert.assertEquals("", alp.getNextLaunchParameter("b"));
- Assert.assertEquals("", alp.getNextLaunchParameter("b"));
-
- alp = new AgentLaunchParameter("|:");
- Assert.assertEquals("", alp.getNextLaunchParameter("b"));
- Assert.assertEquals("", alp.getNextLaunchParameter("a"));
-
- alp = new AgentLaunchParameter("HBASE_MASTER:a,b:DO_NOT_REGISTER:");
- Assert.assertEquals("a,b", alp.getNextLaunchParameter("HBASE_MASTER"));
- Assert.assertEquals("DO_NOT_REGISTER", alp.getNextLaunchParameter("HBASE_MASTER"));
- Assert.assertEquals("DO_NOT_REGISTER", alp.getNextLaunchParameter("HBASE_MASTER"));
-
- alp = new AgentLaunchParameter("HBASE_MASTER:a,b:DO_NOT_REGISTER::c:::");
- Assert.assertEquals("a,b", alp.getNextLaunchParameter("HBASE_MASTER"));
- Assert.assertEquals("DO_NOT_REGISTER", alp.getNextLaunchParameter("HBASE_MASTER"));
- Assert.assertEquals("", alp.getNextLaunchParameter("HBASE_MASTER"));
- Assert.assertEquals("c", alp.getNextLaunchParameter("HBASE_MASTER"));
- Assert.assertEquals("c", alp.getNextLaunchParameter("HBASE_MASTER"));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java
deleted file mode 100644
index 5e1dc7f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.providers.agent.application.metadata.Metainfo;
-import org.apache.slider.tools.TestUtility;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestAgentUtils {
- protected static final Logger log =
- LoggerFactory.getLogger(TestAgentUtils.class);
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
- private static final String metainfo_str = "<metainfo>\n"
- + " <schemaVersion>2.0</schemaVersion>\n"
- + " <application>\n"
- + " <name>MYTESTAPPLICATION</name>\n"
- + " <comment>\n"
- + " My Test Application\n"
- + " </comment>\n"
- + " <version>1.0</version>\n"
- + " <type>YARN-APP</type>\n"
- + " <components>\n"
- + " <component>\n"
- + " <name>REST</name>\n"
- + " <category>MASTER</category>\n"
- + " <commandScript>\n"
- + " <script>scripts/rest.py</script>\n"
- + " <scriptType>PYTHON</scriptType>\n"
- + " <timeout>600</timeout>\n"
- + " </commandScript>\n"
- + " </component>\n"
- + " </components>\n"
- + " </application>\n"
- + "</metainfo>";
-
- @Test
- public void testGetApplicationMetainfo() throws Exception {
- String zipFileName = TestUtility.createAppPackage(
- folder,
- "testpkg",
- "test.zip",
- "target/test-classes/org/apache/slider/common/tools/test");
- Configuration configuration = new Configuration();
- FileSystem fs = FileSystem.getLocal(configuration);
- log.info("fs working dir is {}", fs.getWorkingDirectory().toString());
- SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration);
-
- // Without accompany metainfo file, read metainfo from the zip file
- Metainfo metainfo = AgentUtils.getApplicationMetainfo(
- sliderFileSystem, zipFileName, false);
- Assert.assertNotNull(metainfo.getApplication());
- Assert.assertEquals("STORM", metainfo.getApplication().getName());
-
- // With accompany metainfo file, read metainfo from the accompany file
- String acompanyFileName = zipFileName + ".metainfo.xml";
- File f = new File(acompanyFileName);
- try (BufferedWriter writer = new BufferedWriter(new FileWriter(f))) {
- writer.write(metainfo_str);
- }
- metainfo = AgentUtils.getApplicationMetainfo(
- sliderFileSystem, zipFileName, false);
- Assert.assertNotNull(metainfo.getApplication());
- Assert.assertEquals("MYTESTAPPLICATION", metainfo.getApplication().getName());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java
deleted file mode 100644
index dedf4f6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java
+++ /dev/null
@@ -1,264 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent;
-
-import com.google.common.io.Files;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.slider.common.params.ActionCreateArgs;
-import org.apache.slider.common.params.AddonArgsDelegate;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.conf.ConfTree;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.persist.AppDefinitionPersister;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.PrintWriter;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- *
- */
-public class TestAppDefinitionPersister {
- protected static final Logger log =
- LoggerFactory.getLogger(TestAppDefinitionPersister.class);
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
- /**
- * @BeforeClass public static void initialize() { BasicConfigurator.resetConfiguration();
- * BasicConfigurator.configure(); }*
- */
-
-
- @Test
- public void testAppDefinitionPersister() throws Exception {
- Configuration configuration = new Configuration();
- FileSystem fs = FileSystem.getLocal(configuration);
- log.info("fs working dir is {}", fs.getWorkingDirectory().toString());
- SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration);
-
- AppDefinitionPersister adp = new AppDefinitionPersister(sliderFileSystem);
- String clustername = "c1";
- ActionCreateArgs buildInfo = new ActionCreateArgs();
- buildInfo.appMetaInfo = null;
- buildInfo.appDef = null;
- buildInfo.addonDelegate = new AddonArgsDelegate();
-
- // nothing to do
- adp.processSuppliedDefinitions(clustername, buildInfo, null);
- adp.persistPackages();
- List<AppDefinitionPersister.AppDefinition> appDefinitions = adp.getAppDefinitions();
- Assert.assertTrue(appDefinitions.size() == 0);
-
- ConfTree ct = new ConfTree();
- ConfTreeOperations appConf = new ConfTreeOperations(ct);
- final File tempDir = Files.createTempDir();
- final File metainfo = new File(tempDir, "metainfo.json");
-
- // unreadable metainfo
- buildInfo.appMetaInfo = metainfo;
-
- try {
- adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
- } catch (BadConfigException bce) {
- log.info(bce.getMessage());
- Assert.assertTrue(bce.getMessage().contains(
- "Path specified with "
- + "--metainfo either cannot be read or is not a file"));
- }
-
- try (PrintWriter writer = new PrintWriter(metainfo.getAbsolutePath(), "UTF-8")) {
- writer.println("{");
- writer.println("}");
- }
- buildInfo.appDef = metainfo;
-
- try {
- adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
- } catch (BadConfigException bce) {
- log.info(bce.getMessage());
- Assert.assertTrue(bce.getMessage().contains(
- "Both --metainfo and --appdef cannot be specified"));
- }
-
- // both --metainfojson and --appdef cannot be specified
- buildInfo.appMetaInfo = null;
- buildInfo.appMetaInfoJson = "{}";
- try {
- adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
- } catch (BadConfigException bce) {
- log.info(bce.getMessage());
- Assert.assertTrue(bce.getMessage().contains(
- "Both --metainfojson and --appdef cannot be specified"));
- }
-
- buildInfo.appDef = null;
-
- buildInfo.appMetaInfoJson = "";
- try {
- adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
- } catch (BadConfigException bce) {
- log.info(bce.getMessage());
- Assert.assertTrue(bce.getMessage().contains(
- "Empty string specified with --metainfojson"));
- }
- buildInfo.appMetaInfo = metainfo;
-
- // both --metainfo and --metainfojson cannot be specified
- buildInfo.appMetaInfoJson = "{}";
- try {
- adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
- } catch (BadConfigException bce) {
- log.info(bce.getMessage());
- Assert.assertTrue(bce.getMessage().contains(
- "Both --metainfo and --metainfojson cannot be specified"));
- }
- buildInfo.appMetaInfoJson = null;
-
- appConf.getGlobalOptions().set(AgentKeys.APP_DEF, metainfo.getAbsolutePath());
-
- try {
- adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
- } catch (BadConfigException bce) {
- log.info(bce.getMessage());
- Assert.assertTrue(bce.getMessage().contains(
- "application.def cannot "
- + "not be set if --metainfo is specified in the cmd line"));
- }
-
- appConf.getGlobalOptions().remove(AgentKeys.APP_DEF);
-
- adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
- appDefinitions = adp.getAppDefinitions();
- Assert.assertTrue(appDefinitions.size() == 1);
- Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip"));
- log.info(appDefinitions.get(0).toString());
- Assert.assertTrue(appDefinitions.get(0).appDefPkgOrFolder.toString().endsWith("default"));
- Assert.assertTrue(appDefinitions.get(0).targetFolderInFs.toString().contains("cluster/c1/appdef"));
- Assert.assertEquals("appPkg.zip", appDefinitions.get(0).pkgName);
-
- buildInfo.appDef = tempDir;
- buildInfo.appMetaInfo = null;
-
- appConf.getGlobalOptions().set(AgentKeys.APP_DEF, metainfo.getAbsolutePath());
-
- try {
- adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
- } catch (BadConfigException bce) {
- log.info(bce.getMessage());
- Assert.assertTrue(bce.getMessage().contains("application.def must not be set if --appdef is provided"));
- }
-
- adp.getAppDefinitions().clear();
- appConf.getGlobalOptions().remove(AgentKeys.APP_DEF);
- adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
- appDefinitions = adp.getAppDefinitions();
- Assert.assertTrue(appDefinitions.size() == 1);
- Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip"));
- log.info(appDefinitions.get(0).toString());
- Assert.assertTrue(appDefinitions.get(0).appDefPkgOrFolder.toString().endsWith(tempDir.toString()));
- Assert.assertTrue(appDefinitions.get(0).targetFolderInFs.toString().contains("cluster/c1/appdef"));
- Assert.assertEquals("appPkg.zip", appDefinitions.get(0).pkgName);
-
- adp.getAppDefinitions().clear();
- buildInfo.appDef = null;
- buildInfo.appMetaInfo = null;
- appConf.getGlobalOptions().remove(AgentKeys.APP_DEF);
-
- ArrayList<String> list = new ArrayList<String>() {{
- add("addon1");
- add("");
- add("addon2");
- add(metainfo.getAbsolutePath());
- }};
-
- buildInfo.addonDelegate.addonTuples = list;
- try {
- adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
- } catch (BadConfigException bce) {
- log.info(bce.getMessage());
- Assert.assertTrue(bce.getMessage().contains("addon package can only be specified if main app package is specified"));
- }
-
- buildInfo.appMetaInfo = metainfo;
-
- try {
- adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
- } catch (BadConfigException bce) {
- log.info(bce.getMessage());
- Assert.assertTrue(bce.getMessage().contains("Invalid path for addon package addon1"));
- }
-
- appConf.getGlobalOptions().remove(AgentKeys.APP_DEF);
-
- list = new ArrayList<String>() {{
- add("addon1");
- add(tempDir.getAbsolutePath());
- add("addon2");
- add(metainfo.getAbsolutePath());
- }};
-
- buildInfo.addonDelegate.addonTuples = list;
- adp.getAppDefinitions().clear();
-
- adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
- appDefinitions = adp.getAppDefinitions();
-
- Assert.assertTrue(appDefinitions.size() == 3);
- Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip"));
- Assert.assertTrue(appConf.getGlobalOptions().get("application.addon.addon1").contains(
- "addons/addon1/addon_addon1.zip"));
- Assert.assertTrue(appConf.getGlobalOptions().get("application.addon.addon2").contains(
- "addons/addon2/addon_addon2.zip"));
- log.info(appConf.getGlobalOptions().get("application.addons"));
- Assert.assertTrue(appConf.getGlobalOptions().get("application.addons").contains(
- "application.addon.addon2,application.addon.addon1")
- || appConf.getGlobalOptions().get("application.addons").contains(
- "application.addon.addon1,application.addon.addon2"));
- int seen = 0;
- for (AppDefinitionPersister.AppDefinition adp_ad : appDefinitions) {
- if (adp_ad.pkgName.equals("appPkg.zip")) {
- log.info(adp_ad.toString());
- Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith("default"));
- Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("cluster/c1/appdef"));
- seen++;
- }
- if (adp_ad.pkgName.equals("addon_addon1.zip")) {
- log.info(adp_ad.toString());
- Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith(tempDir.toString()));
- Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("addons/addon1"));
- seen++;
- }
- if (adp_ad.pkgName.equals("addon_addon2.zip")) {
- log.info(adp_ad.toString());
- Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith("metainfo.json"));
- Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("addons/addon2"));
- seen++;
- }
- }
- Assert.assertEquals(3, seen);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentTagProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentTagProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentTagProvider.java
deleted file mode 100644
index 7b38ee3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentTagProvider.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-public class TestComponentTagProvider {
- protected static final Logger log =
- LoggerFactory.getLogger(TestComponentTagProvider.class);
-
- @Test
- public void testTagProvider() throws Exception {
- ComponentTagProvider ctp = new ComponentTagProvider();
- Assert.assertEquals("", ctp.getTag(null, null));
- Assert.assertEquals("", ctp.getTag(null, "cid"));
- Assert.assertEquals("", ctp.getTag("comp1", null));
-
- Assert.assertEquals("1", ctp.getTag("comp1", "cid1"));
- Assert.assertEquals("2", ctp.getTag("comp1", "cid2"));
- Assert.assertEquals("3", ctp.getTag("comp1", "cid3"));
- ctp.releaseTag("comp1", "cid2");
- Assert.assertEquals("2", ctp.getTag("comp1", "cid22"));
-
- ctp.releaseTag("comp1", "cid4");
- ctp.recordAssignedTag("comp1", "cid5", "5");
- Assert.assertEquals("4", ctp.getTag("comp1", "cid4"));
- Assert.assertEquals("4", ctp.getTag("comp1", "cid4"));
- Assert.assertEquals("6", ctp.getTag("comp1", "cid6"));
-
- ctp.recordAssignedTag("comp1", "cid55", "5");
- Assert.assertEquals("5", ctp.getTag("comp1", "cid55"));
-
- ctp.recordAssignedTag("comp2", "cidb3", "3");
- Assert.assertEquals("1", ctp.getTag("comp2", "cidb1"));
- Assert.assertEquals("2", ctp.getTag("comp2", "cidb2"));
- Assert.assertEquals("4", ctp.getTag("comp2", "cidb4"));
-
- ctp.recordAssignedTag("comp2", "cidb5", "six");
- ctp.recordAssignedTag("comp2", "cidb5", "-55");
- ctp.recordAssignedTag("comp2", "cidb5", "tags");
- ctp.recordAssignedTag("comp2", "cidb5", null);
- ctp.recordAssignedTag("comp2", "cidb5", "");
- ctp.recordAssignedTag("comp2", "cidb5", "5");
- Assert.assertEquals("6", ctp.getTag("comp2", "cidb6"));
-
- ctp.recordAssignedTag("comp2", null, "5");
- ctp.recordAssignedTag(null, null, "5");
- ctp.releaseTag("comp1", null);
- ctp.releaseTag(null, "cid4");
- ctp.releaseTag(null, null);
- }
-
- @Test
- public void testTagProviderWithThread() throws Exception {
- ComponentTagProvider ctp = new ComponentTagProvider();
- Thread thread = new Thread(new Taggged(ctp));
- Thread thread2 = new Thread(new Taggged(ctp));
- Thread thread3 = new Thread(new Taggged(ctp));
- thread.start();
- thread2.start();
- thread3.start();
- ctp.getTag("comp1", "cid50");
- thread.join();
- thread2.join();
- thread3.join();
- Assert.assertEquals("101", ctp.getTag("comp1", "cid101"));
- }
-
- public class Taggged implements Runnable {
- private final ComponentTagProvider ctp;
-
- public Taggged(ComponentTagProvider ctp) {
- this.ctp = ctp;
- }
-
- public void run() {
- for (int i = 0; i < 100; i++) {
- String containerId = "cid" + (i + 1);
- this.ctp.getTag("comp1", containerId);
- }
- for (int i = 0; i < 100; i++) {
- String containerId = "cid" + (i + 1);
- this.ctp.getTag("comp1", containerId);
- }
- for (int i = 0; i < 100; i += 2) {
- String containerId = "cid" + (i + 1);
- this.ctp.releaseTag("comp1", containerId);
- }
- for (int i = 0; i < 100; i += 2) {
- String containerId = "cid" + (i + 1);
- this.ctp.getTag("comp1", containerId);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestState.java
deleted file mode 100644
index 6a2e5ab5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestState.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class TestState {
- protected static final Logger log = LoggerFactory.getLogger(TestState.class);
-
- @Test
- public void testState() throws Exception {
- State state = State.STARTED;
- Assert.assertEquals(Command.STOP, state.getSupportedCommand(false, true));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestConfigParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestConfigParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestConfigParser.java
deleted file mode 100644
index 3aa44a1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestConfigParser.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- *
- */
-public class TestConfigParser {
- protected static final Logger log =
- LoggerFactory.getLogger(TestConfigParser.class);
- private static final String config_1_str = "<configuration>\n"
- + " <property>\n"
- + " <name>security.client.protocol.acl</name>\n"
- + " <value>*</value>\n"
- + " <description>ACL for HRegionInterface protocol implementations (ie. \n"
- + " clients talking to HRegionServers)\n"
- + " The ACL is a comma-separated list of user and group names. The user and \n"
- + " group list is separated by a blank. For e.g. \"alice,bob users,wheel\". \n"
- + " A special value of \"*\" means all users are allowed.</description>\n"
- + " </property>\n"
- + "\n"
- + " <property>\n"
- + " <name>security.admin.protocol.acl</name>\n"
- + " <value>*</value>\n"
- + " <description>ACL for HMasterInterface protocol implementation (ie. \n"
- + " clients talking to HMaster for admin operations).\n"
- + " The ACL is a comma-separated list of user and group names. The user and \n"
- + " group list is separated by a blank. For e.g. \"alice,bob users,wheel\". \n"
- + " A special value of \"*\" means all users are allowed.</description>\n"
- + " </property>\n"
- + "\n"
- + " <property>\n"
- + " <name>security.masterregion.protocol.acl</name>\n"
- + " <value>*</value>\n"
- + " <description>ACL for HMasterRegionInterface protocol implementations\n"
- + " (for HRegionServers communicating with HMaster)\n"
- + " The ACL is a comma-separated list of user and group names. The user and \n"
- + " group list is separated by a blank. For e.g. \"alice,bob users,wheel\". \n"
- + " A special value of \"*\" means all users are allowed.</description>\n"
- + " </property>\n"
- + " <property>\n"
- + " <name>emptyVal</name>\n"
- + " <value></value>\n"
- + " <description>non-empty-desc</description>\n"
- + " </property>\n"
- + " <property>\n"
- + " <name>emptyDesc</name>\n"
- + " <value></value>\n"
- + " <description></description>\n"
- + " </property>\n"
- + " <property>\n"
- + " <name>noDesc</name>\n"
- + " <value></value>\n"
- + " </property>\n"
- + "</configuration>";
-
- @Test
- public void testParse() throws IOException {
-
- InputStream config_1 = new ByteArrayInputStream(config_1_str.getBytes());
- DefaultConfig config = new DefaultConfigParser().parse(config_1);
- Assert.assertNotNull(config);
- Assert.assertNotNull(config.getPropertyInfos());
- Assert.assertEquals(6, config.getPropertyInfos().size());
- for (PropertyInfo pInfo : config.getPropertyInfos()) {
- if (pInfo.getName().equals("security.client.protocol.acl")) {
- Assert.assertEquals("*", pInfo.getValue());
- Assert.assertTrue(pInfo.getDescription().startsWith("ACL for HRegionInterface "));
- }
- if (pInfo.getName().equals("emptyVal")) {
- Assert.assertEquals("", pInfo.getValue());
- Assert.assertEquals("non-empty-desc", pInfo.getDescription());
- }
- if (pInfo.getName().equals("emptyDesc")) {
- Assert.assertEquals("", pInfo.getValue());
- Assert.assertEquals("", pInfo.getDescription());
- }
- if (pInfo.getName().equals("noDesc")) {
- Assert.assertEquals("", pInfo.getValue());
- Assert.assertNull(pInfo.getDescription());
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java
deleted file mode 100644
index ba1912a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.providers.agent.AgentProviderService;
-import org.junit.Assert;
-import org.junit.Test;
-import org.mockito.Mockito;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.List;
-
-import static org.mockito.Mockito.doReturn;
-
-/**
- *
- */
-public class TestMetainfoParser {
- protected static final Logger log =
- LoggerFactory.getLogger(TestMetainfoParser.class);
- public static final String METAINFO_XML =
- "/org/apache/slider/providers/agent/application/metadata/metainfo.xml";
-
- @Test
- public void testParse() throws IOException {
-
- InputStream resStream = this.getClass().getResourceAsStream(
- METAINFO_XML);
- MetainfoParser parser = new MetainfoParser();
- Metainfo metainfo = parser.fromXmlStream(resStream);
- Assert.assertNotNull(metainfo);
- Assert.assertNotNull(metainfo.getApplication());
- Application application = metainfo.getApplication();
- assert "STORM".equals(application.getName());
- assert 6 == application.getComponents().size();
- OSPackage pkg = application.getOSSpecifics().get(0).getPackages().get(0);
- assert "tarball".equals(pkg.getType());
- assert "files/apache-storm-0.9.1.2.1.1.0-237.tar.gz".equals(pkg.getName());
- boolean found = false;
- for (Component comp : application.getComponents()) {
- if (comp != null && comp.getName().equals("NIMBUS")) {
- found = true;
- Assert.assertEquals(0, comp.getComponentExports().size());
- }
- if (comp != null && comp.getName().equals("SUPERVISOR")) {
- Assert.assertEquals(1, comp.getComponentExports().size());
- }
- if (comp != null && comp.getName().equals("ANOTHER_COMPONENT")) {
- assert 2 == comp.getCommands().size();
- assert "start command".equals(comp.getCommands().get(0).getExec());
- assert "START".equals(comp.getCommands().get(0).getName());
- assert "stop command".equals(comp.getCommands().get(1).getExec());
- assert "STOP".equals(comp.getCommands().get(1).getName());
- }
- }
- assert found;
- Assert.assertEquals(0, application.getConfigFiles().size());
- assert 1 == application.getPackages().size();
- Package p = application.getPackages().get(0);
- assert "tarball".equals(p.getType());
- assert "test-tarball-name.tgz".equals(p.getName());
- }
-
- @Test
- public void testJsonParse() throws IOException {
- String metaInfo1_json = "{\n"
- + "\"schemaVersion\":\"2.2\",\n"
- + "\"application\":{\n"
- + "\"name\": \"MEMCACHED\","
- + "\"exportGroups\": ["
- + "{"
- + "\"name\": \"Servers\","
- + "\"exports\": ["
- + "{"
- + "\"name\": \"host_port\","
- + "\"value\": \"${MEMCACHED_HOST}:${site.global.port}\""
- + "}"
- + "]"
- + "}"
- + "],"
- + "\"components\": ["
- + "{"
- + "\"name\": \"MEMCACHED\","
- + "\"compExports\": \"Servers-host_port\","
- + "\"commands\": ["
- + "{"
- + "\"exec\": \"java -classpath /usr/myapps/memcached/*:/usr/lib/hadoop/lib/* com.thimbleware.jmemcached.Main\""
- + "}"
- + "]"
- + "},"
- + "{"
- + "\"name\": \"MEMCACHED2\","
- + "\"commands\": ["
- + "{"
- + "\"exec\": \"scripts/config.py\","
- + "\"type\": \"PYTHON\","
- + "\"name\": \"CONFIGURE\""
- + "}"
- + "],"
- + "\"dockerContainers\": ["
- + "{"
- + "\"name\": \"redis\","
- + "\"image\": \"dockerhub/redis\","
- + "\"options\": \"-net=bridge\","
- + "\"mounts\": ["
- + "{"
- + "\"containerMount\": \"/tmp/conf\","
- + "\"hostMount\": \"{$conf:@//site/global/app_root}/conf\""
- + "}"
- + "]"
- + "}"
- + "]"
- + "}"
- + "]"
- + "}"
- + "}";
-
- MetainfoParser parser = new MetainfoParser();
- Metainfo mInfo = parser.fromJsonString(metaInfo1_json);
- Assert.assertEquals("2.2", mInfo.getSchemaVersion());
-
- Application app = mInfo.getApplication();
- Assert.assertNotNull(app);
-
- Assert.assertEquals("MEMCACHED", app.getName());
- List<ExportGroup> egs = app.getExportGroups();
- Assert.assertEquals(1, egs.size());
- ExportGroup eg = egs.get(0);
- Assert.assertEquals("Servers", eg.getName());
- List<Export> exports = eg.getExports();
- Assert.assertEquals(1, exports.size());
- Export export = exports.get(0);
- Assert.assertEquals("host_port", export.getName());
- Assert.assertEquals("${MEMCACHED_HOST}:${site.global.port}", export.getValue());
-
- List<Component> components = app.getComponents();
- Assert.assertEquals(2, components.size());
-
- Component c1 = mInfo.getApplicationComponent("MEMCACHED");
- Assert.assertNotNull(c1);
- Assert.assertEquals("MEMCACHED", c1.getName());
- Assert.assertEquals("Servers-host_port", c1.getCompExports());
- Assert.assertEquals(1, c1.getCommands().size());
- ComponentCommand cmd = c1.getCommands().get(0);
- Assert.assertEquals("START", cmd.getName());
- Assert.assertEquals("SHELL", cmd.getType());
- Assert.assertEquals("java -classpath /usr/myapps/memcached/*:/usr/lib/hadoop/lib/* com.thimbleware.jmemcached.Main",
- cmd.getExec());
-
- Component c2 = mInfo.getApplicationComponent("MEMCACHED2");
- Assert.assertNotNull(c2);
- Assert.assertEquals("MEMCACHED2", c2.getName());
- Assert.assertEquals(1, c2.getCommands().size());
- cmd = c2.getCommands().get(0);
- Assert.assertEquals("CONFIGURE", cmd.getName());
- Assert.assertEquals("PYTHON", cmd.getType());
- Assert.assertEquals("scripts/config.py", cmd.getExec());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
deleted file mode 100644
index 7fceac7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.publisher;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.providers.agent.AgentProviderService;
-import org.apache.slider.server.appmaster.actions.QueueAccess;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- *
- */
-public class TestAgentProviderService extends AgentProviderService {
- protected static final Logger log =
- LoggerFactory.getLogger(TestAgentProviderService.class);
-
- public TestAgentProviderService() {
- super();
- log.info("TestAgentProviderService created");
- }
-
- @Override
- public void bind(StateAccessForProviders stateAccessor,
- QueueAccess queueAccess,
- List<Container> liveContainers) {
- super.bind(stateAccessor, queueAccess, liveContainers);
- Map<String,String> dummyProps = new HashMap<String, String>();
- dummyProps.put("prop1", "val1");
- dummyProps.put("prop2", "val2");
- log.info("publishing dummy-site.xml with values {}", dummyProps);
- publishApplicationInstanceData("dummy-site", "dummy configuration",
- dummyProps.entrySet());
- // publishing global config for testing purposes
- publishApplicationInstanceData("global", "global configuration",
- stateAccessor.getAppConfSnapshot()
- .getGlobalOptions().entrySet());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestSliderProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestSliderProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestSliderProviderFactory.java
deleted file mode 100644
index f49e15a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestSliderProviderFactory.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.publisher;
-
-import org.apache.slider.providers.ProviderService;
-import org.apache.slider.providers.agent.AgentProviderFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- */
-public class TestSliderProviderFactory extends AgentProviderFactory{
- protected static final Logger log =
- LoggerFactory.getLogger(TestSliderProviderFactory.class);
-
- public TestSliderProviderFactory() {
- log.info("Created TestSliderProviderFactory");
- }
-
- @Override
- public ProviderService createServerProvider() {
- log.info("Creating TestAgentProviderService");
- return new TestAgentProviderService();
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/51] [abbrv] hadoop git commit: Addendum patch for YARN-5610.
Contributed by Gour Saha
Posted by ji...@apache.org.
Addendum patch for YARN-5610. Contributed by Gour Saha
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dc026492
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dc026492
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dc026492
Branch: refs/heads/yarn-native-services
Commit: dc0264923b8e4dd4044d81beabc0d315c29a824a
Parents: e2cfd76
Author: Jian He <ji...@apache.org>
Authored: Wed Oct 12 13:33:09 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../yarn/services/resource/Application.java | 44 ++++++++++----------
.../services/resource/ApplicationState.java | 5 +++
.../services/resource/ApplicationStatus.java | 8 ++--
.../hadoop/yarn/services/resource/Artifact.java | 4 +-
.../yarn/services/resource/Component.java | 16 +++----
.../yarn/services/resource/Container.java | 15 ++++---
.../yarn/services/resource/ReadinessCheck.java | 6 +--
7 files changed, 54 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc026492/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
index cfcae95..719bf95 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
@@ -48,8 +48,8 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder;
public class Application extends BaseResource {
private static final long serialVersionUID = -4491694636566094885L;
- private String id = null;
private String name = null;
+ private String id = null;
private Artifact artifact = null;
private Resource resource = null;
private String launchCommand = null;
@@ -63,25 +63,7 @@ public class Application extends BaseResource {
private List<Container> containers = new ArrayList<>();
private ApplicationState state = null;
private Map<String, String> quicklinks = null;
- private String queue;
-
- /**
- * A unique application id.
- **/
- public Application id(String id) {
- this.id = id;
- return this;
- }
-
- @ApiModelProperty(example = "null", required = true, value = "A unique application id.")
- @JsonProperty("id")
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
+ private String queue = null;
/**
* A unique application name.
@@ -102,6 +84,24 @@ public class Application extends BaseResource {
}
/**
+ * A unique application id.
+ **/
+ public Application id(String id) {
+ this.id = id;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A unique application id.")
+ @JsonProperty("id")
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ /**
* Artifact of single-component applications. Mandatory if components
* attribute is not specified.
**/
@@ -423,8 +423,8 @@ public class Application extends BaseResource {
sb.append(" numberOfRunningContainers: ")
.append(toIndentedString(numberOfRunningContainers)).append("\n");
sb.append(" lifetime: ").append(toIndentedString(lifetime)).append("\n");
- sb.append(" placementPolicy: ")
- .append(toIndentedString(placementPolicy)).append("\n");
+ sb.append(" placementPolicy: ").append(toIndentedString(placementPolicy))
+ .append("\n");
sb.append(" components: ").append(toIndentedString(components))
.append("\n");
sb.append(" configuration: ").append(toIndentedString(configuration))
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc026492/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ApplicationState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ApplicationState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ApplicationState.java
index ae96e8a..7f90a9e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ApplicationState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ApplicationState.java
@@ -17,9 +17,14 @@
package org.apache.hadoop.yarn.services.resource;
+import io.swagger.annotations.ApiModel;
+
/**
* The current state of an application.
**/
+
+@ApiModel(description = "The current state of an application.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
public enum ApplicationState {
ACCEPTED, STARTED, READY, STOPPED, FAILED;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc026492/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ApplicationStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ApplicationStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ApplicationStatus.java
index 0166b48..ed826d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ApplicationStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ApplicationStatus.java
@@ -22,7 +22,6 @@ import io.swagger.annotations.ApiModelProperty;
import java.util.Objects;
-import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import com.fasterxml.jackson.annotation.JsonInclude;
@@ -33,7 +32,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
* GET API.
**/
-@ApiModel(description = "The current status of a submitted application, returned as a response to the GET api")
+@ApiModel(description = "The current status of a submitted application, returned as a response to the GET API.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@XmlRootElement
@JsonInclude(JsonInclude.Include.NON_NULL)
@@ -60,20 +59,19 @@ public class ApplicationStatus extends BaseResource {
return diagnostics;
}
- @XmlElement(name = "diagnostics")
public void setDiagnostics(String diagnostics) {
this.diagnostics = diagnostics;
}
/**
- * Application state
+ * Application state.
**/
public ApplicationStatus state(ApplicationState state) {
this.state = state;
return this;
}
- @ApiModelProperty(example = "null", value = "Application state")
+ @ApiModelProperty(example = "null", value = "Application state.")
@JsonProperty("state")
public ApplicationState getState() {
return state;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc026492/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
index aee4d11..af0ad12 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
@@ -76,14 +76,14 @@ public class Artifact {
}
/**
- * Artifact type, like docker, tarball, etc. (optional)
+ * Artifact type, like docker, tarball, etc. (optional).
**/
public Artifact type(TypeEnum type) {
this.type = type;
return this;
}
- @ApiModelProperty(example = "null", value = "Artifact type, like docker, tarball, etc. (optional)")
+ @ApiModelProperty(example = "null", value = "Artifact type, like docker, tarball, etc. (optional).")
@JsonProperty("type")
public TypeEnum getType() {
return type;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc026492/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Component.java
index 3ff6945..1246aa8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Component.java
@@ -202,8 +202,11 @@ public class Component {
* dictates the initial number of components created. Component names
* typically differ with a trailing id, but assumptions should not be made on
* that, as the algorithm can change at any time. Configurations section will
- * be able to use placeholders like ${APP_COMPONENT_NAME} to get its component
- * name at runtime, and thereby differing in value at runtime. The best part
+ * be able to use placeholders like ${USER}, ${CLUSTER_NAME} and
+ * ${COMPONENT_NAME} to be replaced at runtime with user the app is submitted
+ * as, application name and application component name respectively. Launch
+ * command can use placeholders like ${APP_COMPONENT_NAME} and ${APP_NAME} to
+ * get its component name and app name respectively at runtime. The best part
* of this feature is that when the component is flexed up, entirely new
* components (with new trailing ids) are created.
**/
@@ -212,7 +215,7 @@ public class Component {
return this;
}
- @ApiModelProperty(example = "null", value = "Certain applications need to define multiple components using the same artifact and resource profile, differing only in configurations. In such cases, this field helps app owners to avoid creating multiple component definitions with repeated information. The number_of_containers field dictates the initial number of components created. Component names typically differ with a trailing id, but assumptions should not be made on that, as the algorithm can change at any time. Configurations section will be able to use placeholders like ${APP_COMPONENT_NAME} to get its component name at runtime, and thereby differing in value at runtime. The best part of this feature is that when the component is flexed up, entirely new components (with new trailing ids) are created.")
+ @ApiModelProperty(example = "null", value = "Certain applications need to define multiple components using the same artifact and resource profile, differing only in configurations. In such cases, this field helps app owners to avoid creating multiple component definitions with repeated information. The number_of_containers field dictates the initial number of components created. Component names typically differ with a trailing id, but assumptions should not be made on that, as the algorithm can change at any time. Configurations section will be able to use placeholders like ${USER}, ${CLUSTER_NAME} and ${COMPONENT_NAME} to be replaced at runtime with user the app is submitted as, application name and application component name respectively. Launch command can use placeholders like ${APP_COMPONENT_NAME} and ${APP_NAME} to get its component name and app name respectively at runtime. The best part of this feature is that when the component is flexed up, entirely new components (with
new trailing ids) are created.")
@JsonProperty("unique_component_support")
public Boolean getUniqueComponentSupport() {
return uniqueComponentSupport;
@@ -316,8 +319,7 @@ public class Component {
&& Objects.equals(this.artifact, component.artifact)
&& Objects.equals(this.launchCommand, component.launchCommand)
&& Objects.equals(this.resource, component.resource)
- && Objects
- .equals(this.numberOfContainers, component.numberOfContainers)
+ && Objects.equals(this.numberOfContainers, component.numberOfContainers)
&& Objects.equals(this.uniqueComponentSupport,
component.uniqueComponentSupport)
&& Objects.equals(this.runPrivilegedContainer,
@@ -354,8 +356,8 @@ public class Component {
.append(toIndentedString(uniqueComponentSupport)).append("\n");
sb.append(" runPrivilegedContainer: ")
.append(toIndentedString(runPrivilegedContainer)).append("\n");
- sb.append(" placementPolicy: ")
- .append(toIndentedString(placementPolicy)).append("\n");
+ sb.append(" placementPolicy: ").append(toIndentedString(placementPolicy))
+ .append("\n");
sb.append(" configuration: ").append(toIndentedString(configuration))
.append("\n");
sb.append(" quicklinks: ").append(toIndentedString(quicklinks))
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc026492/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java
index 2faf6f2..4e40102 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java
@@ -29,6 +29,10 @@ import javax.xml.bind.annotation.XmlRootElement;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
+/**
+ * An instance of a running application container.
+ **/
+
@ApiModel(description = "An instance of a running application container")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@XmlRootElement
@@ -47,14 +51,14 @@ public class Container extends BaseResource {
/**
* Unique container id of a running application, e.g.
- * container_e3751_1458061340047_0008_01_000002
+ * container_e3751_1458061340047_0008_01_000002.
**/
public Container id(String id) {
this.id = id;
return this;
}
- @ApiModelProperty(example = "null", value = "Unique container id of a running application, e.g. container_e3751_1458061340047_0008_01_000002")
+ @ApiModelProperty(example = "null", value = "Unique container id of a running application, e.g. container_e3751_1458061340047_0008_01_000002.")
@JsonProperty("id")
public String getId() {
return id;
@@ -65,7 +69,8 @@ public class Container extends BaseResource {
}
/**
- * The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.
+ * The time when the container was created, e.g. 2016-03-16T01:01:49.000Z.
+ * This will most likely be different from cluster launch time.
**/
public Container launchTime(Date launchTime) {
this.launchTime = launchTime;
@@ -126,14 +131,14 @@ public class Container extends BaseResource {
/**
* The bare node or host in which the container is running, e.g.
- * cn008.example.com
+ * cn008.example.com.
**/
public Container bareHost(String bareHost) {
this.bareHost = bareHost;
return this;
}
- @ApiModelProperty(example = "null", value = "The bare node or host in which the container is running, e.g. cn008.example.com")
+ @ApiModelProperty(example = "null", value = "The bare node or host in which the container is running, e.g. cn008.example.com.")
@JsonProperty("bare_host")
public String getBareHost() {
return bareHost;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc026492/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ReadinessCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ReadinessCheck.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ReadinessCheck.java
index 80fdf92..10c951a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ReadinessCheck.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ReadinessCheck.java
@@ -57,15 +57,15 @@ public class ReadinessCheck {
private Artifact artifact = null;
/**
- * http (YARN will perform a simple REST call at a regular interval and expect
- * a 204 No content).
+ * E.g. HTTP (YARN will perform a simple REST call at a regular interval and
+ * expect a 204 No content).
**/
public ReadinessCheck type(TypeEnum type) {
this.type = type;
return this;
}
- @ApiModelProperty(example = "null", value = "http (YARN will perform a simple REST call at a regular interval and expect a 204 No content).")
+ @ApiModelProperty(example = "null", value = "E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content).")
@JsonProperty("type")
public TypeEnum getType() {
return type;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/51] [abbrv] hadoop git commit: YARN-5968. Fix slider core module
javadocs. Contributed by Billie Rinaldi
Posted by ji...@apache.org.
YARN-5968. Fix slider core module javadocs. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/01813cfd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/01813cfd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/01813cfd
Branch: refs/heads/yarn-native-services
Commit: 01813cfd13b1acf7da14e12ba3fde2673d04b28c
Parents: 562c826
Author: Jian He <ji...@apache.org>
Authored: Mon Dec 19 13:06:56 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
hadoop-yarn-project/hadoop-yarn/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01813cfd/hadoop-yarn-project/hadoop-yarn/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml
index c929292..ac3654c 100644
--- a/hadoop-yarn-project/hadoop-yarn/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/pom.xml
@@ -75,7 +75,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<configuration>
- <excludePackageNames>org.apache.hadoop.yarn.proto</excludePackageNames>
+ <excludePackageNames>org.apache.hadoop.yarn.proto,org.apache.slider</excludePackageNames>
</configuration>
</plugin>
</plugins>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/51] [abbrv] hadoop git commit: YARN-5513. Move Java only tests
from slider develop to yarn-native-services. Contributed by Gour Saha
Posted by ji...@apache.org.
YARN-5513. Move Java only tests from slider develop to yarn-native-services. Contributed by Gour Saha
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4826d902
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4826d902
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4826d902
Branch: refs/heads/yarn-native-services
Commit: 4826d9026f378d540cd3f99dbcb9c2002f280395
Parents: 0d83f84
Author: Jian He <ji...@apache.org>
Authored: Wed Aug 17 00:42:24 2016 +0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:07:43 2016 -0800
----------------------------------------------------------------------
.../dev-support/findbugs-exclude.xml | 20 +
.../hadoop-yarn-slider-core/pom.xml | 22 +
.../slider/common/tools/TestSliderUtils.java | 159 ++++
.../core/launch/TestAppMasterLauncher.java | 157 ++++
.../TestAppMasterLauncherWithAmReset.java | 92 ++
.../TestPublishedConfigurationOutputter.java | 222 +++++
.../agent/TestAgentClientProvider.java | 77 ++
.../agent/TestAgentLaunchParameter.java | 76 ++
.../slider/providers/agent/TestAgentUtils.java | 94 ++
.../agent/TestAppDefinitionPersister.java | 264 ++++++
.../agent/TestComponentTagProvider.java | 115 +++
.../slider/providers/agent/TestState.java | 33 +
.../application/metadata/TestConfigParser.java | 107 +++
.../metadata/TestMetainfoParser.java | 177 ++++
.../appmaster/TestServiceRecordAttributes.java | 68 ++
.../publisher/TestAgentProviderService.java | 60 ++
.../publisher/TestSliderProviderFactory.java | 40 +
.../server/servicemonitor/TestPortProbe.java | 37 +
.../security/TestCertificateManager.java | 540 +++++++++++
.../TestMultiThreadedStoreGeneration.java | 156 ++++
.../server/services/workflow/MockService.java | 80 ++
.../workflow/ParentWorkflowTestBase.java | 70 ++
.../workflow/ProcessCommandFactory.java | 96 ++
.../services/workflow/SimpleRunnable.java | 46 +
.../workflow/TestWorkflowClosingService.java | 116 +++
.../workflow/TestWorkflowCompositeService.java | 113 +++
.../workflow/TestWorkflowExecutorService.java | 66 ++
.../workflow/TestWorkflowRpcService.java | 107 +++
.../workflow/TestWorkflowSequenceService.java | 151 ++++
.../TestWorkflowServiceTerminatingRunnable.java | 64 ++
.../workflow/WorkflowServiceTestBase.java | 139 +++
.../apache/slider/test/ContractTestUtils.java | 901 +++++++++++++++++++
.../slider/test/MiniZooKeeperCluster.java | 395 ++++++++
.../org/apache/slider/tools/TestUtility.java | 181 ++++
.../slider/common/tools/test/metainfo.txt | 16 +
.../slider/common/tools/test/metainfo.xml | 98 ++
.../slider/common/tools/test/someOtherFile.txt | 16 +
.../slider/common/tools/test/someOtherFile.xml | 17 +
.../agent/application/metadata/metainfo.xml | 180 ++++
39 files changed, 5368 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml
new file mode 100644
index 0000000..b89146a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<FindBugsFilter>
+
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index 591a5ca..d778f44 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -129,6 +129,13 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</dependency>
@@ -268,6 +275,20 @@
</dependency>
<dependency>
+ <groupId>org.easymock</groupId>
+ <artifactId>easymock</artifactId>
+ <version>3.1</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-easymock</artifactId>
+ <version>1.5</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
</dependency>
@@ -292,6 +313,7 @@
<groupId>org.codehaus.jettison</groupId>
<artifactId>jettison</artifactId>
</dependency>
+
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-sslengine</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
new file mode 100644
index 0000000..20e72c0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.common.tools;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationReportPBImpl;
+import org.apache.slider.tools.TestUtility;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+/** Test slider util methods. */
+public class TestSliderUtils {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestSliderUtils.class);
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+
+ @Test
+ public void testGetMetaInfoStreamFromZip() throws Exception {
+ String zipFileName = TestUtility.createAppPackage(
+ folder,
+ "testpkg",
+ "test.zip",
+ "target/test-classes/org/apache/slider/common/tools/test");
+ Configuration configuration = new Configuration();
+ FileSystem fs = FileSystem.getLocal(configuration);
+ log.info("fs working dir is {}", fs.getWorkingDirectory().toString());
+ SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration);
+
+ InputStream stream = SliderUtils.getApplicationResourceInputStream(
+ sliderFileSystem.getFileSystem(),
+ new Path(zipFileName),
+ "metainfo.xml");
+ Assert.assertTrue(stream != null);
+ Assert.assertTrue(stream.available() > 0);
+ }
+
+ @Test
+ public void testTruncate() {
+ Assert.assertEquals(SliderUtils.truncate(null, 5), null);
+ Assert.assertEquals(SliderUtils.truncate("323", -1), "323");
+ Assert.assertEquals(SliderUtils.truncate("3232", 5), "3232");
+ Assert.assertEquals(SliderUtils.truncate("1234567890", 0), "1234567890");
+ Assert.assertEquals(SliderUtils.truncate("123456789012345", 15), "123456789012345");
+ Assert.assertEquals(SliderUtils.truncate("123456789012345", 14), "12345678901...");
+ Assert.assertEquals(SliderUtils.truncate("1234567890", 1), "1");
+ Assert.assertEquals(SliderUtils.truncate("1234567890", 10), "1234567890");
+ Assert.assertEquals(SliderUtils.truncate("", 10), "");
+ }
+
+ @Test
+ public void testApplicationReportComparison() {
+ List<ApplicationReport> instances = getApplicationReports();
+
+ SliderUtils.sortApplicationsByMostRecent(instances);
+
+ Assert.assertEquals(1000, instances.get(0).getStartTime());
+ Assert.assertEquals(1000, instances.get(1).getStartTime());
+ Assert.assertEquals(1000, instances.get(2).getStartTime());
+ Assert.assertEquals(1000, instances.get(3).getStartTime());
+
+ instances = getApplicationReports();
+
+ SliderUtils.sortApplicationReport(instances);
+ Assert.assertEquals(1000, instances.get(0).getStartTime());
+ Assert.assertEquals(1000, instances.get(1).getStartTime());
+ Assert.assertEquals(1000, instances.get(2).getStartTime());
+ Assert.assertEquals(1000, instances.get(3).getStartTime());
+
+ Assert.assertTrue(instances.get(0).getYarnApplicationState() == YarnApplicationState.ACCEPTED ||
+ instances.get(0).getYarnApplicationState() == YarnApplicationState.RUNNING);
+ Assert.assertTrue(instances.get(1).getYarnApplicationState() == YarnApplicationState.ACCEPTED ||
+ instances.get(1).getYarnApplicationState() == YarnApplicationState.RUNNING);
+ Assert.assertTrue(instances.get(2).getYarnApplicationState() == YarnApplicationState.ACCEPTED ||
+ instances.get(2).getYarnApplicationState() == YarnApplicationState.RUNNING);
+ Assert.assertTrue(instances.get(3).getYarnApplicationState() == YarnApplicationState.KILLED);
+ }
+
+ private List<ApplicationReport> getApplicationReports() {
+ List<ApplicationReport> instances = new ArrayList<ApplicationReport>();
+ instances.add(getApplicationReport(1000, 0, "app1", YarnApplicationState.ACCEPTED));
+ instances.add(getApplicationReport(900, 998, "app1", YarnApplicationState.KILLED));
+ instances.add(getApplicationReport(900, 998, "app2", YarnApplicationState.FAILED));
+ instances.add(getApplicationReport(1000, 0, "app2", YarnApplicationState.RUNNING));
+ instances.add(getApplicationReport(800, 837, "app3", YarnApplicationState.FINISHED));
+ instances.add(getApplicationReport(1000, 0, "app3", YarnApplicationState.RUNNING));
+ instances.add(getApplicationReport(900, 998, "app3", YarnApplicationState.KILLED));
+ instances.add(getApplicationReport(800, 837, "app4", YarnApplicationState.FINISHED));
+ instances.add(getApplicationReport(1000, 1050, "app4", YarnApplicationState.KILLED));
+ instances.add(getApplicationReport(900, 998, "app4", YarnApplicationState.FINISHED));
+
+ Assert.assertEquals("app1", instances.get(0).getApplicationType());
+ Assert.assertEquals("app1", instances.get(1).getApplicationType());
+ Assert.assertEquals("app2", instances.get(2).getApplicationType());
+ Assert.assertEquals("app2", instances.get(3).getApplicationType());
+ return instances;
+ }
+
+ private ApplicationReportPBImpl getApplicationReport(long startTime,
+ long finishTime,
+ String name,
+ YarnApplicationState state) {
+ ApplicationReportPBImpl ar = new ApplicationReportPBImpl();
+ ar.setFinishTime(finishTime);
+ ar.setStartTime(startTime);
+ ar.setApplicationType(name);
+ ar.setYarnApplicationState(state);
+ return ar;
+ }
+
+
+ @Test
+ public void testGetHdpVersion() {
+ String hdpVersion = "2.3.2.0-2766";
+ Assert.assertEquals("Version should be empty", null,
+ SliderUtils.getHdpVersion());
+ }
+
+ @Test
+ public void testIsHdp() {
+ Assert.assertFalse("Should be false", SliderUtils.isHdp());
+ }
+
+ @Test
+ public void testWrite() throws IOException {
+ File testWriteFile = folder.newFile("testWrite");
+ SliderUtils.write(testWriteFile, "test".getBytes("UTF-8"), true);
+ Assert.assertTrue(FileUtils.readFileToString(testWriteFile, "UTF-8").equals("test"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
new file mode 100644
index 0000000..b955931
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncher.java
@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.launch;
+
+import java.lang.reflect.Method;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.LogAggregationContext;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.client.SliderYarnClientImpl;
+import org.apache.slider.common.SliderKeys;
+import org.easymock.EasyMock;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestAppMasterLauncher {
+ SliderYarnClientImpl mockYarnClient;
+ YarnClientApplication yarnClientApp;
+ ApplicationSubmissionContext appSubmissionContext;
+ Set<String> tags = Collections.emptySet();
+ AppMasterLauncher appMasterLauncher = null;
+ boolean isOldApi = true;
+ Method rolledLogsIncludeMethod = null;
+ Method rolledLogsExcludeMethod = null;
+
+ @Before
+ public void initialize() throws Exception {
+ mockYarnClient = EasyMock.createNiceMock(SliderYarnClientImpl.class);
+ yarnClientApp = EasyMock.createNiceMock(YarnClientApplication.class);
+ appSubmissionContext = EasyMock
+ .createNiceMock(ApplicationSubmissionContext.class);
+ EasyMock.expect(yarnClientApp.getApplicationSubmissionContext())
+ .andReturn(appSubmissionContext).once();
+ EasyMock.expect(mockYarnClient.createApplication())
+ .andReturn(yarnClientApp).once();
+
+ try {
+ LogAggregationContext.class.getMethod("newInstance", String.class,
+ String.class, String.class, String.class);
+ isOldApi = false;
+ rolledLogsIncludeMethod = LogAggregationContext.class
+ .getMethod("getRolledLogsIncludePattern");
+ rolledLogsExcludeMethod = LogAggregationContext.class
+ .getMethod("getRolledLogsExcludePattern");
+ } catch (Exception e) {
+ isOldApi = true;
+ }
+ }
+
+ /**
+ * These tests will probably fail when compiled against hadoop 2.7+. Please
+ * refer to SLIDER-810. It has been purposely not modified so that it fails
+ * and that someone needs to modify the code in
+ * {@code AbstractLauncher#extractLogAggregationContext(Map)}. Comments are
+ * provided in that method as to what needs to be done.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testExtractLogAggregationContext() throws Exception {
+ Map<String, String> options = new HashMap<String, String>();
+ options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS,
+ " | slider*.txt |agent.out| |");
+ options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS,
+ "command*.json| agent.log* | ");
+
+ EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp);
+ appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null,
+ null, mockYarnClient, false, null, options, tags, null);
+
+ // Verify the include/exclude patterns
+ String expectedInclude = "slider*.txt|agent.out";
+ String expectedExclude = "command*.json|agent.log*";
+ assertPatterns(expectedInclude, expectedExclude);
+
+ EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp);
+
+ }
+
+ @Test
+ public void testExtractLogAggregationContextEmptyIncludePattern()
+ throws Exception {
+ Map<String, String> options = new HashMap<String, String>();
+ options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, " ");
+ options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS,
+ "command*.json| agent.log* | ");
+
+ EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp);
+ appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null,
+ null, mockYarnClient, false, null, options, tags, null);
+
+ // Verify the include/exclude patterns
+ String expectedInclude = isOldApi ? "" : ".*";
+ String expectedExclude = "command*.json|agent.log*";
+ assertPatterns(expectedInclude, expectedExclude);
+
+ EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp);
+ }
+
+ @Test
+ public void testExtractLogAggregationContextEmptyIncludeAndExcludePattern()
+ throws Exception {
+ Map<String, String> options = new HashMap<String, String>();
+ options.put(ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, "");
+ options.put(ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, " ");
+
+ EasyMock.replay(mockYarnClient, appSubmissionContext, yarnClientApp);
+ appMasterLauncher = new AppMasterLauncher("cl1", SliderKeys.APP_TYPE, null,
+ null, mockYarnClient, false, null, options, tags, null);
+
+ // Verify the include/exclude patterns
+ String expectedInclude = isOldApi ? "" : ".*";
+ String expectedExclude = "";
+ assertPatterns(expectedInclude, expectedExclude);
+
+ EasyMock.verify(mockYarnClient, appSubmissionContext, yarnClientApp);
+ }
+
+ private void assertPatterns(String expectedIncludePattern,
+ String expectedExcludePattern) throws Exception {
+ if (isOldApi) {
+ Assert.assertEquals(expectedIncludePattern,
+ appMasterLauncher.logAggregationContext.getIncludePattern());
+ Assert.assertEquals(expectedExcludePattern,
+ appMasterLauncher.logAggregationContext.getExcludePattern());
+ } else {
+ Assert.assertEquals(expectedIncludePattern,
+ (String) rolledLogsIncludeMethod
+ .invoke(appMasterLauncher.logAggregationContext));
+ Assert.assertEquals(expectedExcludePattern,
+ (String) rolledLogsExcludeMethod
+ .invoke(appMasterLauncher.logAggregationContext));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java
new file mode 100644
index 0000000..a8f6b26
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.launch;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.client.SliderYarnClientImpl;
+import org.apache.slider.common.SliderKeys;
+import org.easymock.EasyMock;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestAppMasterLauncherWithAmReset {
+ SliderYarnClientImpl mockYarnClient;
+ YarnClientApplication yarnClientApp;
+ ApplicationSubmissionContext appSubmissionContext;
+ GetNewApplicationResponse newApp;
+ Set<String> tags = Collections.emptySet();
+ AppMasterLauncher appMasterLauncher = null;
+ boolean isOldApi = true;
+
+ @Before
+ public void initialize() throws Exception {
+ mockYarnClient = EasyMock.createNiceMock(SliderYarnClientImpl.class);
+ yarnClientApp = EasyMock.createNiceMock(YarnClientApplication.class);
+ newApp = EasyMock.createNiceMock(GetNewApplicationResponse.class);
+ EasyMock.expect(mockYarnClient.createApplication())
+ .andReturn(new YarnClientApplication(newApp,
+ Records.newRecord(ApplicationSubmissionContext.class)));
+ }
+
+ @Test
+ public void testExtractYarnResourceManagerAmRetryCountWindowMs() throws
+ Exception {
+ Map<String, String> options = new HashMap<String, String>();
+ final String expectedInterval = Integer.toString (120000);
+ options.put(ResourceKeys.YARN_RESOURCEMANAGER_AM_RETRY_COUNT_WINDOW_MS,
+ expectedInterval);
+ EasyMock.replay(mockYarnClient, yarnClientApp);
+
+ appMasterLauncher = new AppMasterLauncher("am1", SliderKeys.APP_TYPE, null,
+ null, mockYarnClient, false, null, options, tags, null);
+
+ ApplicationSubmissionContext ctx = appMasterLauncher.application
+ .getApplicationSubmissionContext();
+ String retryIntervalWindow = Long.toString(ctx
+ .getAttemptFailuresValidityInterval());
+ Assert.assertEquals(expectedInterval, retryIntervalWindow);
+ }
+
+ @Test
+ public void testExtractYarnResourceManagerAmRetryCountWindowMsDefaultValue()
+ throws Exception {
+ Map<String, String> options = new HashMap<String, String>();
+ EasyMock.replay(mockYarnClient, yarnClientApp);
+
+ appMasterLauncher = new AppMasterLauncher("am1", SliderKeys.APP_TYPE, null,
+ null, mockYarnClient, false, null, options, tags, null);
+
+ ApplicationSubmissionContext ctx = appMasterLauncher.application
+ .getApplicationSubmissionContext();
+ long retryIntervalWindow = ctx.getAttemptFailuresValidityInterval();
+ Assert.assertEquals(ResourceKeys.DEFAULT_AM_RETRY_COUNT_WINDOW_MS,
+ retryIntervalWindow);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
new file mode 100644
index 0000000..3706182
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.core.registry.docstore;
+
+import com.google.common.base.Charsets;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.powermock.api.easymock.PowerMock;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.expect;
+import static org.mockito.Matchers.anyString;
+import static org.powermock.api.easymock.PowerMock.createNiceMock;
+
+public class TestPublishedConfigurationOutputter {
+ private static HashMap<String, String> config = new HashMap<>();
+
+ @Rule
+ public TemporaryFolder tmpDir = new TemporaryFolder();
+
+ @Before
+ public void setup() {
+ config.put("key1", "val1");
+ }
+
+ @Test
+ public void testJson() throws IOException {
+ PublishedConfigurationOutputter configurationOutputter =
+ PublishedConfigurationOutputter.createOutputter(ConfigFormat.JSON,
+ new PublishedConfiguration("description",
+ config.entrySet()));
+
+ String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
+ "");
+ assert "{\"key1\":\"val1\"}".equals(output);
+
+ File file = tmpDir.newFile();
+ configurationOutputter.save(file);
+
+ ObjectMapper mapper = new ObjectMapper();
+ @SuppressWarnings("unchecked")
+ Map<String, String> read = mapper.readValue(file, Map.class);
+ assert 1 == read.size();
+ assert "val1".equals(read.get("key1"));
+ }
+
+ @Test
+ public void testXml() throws IOException {
+ PublishedConfigurationOutputter configurationOutputter =
+ PublishedConfigurationOutputter.createOutputter(ConfigFormat.XML,
+ new PublishedConfiguration("description",
+ config.entrySet()));
+
+ String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
+ "");
+ assert output.contains(
+ "<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>");
+
+ File file = tmpDir.newFile();
+ configurationOutputter.save(file);
+
+ assert FileUtils.readFileToString(file, Charsets.UTF_8)
+ .replaceAll("( |\\r|\\n)", "")
+ .contains(
+ "<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>");
+ }
+
+ @Test
+ public void testHadoopXml() throws IOException {
+ PublishedConfigurationOutputter configurationOutputter =
+ PublishedConfigurationOutputter.createOutputter(ConfigFormat.HADOOP_XML,
+ new PublishedConfiguration("description",
+ config.entrySet()));
+
+ String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)",
+ "");
+ assert output.contains("<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>");
+
+ File file = tmpDir.newFile();
+ configurationOutputter.save(file);
+
+ assert FileUtils.readFileToString(file, Charsets.UTF_8)
+ .replaceAll("( |\\r|\\n)", "")
+ .contains( "<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>");
+ }
+
+ @Test
+ public void testProperties() throws IOException {
+ PublishedConfigurationOutputter configurationOutputter =
+ PublishedConfigurationOutputter.createOutputter(ConfigFormat.PROPERTIES,
+ new PublishedConfiguration("description",
+ config.entrySet()));
+
+ String output = configurationOutputter.asString();
+ assert output.contains("key1=val1");
+
+ File file = tmpDir.newFile();
+ configurationOutputter.save(file);
+
+ Properties properties = new Properties();
+ FileInputStream fis = null;
+ try {
+ fis = new FileInputStream(file);
+ properties.load(fis);
+ } finally {
+ if (fis != null) {
+ fis.close();
+ }
+ }
+ assert 1 == properties.size();
+ assert "val1".equals(properties.getProperty("key1"));
+ }
+
+ @Test
+ public void testYaml() throws IOException {
+ PublishedConfigurationOutputter configurationOutputter =
+ PublishedConfigurationOutputter.createOutputter(ConfigFormat.YAML,
+ new PublishedConfiguration("description",
+ config.entrySet()));
+
+ String output = configurationOutputter.asString().replaceAll("(\\r|\\n)",
+ "");
+ assert "key1: val1".equals(output);
+
+ File file = tmpDir.newFile();
+ configurationOutputter.save(file);
+
+ Yaml yaml = new Yaml();
+ FileInputStream fis = null;
+ Map<String, String> read;
+ try {
+ fis = new FileInputStream(file);
+ read = (Map<String, String>) yaml.load(fis);
+ } finally {
+ if (fis != null) {
+ fis.close();
+ }
+ }
+ assert 1 == read.size();
+ assert "val1".equals(read.get("key1"));
+ }
+
+ @Test
+ public void testEnv() throws IOException {
+ HashMap<String, String> envConfig = new HashMap<>(config);
+ envConfig.put("content", "content {{key1}} ");
+
+ PublishedConfigurationOutputter configurationOutputter =
+ PublishedConfigurationOutputter.createOutputter(ConfigFormat.ENV,
+ new PublishedConfiguration("description",
+ envConfig.entrySet()));
+
+ String output = configurationOutputter.asString();
+ assert "content val1 ".equals(output);
+
+ File file = tmpDir.newFile();
+ configurationOutputter.save(file);
+
+ assert "content val1 ".equals(FileUtils.readFileToString(file,
+ Charsets.UTF_8));
+ }
+
+ @Test
+ public void testTemplate1() throws IOException {
+ HashMap<String, String> templateConfig = new HashMap<>(config);
+ templateConfig.put(ConfigUtils.TEMPLATE_FILE, "templateFileName");
+
+ SliderFileSystem fileSystem = createNiceMock(SliderFileSystem.class);
+ expect(fileSystem.buildResourcePath(anyString())).andReturn(new Path("path")).anyTimes();
+ expect(fileSystem.isFile(anyObject(Path.class))).andReturn(true).anyTimes();
+ expect(fileSystem.cat(anyObject(Path.class))).andReturn("content {{key1}}\n more ${key1} content").anyTimes();
+
+ PowerMock.replay(fileSystem);
+
+ ConfigUtils.prepConfigForTemplateOutputter(ConfigFormat.TEMPLATE,
+ templateConfig, fileSystem, "clusterName", null);
+ PublishedConfigurationOutputter configurationOutputter =
+ PublishedConfigurationOutputter.createOutputter(ConfigFormat.TEMPLATE,
+ new PublishedConfiguration("description",
+ templateConfig.entrySet()));
+
+ String output = configurationOutputter.asString();
+ assert "content val1\n more val1 content".equals(output);
+
+ File file = tmpDir.newFile();
+ configurationOutputter.save(file);
+
+ PowerMock.verify(fileSystem);
+
+ assert "content val1\n more val1 content".equals(
+ FileUtils.readFileToString(file, Charsets.UTF_8));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
new file mode 100644
index 0000000..0bea8fa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.tools.TestUtility;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Set;
+
+/**
+ *
+ */
+public class TestAgentClientProvider {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestAgentClientProvider.class);
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+
+ @Test
+ public void testGetApplicationTags() throws Exception {
+ Configuration configuration = new Configuration();
+ FileSystem fs = FileSystem.getLocal(configuration);
+ SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration);
+
+ AgentClientProvider provider = new AgentClientProvider(null);
+ String zipFileName = TestUtility.createAppPackage(
+ folder,
+ "testpkg",
+ "test.zip",
+ "target/test-classes/org/apache/slider/common/tools/test");
+ Set<String> tags = provider.getApplicationTags(sliderFileSystem, zipFileName);
+ assert tags != null;
+ assert !tags.isEmpty();
+ assert tags.contains("Name: STORM");
+ assert tags.contains("Description: Apache Hadoop Stream processing framework");
+ assert tags.contains("Version: 0.9.1.2.1");
+
+ }
+
+ @Test
+ public void testValidateInstanceDefinition() throws Exception {
+ AgentClientProvider provider = new AgentClientProvider(null);
+ AggregateConf instanceDefinition = new AggregateConf();
+
+ try {
+ provider.validateInstanceDefinition(instanceDefinition, null);
+ Assert.assertFalse("Should fail with BadConfigException", true);
+ } catch (BadConfigException e) {
+ log.info(e.toString());
+ Assert.assertTrue(e.getMessage().contains("Application definition must be provided"));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentLaunchParameter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentLaunchParameter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentLaunchParameter.java
new file mode 100644
index 0000000..ec62b54
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentLaunchParameter.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ *
+ */
+public class TestAgentLaunchParameter {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestAgentLaunchParameter.class);
+
+ @Test
+ public void testTestAgentLaunchParameter() throws Exception {
+ AgentLaunchParameter alp = new AgentLaunchParameter("");
+ Assert.assertEquals("", alp.getNextLaunchParameter("abc"));
+ Assert.assertEquals("", alp.getNextLaunchParameter("HBASE_MASTER"));
+
+ alp = new AgentLaunchParameter("a:1:2:3|b:5:6:NONE");
+ Assert.assertEquals("1", alp.getNextLaunchParameter("a"));
+ Assert.assertEquals("2", alp.getNextLaunchParameter("a"));
+ Assert.assertEquals("3", alp.getNextLaunchParameter("a"));
+ Assert.assertEquals("3", alp.getNextLaunchParameter("a"));
+
+ Assert.assertEquals("5", alp.getNextLaunchParameter("b"));
+ Assert.assertEquals("6", alp.getNextLaunchParameter("b"));
+ Assert.assertEquals("", alp.getNextLaunchParameter("b"));
+ Assert.assertEquals("", alp.getNextLaunchParameter("b"));
+ Assert.assertEquals("", alp.getNextLaunchParameter("c"));
+
+ alp = new AgentLaunchParameter("|a:1:3|b::5:NONE:");
+ Assert.assertEquals("1", alp.getNextLaunchParameter("a"));
+ Assert.assertEquals("3", alp.getNextLaunchParameter("a"));
+ Assert.assertEquals("3", alp.getNextLaunchParameter("a"));
+
+ Assert.assertEquals("", alp.getNextLaunchParameter("b"));
+ Assert.assertEquals("5", alp.getNextLaunchParameter("b"));
+ Assert.assertEquals("", alp.getNextLaunchParameter("b"));
+ Assert.assertEquals("", alp.getNextLaunchParameter("b"));
+
+ alp = new AgentLaunchParameter("|:");
+ Assert.assertEquals("", alp.getNextLaunchParameter("b"));
+ Assert.assertEquals("", alp.getNextLaunchParameter("a"));
+
+ alp = new AgentLaunchParameter("HBASE_MASTER:a,b:DO_NOT_REGISTER:");
+ Assert.assertEquals("a,b", alp.getNextLaunchParameter("HBASE_MASTER"));
+ Assert.assertEquals("DO_NOT_REGISTER", alp.getNextLaunchParameter("HBASE_MASTER"));
+ Assert.assertEquals("DO_NOT_REGISTER", alp.getNextLaunchParameter("HBASE_MASTER"));
+
+ alp = new AgentLaunchParameter("HBASE_MASTER:a,b:DO_NOT_REGISTER::c:::");
+ Assert.assertEquals("a,b", alp.getNextLaunchParameter("HBASE_MASTER"));
+ Assert.assertEquals("DO_NOT_REGISTER", alp.getNextLaunchParameter("HBASE_MASTER"));
+ Assert.assertEquals("", alp.getNextLaunchParameter("HBASE_MASTER"));
+ Assert.assertEquals("c", alp.getNextLaunchParameter("HBASE_MASTER"));
+ Assert.assertEquals("c", alp.getNextLaunchParameter("HBASE_MASTER"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java
new file mode 100644
index 0000000..5e1dc7f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.providers.agent.application.metadata.Metainfo;
+import org.apache.slider.tools.TestUtility;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestAgentUtils {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestAgentUtils.class);
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+ private static final String metainfo_str = "<metainfo>\n"
+ + " <schemaVersion>2.0</schemaVersion>\n"
+ + " <application>\n"
+ + " <name>MYTESTAPPLICATION</name>\n"
+ + " <comment>\n"
+ + " My Test Application\n"
+ + " </comment>\n"
+ + " <version>1.0</version>\n"
+ + " <type>YARN-APP</type>\n"
+ + " <components>\n"
+ + " <component>\n"
+ + " <name>REST</name>\n"
+ + " <category>MASTER</category>\n"
+ + " <commandScript>\n"
+ + " <script>scripts/rest.py</script>\n"
+ + " <scriptType>PYTHON</scriptType>\n"
+ + " <timeout>600</timeout>\n"
+ + " </commandScript>\n"
+ + " </component>\n"
+ + " </components>\n"
+ + " </application>\n"
+ + "</metainfo>";
+
+ @Test
+ public void testGetApplicationMetainfo() throws Exception {
+ String zipFileName = TestUtility.createAppPackage(
+ folder,
+ "testpkg",
+ "test.zip",
+ "target/test-classes/org/apache/slider/common/tools/test");
+ Configuration configuration = new Configuration();
+ FileSystem fs = FileSystem.getLocal(configuration);
+ log.info("fs working dir is {}", fs.getWorkingDirectory().toString());
+ SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration);
+
+ // Without accompany metainfo file, read metainfo from the zip file
+ Metainfo metainfo = AgentUtils.getApplicationMetainfo(
+ sliderFileSystem, zipFileName, false);
+ Assert.assertNotNull(metainfo.getApplication());
+ Assert.assertEquals("STORM", metainfo.getApplication().getName());
+
+ // With accompany metainfo file, read metainfo from the accompany file
+ String acompanyFileName = zipFileName + ".metainfo.xml";
+ File f = new File(acompanyFileName);
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(f))) {
+ writer.write(metainfo_str);
+ }
+ metainfo = AgentUtils.getApplicationMetainfo(
+ sliderFileSystem, zipFileName, false);
+ Assert.assertNotNull(metainfo.getApplication());
+ Assert.assertEquals("MYTESTAPPLICATION", metainfo.getApplication().getName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java
new file mode 100644
index 0000000..dedf4f6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent;
+
+import com.google.common.io.Files;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.slider.common.params.ActionCreateArgs;
+import org.apache.slider.common.params.AddonArgsDelegate;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.persist.AppDefinitionPersister;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ */
+public class TestAppDefinitionPersister {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestAppDefinitionPersister.class);
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+
+ /**
+ * @BeforeClass public static void initialize() { BasicConfigurator.resetConfiguration();
+ * BasicConfigurator.configure(); }*
+ */
+
+
+ @Test
+ public void testAppDefinitionPersister() throws Exception {
+ Configuration configuration = new Configuration();
+ FileSystem fs = FileSystem.getLocal(configuration);
+ log.info("fs working dir is {}", fs.getWorkingDirectory().toString());
+ SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration);
+
+ AppDefinitionPersister adp = new AppDefinitionPersister(sliderFileSystem);
+ String clustername = "c1";
+ ActionCreateArgs buildInfo = new ActionCreateArgs();
+ buildInfo.appMetaInfo = null;
+ buildInfo.appDef = null;
+ buildInfo.addonDelegate = new AddonArgsDelegate();
+
+ // nothing to do
+ adp.processSuppliedDefinitions(clustername, buildInfo, null);
+ adp.persistPackages();
+ List<AppDefinitionPersister.AppDefinition> appDefinitions = adp.getAppDefinitions();
+ Assert.assertTrue(appDefinitions.size() == 0);
+
+ ConfTree ct = new ConfTree();
+ ConfTreeOperations appConf = new ConfTreeOperations(ct);
+ final File tempDir = Files.createTempDir();
+ final File metainfo = new File(tempDir, "metainfo.json");
+
+ // unreadable metainfo
+ buildInfo.appMetaInfo = metainfo;
+
+ try {
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ } catch (BadConfigException bce) {
+ log.info(bce.getMessage());
+ Assert.assertTrue(bce.getMessage().contains(
+ "Path specified with "
+ + "--metainfo either cannot be read or is not a file"));
+ }
+
+ try (PrintWriter writer = new PrintWriter(metainfo.getAbsolutePath(), "UTF-8")) {
+ writer.println("{");
+ writer.println("}");
+ }
+ buildInfo.appDef = metainfo;
+
+ try {
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ } catch (BadConfigException bce) {
+ log.info(bce.getMessage());
+ Assert.assertTrue(bce.getMessage().contains(
+ "Both --metainfo and --appdef cannot be specified"));
+ }
+
+ // both --metainfojson and --appdef cannot be specified
+ buildInfo.appMetaInfo = null;
+ buildInfo.appMetaInfoJson = "{}";
+ try {
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ } catch (BadConfigException bce) {
+ log.info(bce.getMessage());
+ Assert.assertTrue(bce.getMessage().contains(
+ "Both --metainfojson and --appdef cannot be specified"));
+ }
+
+ buildInfo.appDef = null;
+
+ buildInfo.appMetaInfoJson = "";
+ try {
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ } catch (BadConfigException bce) {
+ log.info(bce.getMessage());
+ Assert.assertTrue(bce.getMessage().contains(
+ "Empty string specified with --metainfojson"));
+ }
+ buildInfo.appMetaInfo = metainfo;
+
+ // both --metainfo and --metainfojson cannot be specified
+ buildInfo.appMetaInfoJson = "{}";
+ try {
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ } catch (BadConfigException bce) {
+ log.info(bce.getMessage());
+ Assert.assertTrue(bce.getMessage().contains(
+ "Both --metainfo and --metainfojson cannot be specified"));
+ }
+ buildInfo.appMetaInfoJson = null;
+
+ appConf.getGlobalOptions().set(AgentKeys.APP_DEF, metainfo.getAbsolutePath());
+
+ try {
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ } catch (BadConfigException bce) {
+ log.info(bce.getMessage());
+ Assert.assertTrue(bce.getMessage().contains(
+ "application.def cannot "
+ + "not be set if --metainfo is specified in the cmd line"));
+ }
+
+ appConf.getGlobalOptions().remove(AgentKeys.APP_DEF);
+
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ appDefinitions = adp.getAppDefinitions();
+ Assert.assertTrue(appDefinitions.size() == 1);
+ Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip"));
+ log.info(appDefinitions.get(0).toString());
+ Assert.assertTrue(appDefinitions.get(0).appDefPkgOrFolder.toString().endsWith("default"));
+ Assert.assertTrue(appDefinitions.get(0).targetFolderInFs.toString().contains("cluster/c1/appdef"));
+ Assert.assertEquals("appPkg.zip", appDefinitions.get(0).pkgName);
+
+ buildInfo.appDef = tempDir;
+ buildInfo.appMetaInfo = null;
+
+ appConf.getGlobalOptions().set(AgentKeys.APP_DEF, metainfo.getAbsolutePath());
+
+ try {
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ } catch (BadConfigException bce) {
+ log.info(bce.getMessage());
+ Assert.assertTrue(bce.getMessage().contains("application.def must not be set if --appdef is provided"));
+ }
+
+ adp.getAppDefinitions().clear();
+ appConf.getGlobalOptions().remove(AgentKeys.APP_DEF);
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ appDefinitions = adp.getAppDefinitions();
+ Assert.assertTrue(appDefinitions.size() == 1);
+ Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip"));
+ log.info(appDefinitions.get(0).toString());
+ Assert.assertTrue(appDefinitions.get(0).appDefPkgOrFolder.toString().endsWith(tempDir.toString()));
+ Assert.assertTrue(appDefinitions.get(0).targetFolderInFs.toString().contains("cluster/c1/appdef"));
+ Assert.assertEquals("appPkg.zip", appDefinitions.get(0).pkgName);
+
+ adp.getAppDefinitions().clear();
+ buildInfo.appDef = null;
+ buildInfo.appMetaInfo = null;
+ appConf.getGlobalOptions().remove(AgentKeys.APP_DEF);
+
+ ArrayList<String> list = new ArrayList<String>() {{
+ add("addon1");
+ add("");
+ add("addon2");
+ add(metainfo.getAbsolutePath());
+ }};
+
+ buildInfo.addonDelegate.addonTuples = list;
+ try {
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ } catch (BadConfigException bce) {
+ log.info(bce.getMessage());
+ Assert.assertTrue(bce.getMessage().contains("addon package can only be specified if main app package is specified"));
+ }
+
+ buildInfo.appMetaInfo = metainfo;
+
+ try {
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ } catch (BadConfigException bce) {
+ log.info(bce.getMessage());
+ Assert.assertTrue(bce.getMessage().contains("Invalid path for addon package addon1"));
+ }
+
+ appConf.getGlobalOptions().remove(AgentKeys.APP_DEF);
+
+ list = new ArrayList<String>() {{
+ add("addon1");
+ add(tempDir.getAbsolutePath());
+ add("addon2");
+ add(metainfo.getAbsolutePath());
+ }};
+
+ buildInfo.addonDelegate.addonTuples = list;
+ adp.getAppDefinitions().clear();
+
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ appDefinitions = adp.getAppDefinitions();
+
+ Assert.assertTrue(appDefinitions.size() == 3);
+ Assert.assertTrue(appConf.getGlobalOptions().get(AgentKeys.APP_DEF).contains("appdef/appPkg.zip"));
+ Assert.assertTrue(appConf.getGlobalOptions().get("application.addon.addon1").contains(
+ "addons/addon1/addon_addon1.zip"));
+ Assert.assertTrue(appConf.getGlobalOptions().get("application.addon.addon2").contains(
+ "addons/addon2/addon_addon2.zip"));
+ log.info(appConf.getGlobalOptions().get("application.addons"));
+ Assert.assertTrue(appConf.getGlobalOptions().get("application.addons").contains(
+ "application.addon.addon2,application.addon.addon1")
+ || appConf.getGlobalOptions().get("application.addons").contains(
+ "application.addon.addon1,application.addon.addon2"));
+ int seen = 0;
+ for (AppDefinitionPersister.AppDefinition adp_ad : appDefinitions) {
+ if (adp_ad.pkgName.equals("appPkg.zip")) {
+ log.info(adp_ad.toString());
+ Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith("default"));
+ Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("cluster/c1/appdef"));
+ seen++;
+ }
+ if (adp_ad.pkgName.equals("addon_addon1.zip")) {
+ log.info(adp_ad.toString());
+ Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith(tempDir.toString()));
+ Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("addons/addon1"));
+ seen++;
+ }
+ if (adp_ad.pkgName.equals("addon_addon2.zip")) {
+ log.info(adp_ad.toString());
+ Assert.assertTrue(adp_ad.appDefPkgOrFolder.toString().endsWith("metainfo.json"));
+ Assert.assertTrue(adp_ad.targetFolderInFs.toString().contains("addons/addon2"));
+ seen++;
+ }
+ }
+ Assert.assertEquals(3, seen);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentTagProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentTagProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentTagProvider.java
new file mode 100644
index 0000000..7b38ee3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentTagProvider.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class TestComponentTagProvider {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestComponentTagProvider.class);
+
+ @Test
+ public void testTagProvider() throws Exception {
+ ComponentTagProvider ctp = new ComponentTagProvider();
+ Assert.assertEquals("", ctp.getTag(null, null));
+ Assert.assertEquals("", ctp.getTag(null, "cid"));
+ Assert.assertEquals("", ctp.getTag("comp1", null));
+
+ Assert.assertEquals("1", ctp.getTag("comp1", "cid1"));
+ Assert.assertEquals("2", ctp.getTag("comp1", "cid2"));
+ Assert.assertEquals("3", ctp.getTag("comp1", "cid3"));
+ ctp.releaseTag("comp1", "cid2");
+ Assert.assertEquals("2", ctp.getTag("comp1", "cid22"));
+
+ ctp.releaseTag("comp1", "cid4");
+ ctp.recordAssignedTag("comp1", "cid5", "5");
+ Assert.assertEquals("4", ctp.getTag("comp1", "cid4"));
+ Assert.assertEquals("4", ctp.getTag("comp1", "cid4"));
+ Assert.assertEquals("6", ctp.getTag("comp1", "cid6"));
+
+ ctp.recordAssignedTag("comp1", "cid55", "5");
+ Assert.assertEquals("5", ctp.getTag("comp1", "cid55"));
+
+ ctp.recordAssignedTag("comp2", "cidb3", "3");
+ Assert.assertEquals("1", ctp.getTag("comp2", "cidb1"));
+ Assert.assertEquals("2", ctp.getTag("comp2", "cidb2"));
+ Assert.assertEquals("4", ctp.getTag("comp2", "cidb4"));
+
+ ctp.recordAssignedTag("comp2", "cidb5", "six");
+ ctp.recordAssignedTag("comp2", "cidb5", "-55");
+ ctp.recordAssignedTag("comp2", "cidb5", "tags");
+ ctp.recordAssignedTag("comp2", "cidb5", null);
+ ctp.recordAssignedTag("comp2", "cidb5", "");
+ ctp.recordAssignedTag("comp2", "cidb5", "5");
+ Assert.assertEquals("6", ctp.getTag("comp2", "cidb6"));
+
+ ctp.recordAssignedTag("comp2", null, "5");
+ ctp.recordAssignedTag(null, null, "5");
+ ctp.releaseTag("comp1", null);
+ ctp.releaseTag(null, "cid4");
+ ctp.releaseTag(null, null);
+ }
+
+ @Test
+ public void testTagProviderWithThread() throws Exception {
+ ComponentTagProvider ctp = new ComponentTagProvider();
+ Thread thread = new Thread(new Taggged(ctp));
+ Thread thread2 = new Thread(new Taggged(ctp));
+ Thread thread3 = new Thread(new Taggged(ctp));
+ thread.start();
+ thread2.start();
+ thread3.start();
+ ctp.getTag("comp1", "cid50");
+ thread.join();
+ thread2.join();
+ thread3.join();
+ Assert.assertEquals("101", ctp.getTag("comp1", "cid101"));
+ }
+
+ public class Taggged implements Runnable {
+ private final ComponentTagProvider ctp;
+
+ public Taggged(ComponentTagProvider ctp) {
+ this.ctp = ctp;
+ }
+
+ public void run() {
+ for (int i = 0; i < 100; i++) {
+ String containerId = "cid" + (i + 1);
+ this.ctp.getTag("comp1", containerId);
+ }
+ for (int i = 0; i < 100; i++) {
+ String containerId = "cid" + (i + 1);
+ this.ctp.getTag("comp1", containerId);
+ }
+ for (int i = 0; i < 100; i += 2) {
+ String containerId = "cid" + (i + 1);
+ this.ctp.releaseTag("comp1", containerId);
+ }
+ for (int i = 0; i < 100; i += 2) {
+ String containerId = "cid" + (i + 1);
+ this.ctp.getTag("comp1", containerId);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestState.java
new file mode 100644
index 0000000..6a2e5ab5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/TestState.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestState {
+ protected static final Logger log = LoggerFactory.getLogger(TestState.class);
+
+ @Test
+ public void testState() throws Exception {
+ State state = State.STARTED;
+ Assert.assertEquals(Command.STOP, state.getSupportedCommand(false, true));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestConfigParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestConfigParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestConfigParser.java
new file mode 100644
index 0000000..3aa44a1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestConfigParser.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ *
+ */
+public class TestConfigParser {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestConfigParser.class);
+ private static final String config_1_str = "<configuration>\n"
+ + " <property>\n"
+ + " <name>security.client.protocol.acl</name>\n"
+ + " <value>*</value>\n"
+ + " <description>ACL for HRegionInterface protocol implementations (ie. \n"
+ + " clients talking to HRegionServers)\n"
+ + " The ACL is a comma-separated list of user and group names. The user and \n"
+ + " group list is separated by a blank. For e.g. \"alice,bob users,wheel\". \n"
+ + " A special value of \"*\" means all users are allowed.</description>\n"
+ + " </property>\n"
+ + "\n"
+ + " <property>\n"
+ + " <name>security.admin.protocol.acl</name>\n"
+ + " <value>*</value>\n"
+ + " <description>ACL for HMasterInterface protocol implementation (ie. \n"
+ + " clients talking to HMaster for admin operations).\n"
+ + " The ACL is a comma-separated list of user and group names. The user and \n"
+ + " group list is separated by a blank. For e.g. \"alice,bob users,wheel\". \n"
+ + " A special value of \"*\" means all users are allowed.</description>\n"
+ + " </property>\n"
+ + "\n"
+ + " <property>\n"
+ + " <name>security.masterregion.protocol.acl</name>\n"
+ + " <value>*</value>\n"
+ + " <description>ACL for HMasterRegionInterface protocol implementations\n"
+ + " (for HRegionServers communicating with HMaster)\n"
+ + " The ACL is a comma-separated list of user and group names. The user and \n"
+ + " group list is separated by a blank. For e.g. \"alice,bob users,wheel\". \n"
+ + " A special value of \"*\" means all users are allowed.</description>\n"
+ + " </property>\n"
+ + " <property>\n"
+ + " <name>emptyVal</name>\n"
+ + " <value></value>\n"
+ + " <description>non-empty-desc</description>\n"
+ + " </property>\n"
+ + " <property>\n"
+ + " <name>emptyDesc</name>\n"
+ + " <value></value>\n"
+ + " <description></description>\n"
+ + " </property>\n"
+ + " <property>\n"
+ + " <name>noDesc</name>\n"
+ + " <value></value>\n"
+ + " </property>\n"
+ + "</configuration>";
+
+ @Test
+ public void testParse() throws IOException {
+
+ InputStream config_1 = new ByteArrayInputStream(config_1_str.getBytes());
+ DefaultConfig config = new DefaultConfigParser().parse(config_1);
+ Assert.assertNotNull(config);
+ Assert.assertNotNull(config.getPropertyInfos());
+ Assert.assertEquals(6, config.getPropertyInfos().size());
+ for (PropertyInfo pInfo : config.getPropertyInfos()) {
+ if (pInfo.getName().equals("security.client.protocol.acl")) {
+ Assert.assertEquals("*", pInfo.getValue());
+ Assert.assertTrue(pInfo.getDescription().startsWith("ACL for HRegionInterface "));
+ }
+ if (pInfo.getName().equals("emptyVal")) {
+ Assert.assertEquals("", pInfo.getValue());
+ Assert.assertEquals("non-empty-desc", pInfo.getDescription());
+ }
+ if (pInfo.getName().equals("emptyDesc")) {
+ Assert.assertEquals("", pInfo.getValue());
+ Assert.assertEquals("", pInfo.getDescription());
+ }
+ if (pInfo.getName().equals("noDesc")) {
+ Assert.assertEquals("", pInfo.getValue());
+ Assert.assertNull(pInfo.getDescription());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java
new file mode 100644
index 0000000..ba1912a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.providers.agent.AgentProviderService;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+
+import static org.mockito.Mockito.doReturn;
+
+/**
+ *
+ */
+public class TestMetainfoParser {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestMetainfoParser.class);
+ public static final String METAINFO_XML =
+ "/org/apache/slider/providers/agent/application/metadata/metainfo.xml";
+
+ @Test
+ public void testParse() throws IOException {
+
+ InputStream resStream = this.getClass().getResourceAsStream(
+ METAINFO_XML);
+ MetainfoParser parser = new MetainfoParser();
+ Metainfo metainfo = parser.fromXmlStream(resStream);
+ Assert.assertNotNull(metainfo);
+ Assert.assertNotNull(metainfo.getApplication());
+ Application application = metainfo.getApplication();
+ assert "STORM".equals(application.getName());
+ assert 6 == application.getComponents().size();
+ OSPackage pkg = application.getOSSpecifics().get(0).getPackages().get(0);
+ assert "tarball".equals(pkg.getType());
+ assert "files/apache-storm-0.9.1.2.1.1.0-237.tar.gz".equals(pkg.getName());
+ boolean found = false;
+ for (Component comp : application.getComponents()) {
+ if (comp != null && comp.getName().equals("NIMBUS")) {
+ found = true;
+ Assert.assertEquals(0, comp.getComponentExports().size());
+ }
+ if (comp != null && comp.getName().equals("SUPERVISOR")) {
+ Assert.assertEquals(1, comp.getComponentExports().size());
+ }
+ if (comp != null && comp.getName().equals("ANOTHER_COMPONENT")) {
+ assert 2 == comp.getCommands().size();
+ assert "start command".equals(comp.getCommands().get(0).getExec());
+ assert "START".equals(comp.getCommands().get(0).getName());
+ assert "stop command".equals(comp.getCommands().get(1).getExec());
+ assert "STOP".equals(comp.getCommands().get(1).getName());
+ }
+ }
+ assert found;
+ Assert.assertEquals(0, application.getConfigFiles().size());
+ assert 1 == application.getPackages().size();
+ Package p = application.getPackages().get(0);
+ assert "tarball".equals(p.getType());
+ assert "test-tarball-name.tgz".equals(p.getName());
+ }
+
+ @Test
+ public void testJsonParse() throws IOException {
+ String metaInfo1_json = "{\n"
+ + "\"schemaVersion\":\"2.2\",\n"
+ + "\"application\":{\n"
+ + "\"name\": \"MEMCACHED\","
+ + "\"exportGroups\": ["
+ + "{"
+ + "\"name\": \"Servers\","
+ + "\"exports\": ["
+ + "{"
+ + "\"name\": \"host_port\","
+ + "\"value\": \"${MEMCACHED_HOST}:${site.global.port}\""
+ + "}"
+ + "]"
+ + "}"
+ + "],"
+ + "\"components\": ["
+ + "{"
+ + "\"name\": \"MEMCACHED\","
+ + "\"compExports\": \"Servers-host_port\","
+ + "\"commands\": ["
+ + "{"
+ + "\"exec\": \"java -classpath /usr/myapps/memcached/*:/usr/lib/hadoop/lib/* com.thimbleware.jmemcached.Main\""
+ + "}"
+ + "]"
+ + "},"
+ + "{"
+ + "\"name\": \"MEMCACHED2\","
+ + "\"commands\": ["
+ + "{"
+ + "\"exec\": \"scripts/config.py\","
+ + "\"type\": \"PYTHON\","
+ + "\"name\": \"CONFIGURE\""
+ + "}"
+ + "],"
+ + "\"dockerContainers\": ["
+ + "{"
+ + "\"name\": \"redis\","
+ + "\"image\": \"dockerhub/redis\","
+ + "\"options\": \"-net=bridge\","
+ + "\"mounts\": ["
+ + "{"
+ + "\"containerMount\": \"/tmp/conf\","
+ + "\"hostMount\": \"{$conf:@//site/global/app_root}/conf\""
+ + "}"
+ + "]"
+ + "}"
+ + "]"
+ + "}"
+ + "]"
+ + "}"
+ + "}";
+
+ MetainfoParser parser = new MetainfoParser();
+ Metainfo mInfo = parser.fromJsonString(metaInfo1_json);
+ Assert.assertEquals("2.2", mInfo.getSchemaVersion());
+
+ Application app = mInfo.getApplication();
+ Assert.assertNotNull(app);
+
+ Assert.assertEquals("MEMCACHED", app.getName());
+ List<ExportGroup> egs = app.getExportGroups();
+ Assert.assertEquals(1, egs.size());
+ ExportGroup eg = egs.get(0);
+ Assert.assertEquals("Servers", eg.getName());
+ List<Export> exports = eg.getExports();
+ Assert.assertEquals(1, exports.size());
+ Export export = exports.get(0);
+ Assert.assertEquals("host_port", export.getName());
+ Assert.assertEquals("${MEMCACHED_HOST}:${site.global.port}", export.getValue());
+
+ List<Component> components = app.getComponents();
+ Assert.assertEquals(2, components.size());
+
+ Component c1 = mInfo.getApplicationComponent("MEMCACHED");
+ Assert.assertNotNull(c1);
+ Assert.assertEquals("MEMCACHED", c1.getName());
+ Assert.assertEquals("Servers-host_port", c1.getCompExports());
+ Assert.assertEquals(1, c1.getCommands().size());
+ ComponentCommand cmd = c1.getCommands().get(0);
+ Assert.assertEquals("START", cmd.getName());
+ Assert.assertEquals("SHELL", cmd.getType());
+ Assert.assertEquals("java -classpath /usr/myapps/memcached/*:/usr/lib/hadoop/lib/* com.thimbleware.jmemcached.Main",
+ cmd.getExec());
+
+ Component c2 = mInfo.getApplicationComponent("MEMCACHED2");
+ Assert.assertNotNull(c2);
+ Assert.assertEquals("MEMCACHED2", c2.getName());
+ Assert.assertEquals(1, c2.getCommands().size());
+ cmd = c2.getCommands().get(0);
+ Assert.assertEquals("CONFIGURE", cmd.getName());
+ Assert.assertEquals("PYTHON", cmd.getType());
+ Assert.assertEquals("scripts/config.py", cmd.getExec());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java
new file mode 100644
index 0000000..a1986cd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/TestServiceRecordAttributes.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster;
+
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.core.conf.MapOperations;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ *
+ */
+public class TestServiceRecordAttributes extends Assert {
+
+ @Test
+ public void testAppConfigProvidedServiceRecordAttributes() throws Exception {
+ Map<String, String> options = new HashMap<>();
+ options.put("slider.some.arbitrary.option", "arbitrary value");
+ options.put("service.record.attribute.one_attribute", "one_attribute_value");
+ options.put("service.record.attribute.second_attribute", "second_attribute_value");
+ MapOperations serviceProps = new MapOperations(SliderKeys.COMPONENT_AM, options);
+ options = new HashMap<>();
+ options.put("some.component.attribute", "component_attribute_value");
+ options.put("service.record.attribute.component_attribute", "component_attribute_value");
+ MapOperations compProps = new MapOperations("TEST_COMP", options);
+
+ SliderAppMaster appMaster = new SliderAppMaster();
+
+ ServiceRecord appServiceRecord = new ServiceRecord();
+
+ appMaster.setProvidedServiceRecordAttributes(serviceProps, appServiceRecord);
+
+ assertNull("property should not be attribute",
+ appServiceRecord.get("slider.some.arbitrary.option"));
+ assertEquals("wrong value", "one_attribute_value",
+ appServiceRecord.get("one_attribute"));
+ assertEquals("wrong value", "second_attribute_value",
+ appServiceRecord.get("second_attribute"));
+
+ ServiceRecord compServiceRecord = new ServiceRecord();
+
+ appMaster.setProvidedServiceRecordAttributes(compProps, compServiceRecord);
+
+ assertNull("should not be attribute",
+ compServiceRecord.get("some.component.attribute"));
+ assertEquals("wrong value", "component_attribute_value",
+ compServiceRecord.get("component_attribute"));
+
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/51] [abbrv] hadoop git commit: YARN-5218. Initial core change for
DNS for YARN. Contributed by Jonathan Maron
Posted by ji...@apache.org.
YARN-5218. Initial core change for DNS for YARN. Contributed by Jonathan Maron
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3031e921
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3031e921
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3031e921
Branch: refs/heads/yarn-native-services
Commit: 3031e9213dea9bd6c1fa42e9ea18538a3bfe4b1b
Parents: 47a16db
Author: Jian He <ji...@apache.org>
Authored: Sun Jun 12 11:32:03 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
hadoop-project/pom.xml | 8 +
.../dev-support/findbugs-exclude.xml | 15 +
.../hadoop-yarn/hadoop-yarn-registry/pom.xml | 5 +
.../registry/client/api/DNSOperations.java | 60 +
.../client/api/DNSOperationsFactory.java | 78 +
.../registry/client/api/RegistryConstants.java | 111 +-
.../registry/client/impl/zk/CuratorService.java | 266 ++-
.../registry/client/impl/zk/ListenerHandle.java | 25 +
.../registry/client/impl/zk/PathListener.java | 30 +
.../types/yarn/YarnRegistryAttributes.java | 16 +-
.../dns/ApplicationServiceRecordProcessor.java | 353 ++++
.../server/dns/BaseServiceRecordProcessor.java | 469 ++++++
.../dns/ContainerServiceRecordProcessor.java | 278 ++++
.../server/dns/RecordCreatorFactory.java | 275 ++++
.../hadoop/registry/server/dns/RegistryDNS.java | 1534 ++++++++++++++++++
.../registry/server/dns/RegistryDNSServer.java | 290 ++++
.../registry/server/dns/SecureableZone.java | 151 ++
.../server/dns/ServiceRecordProcessor.java | 53 +
.../registry/server/dns/ZoneSelector.java | 33 +
.../registry/server/dns/package-info.java | 26 +
.../registry/server/dns/TestRegistryDNS.java | 561 +++++++
.../server/dns/TestSecureRegistryDNS.java | 44 +
.../test/resources/0.17.172.in-addr.arpa.zone | 36 +
.../src/test/resources/test.private | 32 +
24 files changed, 4661 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index d46bde0..0f07172 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -84,6 +84,7 @@
<zookeeper.version>3.4.6</zookeeper.version>
<curator.version>2.7.1</curator.version>
<findbugs.version>3.0.0</findbugs.version>
+ <dnsjava.version>2.1.7</dnsjava.version>
<tomcat.version>6.0.48</tomcat.version>
<guice.version>4.0</guice.version>
@@ -1174,6 +1175,13 @@
</exclusion>
</exclusions>
</dependency>
+
+ <dependency>
+ <groupId>dnsjava</groupId>
+ <artifactId>dnsjava</artifactId>
+ <version>${dnsjava.version}</version>
+ </dependency>
+
<dependency>
<groupId>org.skyscreamer</groupId>
<artifactId>jsonassert</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index ab36a4e..1651a2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -583,4 +583,19 @@
<Bug pattern="UL_UNRELEASED_LOCK_EXCEPTION_PATH" />
</Match>
+ <Match>
+ <Class name="org.apache.hadoop.registry.server.dns.RegistryDNS" />
+ <Method name="addNIOTCP" />
+ <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.registry.server.dns.RegistryDNS" />
+ <Method name="addNIOUDP" />
+ <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.registry.server.dns.RegistryDNS" />
+ <Method name="serveNIOTCP" />
+ <Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
+ </Match>
</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
index 811964a..69313c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
@@ -80,6 +80,11 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>dnsjava</groupId>
+ <artifactId>dnsjava</artifactId>
+ </dependency>
+
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/DNSOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/DNSOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/DNSOperations.java
new file mode 100644
index 0000000..3abfb6c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/DNSOperations.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.registry.client.api;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.service.Service;
+
+import java.io.IOException;
+
+/**
+ * DNS Operations.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface DNSOperations extends Service {
+
+ /**
+ * Register a service based on a service record.
+ *
+ * @param path the ZK path.
+ * @param record record providing DNS registration info.
+ * @throws IOException Any other IO Exception.
+ */
+ void register(String path, ServiceRecord record)
+ throws IOException;
+
+
+ /**
+ * Delete a service's registered endpoints.
+ *
+ * If the operation returns without an error then the entry has been
+ * deleted.
+ *
+ * @param path the ZK path.
+ * @param record service record
+ * @throws IOException Any other IO Exception
+ *
+ */
+ void delete(String path, ServiceRecord record)
+ throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/DNSOperationsFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/DNSOperationsFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/DNSOperationsFactory.java
new file mode 100644
index 0000000..1a8bb3e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/DNSOperationsFactory.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.registry.client.api;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.server.dns.RegistryDNS;
+
+/**
+ * A factory for DNS operation service instances.
+ */
+public final class DNSOperationsFactory implements RegistryConstants {
+
+ /**
+ * DNS Implementation type.
+ */
+ public enum DNSImplementation {
+ DNSJAVA
+ }
+
+ private DNSOperationsFactory() {
+ }
+
+ /**
+ * Create and initialize a DNS operations instance.
+ *
+ * @param conf configuration
+ * @return a DNS operations instance
+ */
+ public static DNSOperations createInstance(Configuration conf) {
+ return createInstance("DNSOperations", DNSImplementation.DNSJAVA, conf);
+ }
+
+ /**
+ * Create and initialize a registry operations instance.
+ * Access rights will be determined from the configuration.
+ *
+ * @param name name of the instance
+ * @param impl the DNS implementation.
+ * @param conf configuration
+ * @return a registry operations instance
+ */
+ public static DNSOperations createInstance(String name,
+ DNSImplementation impl,
+ Configuration conf) {
+ Preconditions.checkArgument(conf != null, "Null configuration");
+ DNSOperations operations = null;
+ switch (impl) {
+ case DNSJAVA:
+ operations = new RegistryDNS(name);
+ break;
+
+ default:
+ throw new IllegalArgumentException(
+ String.format("%s is not available", impl.toString()));
+ }
+
+ //operations.init(conf);
+ return operations;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
index a6fe216..7115a4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
@@ -44,17 +44,106 @@ public interface RegistryConstants {
String ZK_PREFIX = REGISTRY_PREFIX + "zk.";
/**
+ * Prefix for dns-specific options: {@value}
+ * <p>
+ * For clients using other protocols, these options are not supported.
+ */
+ String DNS_PREFIX = REGISTRY_PREFIX + "dns.";
+
+ /**
* flag to indicate whether or not the registry should
- * be enabled in the RM: {@value}
+ * be enabled in the RM: {@value}.
*/
String KEY_REGISTRY_ENABLED = REGISTRY_PREFIX + "rm.enabled";
/**
- * Defaut value for enabling the registry in the RM: {@value}
+ * Defaut value for enabling the registry in the RM: {@value}.
*/
boolean DEFAULT_REGISTRY_ENABLED = false;
/**
+ * flag to indicate whether or not the registry should
+ * be enabled in the RM: {@value}.
+ */
+ String KEY_DNS_ENABLED = DNS_PREFIX + "enabled";
+
+ /**
+ * Defaut value for enabling the DNS in the Registry: {@value}.
+ */
+ boolean DEFAULT_DNS_ENABLED = false;
+
+ /**
+ * DNS domain name key.
+ */
+ String KEY_DNS_DOMAIN = DNS_PREFIX + "domain-name";
+
+ /**
+ * DNS bind address.
+ */
+ String KEY_DNS_BIND_ADDRESS = DNS_PREFIX + "bind-address";
+
+ /**
+ * DNS port number key.
+ */
+ String KEY_DNS_PORT = DNS_PREFIX + "bind-port";
+
+ /**
+ * Default DNS port number.
+ */
+ int DEFAULT_DNS_PORT = 53;
+
+ /**
+ * DNSSEC Enabled?
+ */
+ String KEY_DNSSEC_ENABLED = DNS_PREFIX + "dnssec.enabled";
+
+ /**
+ * DNSSEC Enabled?
+ */
+ String KEY_DNSSEC_PUBLIC_KEY = DNS_PREFIX + "public-key";
+
+ /**
+ * DNSSEC private key file.
+ */
+ String KEY_DNSSEC_PRIVATE_KEY_FILE = DNS_PREFIX + "private-key-file";
+
+ /**
+ * Default DNSSEC private key file path.
+ */
+ String DEFAULT_DNSSEC_PRIVATE_KEY_FILE =
+ "/etc/hadoop/conf/registryDNS.private";
+
+ /**
+ * Zone subnet.
+ */
+ String KEY_DNS_ZONE_SUBNET = DNS_PREFIX + "zone-subnet";
+
+ /**
+ * Zone subnet mask.
+ */
+ String KEY_DNS_ZONE_MASK = DNS_PREFIX + "zone-mask";
+
+ /**
+ * Zone subnet IP min.
+ */
+ String KEY_DNS_ZONE_IP_MIN = DNS_PREFIX + "zone-ip-min";
+
+ /**
+ * Zone subnet IP max.
+ */
+ String KEY_DNS_ZONE_IP_MAX = DNS_PREFIX + "zone-ip-max";
+
+ /**
+ * DNS Record TTL.
+ */
+ String KEY_DNS_TTL = DNS_PREFIX + "dns-ttl";
+
+ /**
+ * DNS Record TTL.
+ */
+ String KEY_DNS_ZONES_DIR = DNS_PREFIX + "zones-dir";
+
+ /**
* Key to set if the registry is secure: {@value}.
* Turning it on changes the permissions policy from "open access"
* to restrictions on kerberos with the option of
@@ -69,12 +158,12 @@ public interface RegistryConstants {
boolean DEFAULT_REGISTRY_SECURE = false;
/**
- * Root path in the ZK tree for the registry: {@value}
+ * Root path in the ZK tree for the registry: {@value}.
*/
String KEY_REGISTRY_ZK_ROOT = ZK_PREFIX + "root";
/**
- * Default root of the yarn registry: {@value}
+ * Default root of the yarn registry: {@value}.
*/
String DEFAULT_ZK_REGISTRY_ROOT = "/registry";
@@ -92,7 +181,7 @@ public interface RegistryConstants {
/**
* Registry client uses Kerberos: authentication is automatic from
- * logged in user
+ * logged in user.
*/
String REGISTRY_CLIENT_AUTH_KERBEROS = "kerberos";
@@ -104,12 +193,12 @@ public interface RegistryConstants {
String REGISTRY_CLIENT_AUTH_DIGEST = "digest";
/**
- * No authentication; client is anonymous
+ * No authentication; client is anonymous.
*/
String REGISTRY_CLIENT_AUTH_ANONYMOUS = "";
/**
- * Registry client authentication ID
+ * Registry client authentication ID.
* <p>
* This is only used in secure clusters with
* {@link #KEY_REGISTRY_CLIENT_AUTH} set to
@@ -134,17 +223,17 @@ public interface RegistryConstants {
/**
* List of hostname:port pairs defining the
- * zookeeper quorum binding for the registry {@value}
+ * zookeeper quorum binding for the registry {@value}.
*/
String KEY_REGISTRY_ZK_QUORUM = ZK_PREFIX + "quorum";
/**
- * The default zookeeper quorum binding for the registry: {@value}
+ * The default zookeeper quorum binding for the registry: {@value}.
*/
String DEFAULT_REGISTRY_ZK_QUORUM = "localhost:2181";
/**
- * Zookeeper session timeout in milliseconds: {@value}
+ * Zookeeper session timeout in milliseconds: {@value}.
*/
String KEY_REGISTRY_ZK_SESSION_TIMEOUT =
ZK_PREFIX + "session.timeout.ms";
@@ -259,7 +348,7 @@ public interface RegistryConstants {
String KEY_REGISTRY_CLIENT_JAAS_CONTEXT = REGISTRY_PREFIX + "jaas.context";
/**
- * default client-side registry JAAS context: {@value}
+ * default client-side registry JAAS context: {@value}.
*/
String DEFAULT_REGISTRY_CLIENT_JAAS_CONTEXT = "Client";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
index 7f35c3f..ad008c4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
@@ -28,6 +28,9 @@ import org.apache.curator.framework.api.BackgroundCallback;
import org.apache.curator.framework.api.CreateBuilder;
import org.apache.curator.framework.api.DeleteBuilder;
import org.apache.curator.framework.api.GetChildrenBuilder;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
import org.apache.curator.retry.BoundedExponentialBackoffRetry;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -36,14 +39,14 @@ import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.service.ServiceStateException;
import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.registry.client.exceptions.AuthenticationFailedException;
import org.apache.hadoop.registry.client.exceptions.NoChildrenForEphemeralsException;
import org.apache.hadoop.registry.client.exceptions.NoPathPermissionsException;
import org.apache.hadoop.registry.client.exceptions.RegistryIOException;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.service.ServiceStateException;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs;
@@ -69,12 +72,12 @@ public class CuratorService extends CompositeService
LoggerFactory.getLogger(CuratorService.class);
/**
- * the Curator binding
+ * the Curator binding.
*/
private CuratorFramework curator;
/**
- * Path to the registry root
+ * Path to the registry root.
*/
private String registryRoot;
@@ -85,17 +88,17 @@ public class CuratorService extends CompositeService
private final RegistryBindingSource bindingSource;
/**
- * Security service
+ * Security service.
*/
private RegistrySecurity registrySecurity;
/**
- * the connection binding text for messages
+ * the connection binding text for messages.
*/
private String connectionDescription;
/**
- * Security connection diagnostics
+ * Security connection diagnostics.
*/
private String securityConnectionDiagnostics = "";
@@ -106,10 +109,16 @@ public class CuratorService extends CompositeService
private EnsembleProvider ensembleProvider;
/**
+ * Registry tree cache.
+ */
+ private TreeCache treeCache;
+
+ /**
* Construct the service.
- * @param name service name
+ *
+ * @param name service name
* @param bindingSource source of binding information.
- * If null: use this instance
+ * If null: use this instance
*/
public CuratorService(String name, RegistryBindingSource bindingSource) {
super(name);
@@ -122,7 +131,8 @@ public class CuratorService extends CompositeService
/**
* Create an instance using this service as the binding source (i.e. read
- * configuration options from the registry)
+ * configuration options from the registry).
+ *
* @param name service name
*/
public CuratorService(String name) {
@@ -131,7 +141,8 @@ public class CuratorService extends CompositeService
/**
* Init the service.
- * This is where the security bindings are set up
+ * This is where the security bindings are set up.
+ *
* @param conf configuration of the service
* @throws Exception
*/
@@ -155,6 +166,7 @@ public class CuratorService extends CompositeService
/**
* Start the service.
* This is where the curator instance is started.
+ *
* @throws Exception
*/
@Override
@@ -167,29 +179,35 @@ public class CuratorService extends CompositeService
}
/**
- * Close the ZK connection if it is open
+ * Close the ZK connection if it is open.
*/
@Override
protected void serviceStop() throws Exception {
IOUtils.closeStream(curator);
+
+ if (treeCache != null) {
+ treeCache.close();
+ }
super.serviceStop();
}
/**
- * Internal check that a service is in the live state
+ * Internal check that a service is in the live state.
+ *
* @throws ServiceStateException if not
*/
private void checkServiceLive() throws ServiceStateException {
if (!isInState(STATE.STARTED)) {
throw new ServiceStateException(
"Service " + getName() + " is in wrong state: "
- + getServiceState());
+ + getServiceState());
}
}
/**
* Flag to indicate whether or not the registry is secure.
* Valid once the service is inited.
+ *
* @return service security policy
*/
public boolean isSecure() {
@@ -197,7 +215,8 @@ public class CuratorService extends CompositeService
}
/**
- * Get the registry security helper
+ * Get the registry security helper.
+ *
* @return the registry security helper
*/
protected RegistrySecurity getRegistrySecurity() {
@@ -205,7 +224,8 @@ public class CuratorService extends CompositeService
}
/**
- * Build the security diagnostics string
+ * Build the security diagnostics string.
+ *
* @return a string for diagnostics
*/
protected String buildSecurityDiagnostics() {
@@ -224,6 +244,7 @@ public class CuratorService extends CompositeService
* Create a new curator instance off the root path; using configuration
* options provided in the service configuration to set timeouts and
* retry policy.
+ *
* @return the newly created creator
*/
private CuratorFramework createCurator() throws IOException {
@@ -250,14 +271,15 @@ public class CuratorService extends CompositeService
// set the security options
// build up the curator itself
- CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder();
+ CuratorFrameworkFactory.Builder builder =
+ CuratorFrameworkFactory.builder();
builder.ensembleProvider(ensembleProvider)
- .connectionTimeoutMs(connectionTimeout)
- .sessionTimeoutMs(sessionTimeout)
+ .connectionTimeoutMs(connectionTimeout)
+ .sessionTimeoutMs(sessionTimeout)
- .retryPolicy(new BoundedExponentialBackoffRetry(retryInterval,
- retryCeiling,
- retryTimes));
+ .retryPolicy(new BoundedExponentialBackoffRetry(retryInterval,
+ retryCeiling,
+ retryTimes));
// set up the builder AND any JVM context
registrySecurity.applySecurityEnvironment(builder);
@@ -273,21 +295,23 @@ public class CuratorService extends CompositeService
@Override
public String toString() {
return super.toString()
- + " " + bindingDiagnosticDetails();
+ + " " + bindingDiagnosticDetails();
}
/**
- * Get the binding diagnostics
+ * Get the binding diagnostics.
+ *
* @return a diagnostics string valid after the service is started.
*/
public String bindingDiagnosticDetails() {
return " Connection=\"" + connectionDescription + "\""
- + " root=\"" + registryRoot + "\""
- + " " + securityConnectionDiagnostics;
+ + " root=\"" + registryRoot + "\""
+ + " " + securityConnectionDiagnostics;
}
/**
- * Create a full path from the registry root and the supplied subdir
+ * Create a full path from the registry root and the supplied subdir.
+ *
* @param path path of operation
* @return an absolute path
* @throws IllegalArgumentException if the path is invalide
@@ -299,6 +323,7 @@ public class CuratorService extends CompositeService
/**
* Get the registry binding source ... this can be used to
* create new ensemble providers
+ *
* @return the registry binding source in use
*/
public RegistryBindingSource getBindingSource() {
@@ -308,23 +333,23 @@ public class CuratorService extends CompositeService
/**
* Create the ensemble provider for this registry, by invoking
* {@link RegistryBindingSource#supplyBindingInformation()} on
- * the provider stored in {@link #bindingSource}
+ * the provider stored in {@link #bindingSource}.
* Sets {@link #ensembleProvider} to that value;
* sets {@link #connectionDescription} to the binding info
* for use in toString and logging;
- *
*/
protected void createEnsembleProvider() {
BindingInformation binding = bindingSource.supplyBindingInformation();
connectionDescription = binding.description
- + " " + securityConnectionDiagnostics;
+ + " " + securityConnectionDiagnostics;
ensembleProvider = binding.ensembleProvider;
}
/**
* Supply the binding information.
* This implementation returns a fixed ensemble bonded to
- * the quorum supplied by {@link #buildConnectionString()}
+ * the quorum supplied by {@link #buildConnectionString()}.
+ *
* @return the binding information
*/
@Override
@@ -339,17 +364,19 @@ public class CuratorService extends CompositeService
/**
* Override point: get the connection string used to connect to
- * the ZK service
+ * the ZK service.
+ *
* @return a registry quorum
*/
protected String buildConnectionString() {
return getConfig().getTrimmed(KEY_REGISTRY_ZK_QUORUM,
- DEFAULT_REGISTRY_ZK_QUORUM);
+ DEFAULT_REGISTRY_ZK_QUORUM);
}
/**
- * Create an IOE when an operation fails
- * @param path path of operation
+ * Create an IOE when an operation fails.
+ *
+ * @param path path of operation
* @param operation operation attempted
* @param exception caught the exception caught
* @return an IOE to throw that contains the path and operation details.
@@ -361,8 +388,9 @@ public class CuratorService extends CompositeService
}
/**
- * Create an IOE when an operation fails
- * @param path path of operation
+ * Create an IOE when an operation fails.
+ *
+ * @param path path of operation
* @param operation operation attempted
* @param exception caught the exception caught
* @return an IOE to throw that contains the path and operation details.
@@ -385,9 +413,10 @@ public class CuratorService extends CompositeService
} else if (exception instanceof KeeperException.AuthFailedException) {
ioe = new AuthenticationFailedException(path,
"Authentication Failed: " + exception
- + "; " + securityConnectionDiagnostics,
+ + "; " + securityConnectionDiagnostics,
exception);
- } else if (exception instanceof KeeperException.NoChildrenForEphemeralsException) {
+ } else if (exception instanceof
+ KeeperException.NoChildrenForEphemeralsException) {
ioe = new NoChildrenForEphemeralsException(path,
"Cannot create a path under an ephemeral node: " + exception,
exception);
@@ -402,7 +431,7 @@ public class CuratorService extends CompositeService
} else {
ioe = new RegistryIOException(path,
"Failure of " + operation + " on " + path + ": " +
- exception.toString(),
+ exception.toString(),
exception);
}
if (ioe.getCause() == null) {
@@ -417,8 +446,8 @@ public class CuratorService extends CompositeService
* may create the same path before the create() operation is executed/
* propagated to the ZK node polled.
*
- * @param path path to create
- * @param acl ACL for path -used when creating a new entry
+ * @param path path to create
+ * @param acl ACL for path -used when creating a new entry
* @param createParents flag to trigger parent creation
* @return true iff the path was created
* @throws IOException
@@ -432,10 +461,11 @@ public class CuratorService extends CompositeService
}
/**
- * Stat the file
+ * Stat the file.
+ *
* @param path path of operation
* @return a curator stat entry
- * @throws IOException on a failure
+ * @throws IOException on a failure
* @throws PathNotFoundException if the path was not found
*/
public Stat zkStat(String path) throws IOException {
@@ -457,7 +487,8 @@ public class CuratorService extends CompositeService
}
/**
- * Get the ACLs of a path
+ * Get the ACLs of a path.
+ *
* @param path path of operation
* @return a possibly empty list of ACLs
* @throws IOException
@@ -481,12 +512,13 @@ public class CuratorService extends CompositeService
}
/**
- * Probe for a path existing
+ * Probe for a path existing.
+ *
* @param path path of operation
* @return true if the path was visible from the ZK server
* queried.
* @throws IOException on any exception other than
- * {@link PathNotFoundException}
+ * {@link PathNotFoundException}
*/
public boolean zkPathExists(String path) throws IOException {
checkServiceLive();
@@ -503,7 +535,8 @@ public class CuratorService extends CompositeService
}
/**
- * Verify a path exists
+ * Verify a path exists.
+ *
* @param path path of operation
* @throws PathNotFoundException if the path is absent
* @throws IOException
@@ -514,11 +547,12 @@ public class CuratorService extends CompositeService
}
/**
- * Create a directory. It is not an error if it already exists
- * @param path path to create
- * @param mode mode for path
+ * Create a directory. It is not an error if it already exists.
+ *
+ * @param path path to create
+ * @param mode mode for path
* @param createParents flag to trigger parent creation
- * @param acls ACL for path
+ * @param acls ACL for path
* @throws IOException any problem
*/
public boolean zkMkPath(String path,
@@ -558,9 +592,10 @@ public class CuratorService extends CompositeService
}
/**
- * Recursively make a path
+ * Recursively make a path.
+ *
* @param path path to create
- * @param acl ACL for path
+ * @param acl ACL for path
* @throws IOException any problem
*/
public void zkMkParentPath(String path,
@@ -574,7 +609,8 @@ public class CuratorService extends CompositeService
/**
* Create a path with given data. byte[0] is used for a path
- * without data
+ * without data.
+ *
* @param path path of operation
* @param data initial data
* @param acls
@@ -600,7 +636,8 @@ public class CuratorService extends CompositeService
}
/**
- * Update the data for a path
+ * Update the data for a path.
+ *
* @param path path of operation
* @param data new data
* @throws IOException
@@ -620,13 +657,14 @@ public class CuratorService extends CompositeService
}
/**
- * Create or update an entry
- * @param path path
- * @param data data
- * @param acl ACL for path -used when creating a new entry
+ * Create or update an entry.
+ *
+ * @param path path
+ * @param data data
+ * @param acl ACL for path -used when creating a new entry
* @param overwrite enable overwrite
- * @throws IOException
* @return true if the entry was created, false if it was simply updated.
+ * @throws IOException
*/
public boolean zkSet(String path,
CreateMode mode,
@@ -649,12 +687,13 @@ public class CuratorService extends CompositeService
/**
* Delete a directory/directory tree.
- * It is not an error to delete a path that does not exist
- * @param path path of operation
- * @param recursive flag to trigger recursive deletion
+ * It is not an error to delete a path that does not exist.
+ *
+ * @param path path of operation
+ * @param recursive flag to trigger recursive deletion
* @param backgroundCallback callback; this being set converts the operation
- * into an async/background operation.
- * task
+ * into an async/background operation.
+ * task
* @throws IOException on problems other than no-such-path
*/
public void zkDelete(String path,
@@ -682,7 +721,8 @@ public class CuratorService extends CompositeService
}
/**
- * List all children of a path
+ * List all children of a path.
+ *
* @param path path of operation
* @return a possibly empty list of children
* @throws IOException
@@ -703,7 +743,8 @@ public class CuratorService extends CompositeService
}
/**
- * Read data on a path
+ * Read data on a path.
+ *
* @param path path of operation
* @return the data
* @throws IOException read failure
@@ -724,9 +765,10 @@ public class CuratorService extends CompositeService
/**
* Return a path dumper instance which can do a full dump
* of the registry tree in its <code>toString()</code>
- * operation
- * @return a class to dump the registry
+ * operation.
+ *
* @param verbose verbose flag - includes more details (such as ACLs)
+ * @return a class to dump the registry
*/
public ZKPathDumper dumpPath(boolean verbose) {
return new ZKPathDumper(curator, registryRoot, verbose);
@@ -734,7 +776,8 @@ public class CuratorService extends CompositeService
/**
* Add a new write access entry for all future write operations.
- * @param id ID to use
+ *
+ * @param id ID to use
* @param pass password
* @throws IOException on any failure to build the digest
*/
@@ -746,16 +789,16 @@ public class CuratorService extends CompositeService
}
/**
- * Clear all write accessors
+ * Clear all write accessors.
*/
public void clearWriteAccessors() {
getRegistrySecurity().resetDigestACLs();
}
-
/**
* Diagnostics method to dump a registry robustly.
- * Any exception raised is swallowed
+ * Any exception raised is swallowed.
+ *
* @param verbose verbose path dump
* @return the registry tree
*/
@@ -769,4 +812,79 @@ public class CuratorService extends CompositeService
}
return "";
}
+
+ /**
+ * Registers a listener to path related events.
+ *
+ * @param listener the listener.
+ * @return a handle allowing for the management of the listener.
+ * @throws Exception if registration fails due to error.
+ */
+ public ListenerHandle registerPathListener(final PathListener listener)
+ throws Exception {
+
+ final TreeCacheListener pathChildrenCacheListener =
+ new TreeCacheListener() {
+
+ public void childEvent(CuratorFramework curatorFramework,
+ TreeCacheEvent event)
+ throws Exception {
+ String path = null;
+ if (event != null && event.getData() != null) {
+ path = event.getData().getPath();
+ }
+ assert event != null;
+ switch (event.getType()) {
+ case NODE_ADDED:
+ LOG.info("Informing listener of added node {}", path);
+ listener.nodeAdded(path);
+
+ break;
+
+ case NODE_REMOVED:
+ LOG.info("Informing listener of removed node {}", path);
+ listener.nodeRemoved(path);
+
+ break;
+
+ case NODE_UPDATED:
+ LOG.info("Informing listener of updated node {}", path);
+ listener.nodeAdded(path);
+
+ break;
+
+ default:
+ // do nothing
+ break;
+
+ }
+ }
+ };
+ treeCache.getListenable().addListener(pathChildrenCacheListener);
+
+ return new ListenerHandle() {
+ @Override
+ public void remove() {
+ treeCache.getListenable().removeListener(pathChildrenCacheListener);
+ }
+ };
+
+ }
+
+ // TODO: should caches be stopped and then restarted if need be?
+
+ /**
+ * Create the tree cache that monitors the registry for node addition, update,
+ * and deletion.
+ *
+ * @throws Exception if any issue arises during monitoring.
+ */
+ public void monitorRegistryEntries()
+ throws Exception {
+ String registryPath =
+ getConfig().get(RegistryConstants.KEY_REGISTRY_ZK_ROOT,
+ RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
+ treeCache = new TreeCache(curator, registryPath);
+ treeCache.start();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/ListenerHandle.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/ListenerHandle.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/ListenerHandle.java
new file mode 100644
index 0000000..e43dbbe
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/ListenerHandle.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.registry.client.impl.zk;
+
+/**
+ *
+ */
+public interface ListenerHandle {
+ void remove();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/PathListener.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/PathListener.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/PathListener.java
new file mode 100644
index 0000000..db1e509
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/PathListener.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.registry.client.impl.zk;
+
+import java.io.IOException;
+
+/**
+ *
+ */
+public interface PathListener {
+
+ void nodeAdded(String path) throws IOException;
+
+ void nodeRemoved(String path) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/yarn/YarnRegistryAttributes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/yarn/YarnRegistryAttributes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/yarn/YarnRegistryAttributes.java
index 7b78932..5eaa9c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/yarn/YarnRegistryAttributes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/types/yarn/YarnRegistryAttributes.java
@@ -19,13 +19,23 @@
package org.apache.hadoop.registry.client.types.yarn;
/**
- * YARN specific attributes in the registry
+ * YARN specific attributes in the registry.
*/
-public class YarnRegistryAttributes {
+public final class YarnRegistryAttributes {
/**
- * ID. For containers: container ID. For application instances, application ID.
+ * Hidden constructor.
+ */
+ private YarnRegistryAttributes() {
+ }
+
+ /**
+ * ID. For containers: container ID. For application instances,
+ * application ID.
*/
public static final String YARN_ID = "yarn:id";
public static final String YARN_PERSISTENCE = "yarn:persistence";
+ public static final String YARN_PATH = "yarn:path";
+ public static final String YARN_HOSTNAME = "yarn:hostname";
+ public static final String YARN_IP = "yarn:ip";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java
new file mode 100644
index 0000000..e6a1b5b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ApplicationServiceRecordProcessor.java
@@ -0,0 +1,353 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.registry.server.dns;
+
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.xbill.DNS.Name;
+import org.xbill.DNS.Type;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A processor for generating application DNS records from registry service
+ * records.
+ */
+public class ApplicationServiceRecordProcessor extends
+ BaseServiceRecordProcessor {
+
+ /**
+ * Create an application service record processor.
+ *
+ * @param record the service record
+ * @param path the service record registry node path
+ * @param domain the DNS zone/domain name
+ * @param zoneSelector returns the zone associated with the provided name.
+ * @throws Exception if an issue is generated during instantiation.
+ */
+ public ApplicationServiceRecordProcessor(
+ ServiceRecord record, String path, String domain,
+ ZoneSelector zoneSelector) throws Exception {
+ super(record, path, domain, zoneSelector);
+ }
+
+ /**
+ * Initializes the DNS record type to descriptor mapping based on the
+ * provided service record.
+ *
+ * @param serviceRecord the registry service record.
+ * @throws Exception if an issue is encountered.
+ */
+ @Override public void initTypeToInfoMapping(ServiceRecord serviceRecord)
+ throws Exception {
+ for (int type : getRecordTypes()) {
+ switch (type) {
+ case Type.A:
+ createAInfo(serviceRecord);
+ break;
+ case Type.AAAA:
+ createAAAAInfo(serviceRecord);
+ break;
+ case Type.TXT:
+ createTXTInfo(serviceRecord);
+ break;
+ case Type.CNAME:
+ createCNAMEInfo(serviceRecord);
+ break;
+ case Type.SRV:
+ createSRVInfo(serviceRecord);
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown type " + type);
+
+ }
+ }
+ }
+
+ /**
+ * Create an application TXT record descriptor.
+ *
+ * @param serviceRecord the service record.
+ * @throws Exception if there is an issue during descriptor creation.
+ */
+ protected void createTXTInfo(ServiceRecord serviceRecord) throws Exception {
+ List<Endpoint> endpoints = serviceRecord.external;
+ List<RecordDescriptor> recordDescriptors = new ArrayList<>();
+ TXTApplicationRecordDescriptor txtInfo;
+ for (Endpoint endpoint : endpoints) {
+ txtInfo = new TXTApplicationRecordDescriptor(
+ serviceRecord, endpoint);
+ recordDescriptors.add(txtInfo);
+ }
+ registerRecordDescriptor(Type.TXT, recordDescriptors);
+ }
+
+ /**
+ * Create an application SRV record descriptor.
+ *
+ * @param serviceRecord the service record.
+ * @throws Exception if there is an issue during descriptor creation.
+ */
+ protected void createSRVInfo(ServiceRecord serviceRecord) throws Exception {
+ List<Endpoint> endpoints = serviceRecord.external;
+ List<RecordDescriptor> recordDescriptors = new ArrayList<>();
+ SRVApplicationRecordDescriptor srvInfo;
+ for (Endpoint endpoint : endpoints) {
+ srvInfo = new SRVApplicationRecordDescriptor(
+ serviceRecord, endpoint);
+ recordDescriptors.add(srvInfo);
+ }
+ registerRecordDescriptor(Type.SRV, recordDescriptors);
+ }
+
+ /**
+ * Create an application CNAME record descriptor.
+ *
+ * @param serviceRecord the service record.
+ * @throws Exception if there is an issue during descriptor creation.
+ */
+ protected void createCNAMEInfo(ServiceRecord serviceRecord) throws Exception {
+ List<Endpoint> endpoints = serviceRecord.external;
+ List<RecordDescriptor> recordDescriptors = new ArrayList<>();
+ CNAMEApplicationRecordDescriptor cnameInfo;
+ for (Endpoint endpoint : endpoints) {
+ cnameInfo = new CNAMEApplicationRecordDescriptor(
+ serviceRecord, endpoint);
+ recordDescriptors.add(cnameInfo);
+ }
+ registerRecordDescriptor(Type.CNAME, recordDescriptors);
+ }
+
+ /**
+ * Create an application AAAA record descriptor.
+ *
+ * @param record the service record.
+ * @throws Exception if there is an issue during descriptor creation.
+ */
+ protected void createAAAAInfo(ServiceRecord record)
+ throws Exception {
+ AAAAApplicationRecordDescriptor
+ recordInfo = new AAAAApplicationRecordDescriptor(
+ getPath(), record);
+ registerRecordDescriptor(Type.AAAA, recordInfo);
+ }
+
+ /**
+ * Create an application A record descriptor.
+ *
+ * @param record the service record.
+ * @throws Exception if there is an issue during descriptor creation.
+ */
+ protected void createAInfo(ServiceRecord record) throws Exception {
+ AApplicationRecordDescriptor recordInfo = new AApplicationRecordDescriptor(
+ getPath(), record);
+ registerRecordDescriptor(Type.A, recordInfo);
+ }
+
+ /**
+ * Returns the record types associated with a container service record.
+ *
+ * @return the record type array
+ */
+ @Override public int[] getRecordTypes() {
+ return new int[] {Type.A, Type.AAAA, Type.CNAME, Type.SRV, Type.TXT};
+ }
+
+ /**
+ * An application TXT record descriptor.
+ */
+ class TXTApplicationRecordDescriptor
+ extends ApplicationRecordDescriptor<List<String>> {
+
+ /**
+ * Creates an application TXT record descriptor.
+ *
+ * @param record service record
+ * @throws Exception
+ */
+ public TXTApplicationRecordDescriptor(ServiceRecord record,
+ Endpoint endpoint) throws Exception {
+ super(record, endpoint);
+ }
+
+ /**
+ * Initializes the descriptor parameters.
+ *
+ * @param serviceRecord the service record.
+ */
+ @Override protected void init(ServiceRecord serviceRecord)
+ throws Exception {
+ if (getEndpoint() != null) {
+ this.setNames(new Name[] {getServiceName(), getEndpointName()});
+ this.setTarget(getTextRecords(getEndpoint()));
+ }
+ }
+
+ }
+
+ /**
+ * An application SRV record descriptor.
+ */
+ class SRVApplicationRecordDescriptor extends
+ ApplicationRecordDescriptor<RecordCreatorFactory.HostPortInfo> {
+
+ /**
+ * Creates an application SRV record descriptor.
+ *
+ * @param record service record
+ * @throws Exception
+ */
+ public SRVApplicationRecordDescriptor(ServiceRecord record,
+ Endpoint endpoint) throws Exception {
+ super(record, endpoint);
+ }
+
+ /**
+ * Initializes the descriptor parameters.
+ *
+ * @param serviceRecord the service record.
+ */
+ @Override protected void init(ServiceRecord serviceRecord)
+ throws Exception {
+ if (getEndpoint() != null) {
+ this.setNames(new Name[] {getServiceName(), getEndpointName()});
+ this.setTarget(new RecordCreatorFactory.HostPortInfo(
+ Name.fromString(getHost(getEndpoint()) + "."), getPort(
+ getEndpoint())));
+ }
+ }
+
+ }
+
+ /**
+ * An application CNAME record descriptor.
+ */
+ class CNAMEApplicationRecordDescriptor extends
+ ApplicationRecordDescriptor<Name> {
+
+ /**
+ * Creates an application CNAME record descriptor.
+ *
+ * @param path registry path for service record
+ * @param record service record
+ * @throws Exception
+ */
+ public CNAMEApplicationRecordDescriptor(String path,
+ ServiceRecord record) throws Exception {
+ super(record);
+ }
+
+ /**
+ * Creates an application CNAME record descriptor. This descriptor is the
+ * source for API related CNAME records.
+ *
+ * @param record service record
+ * @param endpoint the API endpoint
+ * @throws Exception
+ */
+ public CNAMEApplicationRecordDescriptor(ServiceRecord record,
+ Endpoint endpoint) throws Exception {
+ super(record, endpoint);
+ }
+
+ /**
+ * Initializes the descriptor parameters.
+ *
+ * @param serviceRecord the service record.
+ */
+ @Override protected void init(ServiceRecord serviceRecord)
+ throws Exception {
+ if (getEndpoint() != null) {
+ this.setNames(new Name[] {getEndpointName()});
+ this.setTarget(getServiceName());
+ }
+ }
+
+ }
+
+ /**
+ * An application A record descriptor.
+ */
+ class AApplicationRecordDescriptor
+ extends ApplicationRecordDescriptor<InetAddress> {
+
+ /**
+ * Creates an application A record descriptor.
+ *
+ * @param path registry path for service record
+ * @param record service record
+ * @throws Exception
+ */
+ public AApplicationRecordDescriptor(String path,
+ ServiceRecord record) throws Exception {
+ super(record);
+ }
+
+ /**
+ * Initializes the descriptor parameters.
+ *
+ * @param serviceRecord the service record.
+ */
+ @Override protected void init(ServiceRecord serviceRecord)
+ throws Exception {
+ this.setNames(new Name[] {getServiceName()});
+ List<Endpoint> endpoints = serviceRecord.external;
+ // TODO: do we need a "hostname" attribute for an application record or
+ // can we rely on the first endpoint record.
+ this.setTarget(InetAddress.getByName(
+ getHost(endpoints.get(0))));
+ }
+
+ }
+
+ /**
+ * An application AAAA record descriptor.
+ */
+ class AAAAApplicationRecordDescriptor extends AApplicationRecordDescriptor {
+
+ /**
+ * Creates an application AAAA record descriptor.
+ *
+ * @param path registry path for service record
+ * @param record service record
+ * @throws Exception
+ */
+ public AAAAApplicationRecordDescriptor(String path,
+ ServiceRecord record) throws Exception {
+ super(path, record);
+ }
+
+ /**
+ * Initializes the descriptor parameters.
+ *
+ * @param serviceRecord the service record.
+ */
+ @Override protected void init(ServiceRecord serviceRecord)
+ throws Exception {
+ super.init(serviceRecord);
+ try {
+ this.setTarget(getIpv6Address(getTarget()));
+ } catch (UnknownHostException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
new file mode 100644
index 0000000..1289fb3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
@@ -0,0 +1,469 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.registry.server.dns;
+
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.types.AddressTypes;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.xbill.DNS.Name;
+import org.xbill.DNS.ReverseMap;
+import org.xbill.DNS.TextParseException;
+
+import java.io.IOException;
+import java.net.Inet6Address;
+import java.net.InetAddress;
+import java.net.URI;
+import java.net.UnknownHostException;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Provides common service record processing logic.
+ */
+public abstract class BaseServiceRecordProcessor
+ implements ServiceRecordProcessor {
+
+ private final ZoneSelector zoneSelctor;
+ private Map<Integer, List<RecordDescriptor>> typeToDescriptorMap =
+ new HashMap<>();
+ private String path;
+ private String domain;
+
+ private static final Pattern USER_NAME = Pattern.compile("/users/(\\w*)/?");
+ private static final String SLIDER_API_PREFIX =
+ "classpath:org.apache.slider.";
+ private static final String HTTP_API_TYPE = "http://";
+
+ /**
+ * Creates a service record processor.
+ *
+ * @param record the service record.
+ * @param path the node path for the record in the registry.
+ * @param domain the target DNS domain for the service record
+ * associated DNS records.
+ * @param zoneSelector A selector of the best zone for a given DNS name.
+ * @throws Exception if an issue is generated during instantiation.
+ */
+ public BaseServiceRecordProcessor(ServiceRecord record, String path,
+ String domain, ZoneSelector zoneSelector)
+ throws Exception {
+ this.setPath(path);
+ this.domain = domain;
+ this.zoneSelctor = zoneSelector;
+ initTypeToInfoMapping(record);
+ }
+
+ /**
+ * Return the username found in the ZK path.
+ *
+ * @param recPath the ZK recPath.
+ * @return the user name.
+ */
+ protected String getUsername(String recPath) {
+ String user = "anonymous";
+ Matcher matcher = USER_NAME.matcher(recPath);
+ if (matcher.find()) {
+ user = matcher.group(1);
+ }
+ return user;
+ }
+
+ /**
+ * Return the IPv6 mapped address for the provided IPv4 address. Utilized
+ * to create corresponding AAAA records.
+ *
+ * @param address the IPv4 address.
+ * @return the mapped IPv6 address.
+ * @throws UnknownHostException
+ */
+ static InetAddress getIpv6Address(InetAddress address)
+ throws UnknownHostException {
+ String[] octets = address.getHostAddress().split("\\.");
+ byte[] octetBytes = new byte[4];
+ for (int i = 0; i < 4; ++i) {
+ octetBytes[i] = (byte) Integer.parseInt(octets[i]);
+ }
+
+ byte[] ipv4asIpV6addr = new byte[16];
+ ipv4asIpV6addr[10] = (byte) 0xff;
+ ipv4asIpV6addr[11] = (byte) 0xff;
+ ipv4asIpV6addr[12] = octetBytes[0];
+ ipv4asIpV6addr[13] = octetBytes[1];
+ ipv4asIpV6addr[14] = octetBytes[2];
+ ipv4asIpV6addr[15] = octetBytes[3];
+
+ return Inet6Address.getByAddress(null, ipv4asIpV6addr, 0);
+ }
+
+ /**
+ * Reverse the string representation of the input IP address.
+ *
+ * @param ip the string representation of the IP address.
+ * @return the reversed IP address.
+ * @throws UnknownHostException if the ip is unknown.
+ */
+ protected Name reverseIP(String ip) throws UnknownHostException {
+ return ReverseMap.fromAddress(ip);
+ }
+
+ /**
+ * Manages the creation and registration of service record generated DNS
+ * records.
+ *
+ * @param command the DNS registration command object (e.g. add_record,
+ * remove record)
+ * @throws IOException if the creation or registration generates an issue.
+ */
+ @SuppressWarnings({"unchecked"})
+ public void manageDNSRecords(RegistryDNS.RegistryCommand command)
+ throws IOException {
+ for (Map.Entry<Integer, List<RecordDescriptor>> entry :
+ typeToDescriptorMap.entrySet()) {
+ for (RecordDescriptor recordDescriptor : entry.getValue()) {
+ for (Name name : recordDescriptor.getNames()) {
+ RecordCreatorFactory.RecordCreator recordCreator =
+ RecordCreatorFactory.getRecordCreator(entry.getKey());
+ command.exec(zoneSelctor.findBestZone(name),
+ recordCreator.create(name, recordDescriptor.getTarget()));
+ }
+ }
+ }
+ }
+
+ /**
+ * Add the DNS record descriptor object to the record type to descriptor
+ * mapping.
+ *
+ * @param type the DNS record type.
+ * @param recordDescriptor the DNS record descriptor
+ */
+ protected void registerRecordDescriptor(int type,
+ RecordDescriptor recordDescriptor) {
+ List<RecordDescriptor> infos = new ArrayList<>();
+ infos.add(recordDescriptor);
+ typeToDescriptorMap.put(type, infos);
+ }
+
+ /**
+ * Add the DNS record descriptor objects to the record type to descriptor
+ * mapping.
+ *
+ * @param type the DNS record type.
+ * @param recordDescriptors the DNS record descriptors
+ */
+ protected void registerRecordDescriptor(int type,
+ List<RecordDescriptor> recordDescriptors) {
+ typeToDescriptorMap.put(type, recordDescriptors);
+ }
+
+ /**
+ * Return the path associated with the record.
+ * @return the path.
+ */
+ protected String getPath() {
+ return path;
+ }
+
+ /**
+ * Set the path associated with the record.
+ * @param path the path.
+ */
+ protected void setPath(String path) {
+ this.path = path;
+ }
+
+ /**
+ * A descriptor container the information to be populated into a DNS record.
+ *
+ * @param <T> the DNS record type/class.
+ */
+ abstract class RecordDescriptor<T> {
+ private final ServiceRecord record;
+ private Name[] names;
+ private T target;
+
+ /**
+ * Creates a DNS record descriptor.
+ *
+ * @param record the associated service record.
+ */
+ public RecordDescriptor(ServiceRecord record) {
+ this.record = record;
+ }
+
+ /**
+ * Returns the DNS names associated with the record type and information.
+ *
+ * @return the array of names.
+ */
+ public Name[] getNames() {
+ return names;
+ }
+
+ /**
+ * Return the target object for the DNS record.
+ *
+ * @return the DNS record target.
+ */
+ public T getTarget() {
+ return target;
+ }
+
+ /**
+ * Initializes the names and information for this DNS record descriptor.
+ *
+ * @param serviceRecord the service record.
+ * @throws Exception
+ */
+ protected abstract void init(ServiceRecord serviceRecord) throws Exception;
+
+ /**
+ * Returns the service record.
+ * @return the service record.
+ */
+ public ServiceRecord getRecord() {
+ return record;
+ }
+
+ /**
+ * Sets the names associated with the record type and information.
+ * @param names the names.
+ */
+ public void setNames(Name[] names) {
+ this.names = names;
+ }
+
+ /**
+ * Sets the target object associated with the record.
+ * @param target the target.
+ */
+ public void setTarget(T target) {
+ this.target = target;
+ }
+ }
+
+ /**
+ * A container-based DNS record descriptor.
+ *
+ * @param <T> the DNS record type/class.
+ */
+ abstract class ContainerRecordDescriptor<T> extends RecordDescriptor<T> {
+
+ public ContainerRecordDescriptor(String path, ServiceRecord record)
+ throws Exception {
+ super(record);
+ init(record);
+ }
+
+ /**
+ * Returns the DNS name constructed from the YARN container ID.
+ *
+ * @return the container ID name.
+ * @throws TextParseException
+ */
+ protected Name getContainerIDName() throws TextParseException {
+ String containerID = RegistryPathUtils.lastPathEntry(getPath());
+ containerID = containerID.replace("container", "ctr");
+ return Name.fromString(String.format("%s.%s", containerID, domain));
+ }
+
+ /**
+ * Returns the DNS name constructed from the container role/component name.
+ *
+ * @return the DNS naem.
+ * @throws PathNotFoundException
+ * @throws TextParseException
+ */
+ protected Name getContainerName()
+ throws PathNotFoundException, TextParseException {
+ String service = RegistryPathUtils.lastPathEntry(
+ RegistryPathUtils.parentOf(RegistryPathUtils.parentOf(getPath())));
+ String description = getRecord().description.toLowerCase();
+ String user = getUsername(getPath());
+ return Name.fromString(MessageFormat.format("{0}.{1}.{2}.{3}",
+ description,
+ service,
+ user,
+ domain));
+ }
+
+ }
+
+ /**
+ * An application-based DNS record descriptor.
+ *
+ * @param <T> the DNS record type/class.
+ */
+ abstract class ApplicationRecordDescriptor<T> extends RecordDescriptor<T> {
+
+ private Endpoint srEndpoint;
+
+ /**
+ * Creates an application associated DNS record descriptor.
+ *
+ * @param record the service record.
+ * @throws Exception
+ */
+ public ApplicationRecordDescriptor(ServiceRecord record)
+ throws Exception {
+ this(record, null);
+ }
+
+ /**
+ * Creates an application associated DNS record descriptor. The endpoint
+ * is leverated to create an associated application API record.
+ *
+ * @param record the service record.
+ * @param endpoint an API endpoint.
+ * @throws Exception
+ */
+ public ApplicationRecordDescriptor(ServiceRecord record,
+ Endpoint endpoint) throws Exception {
+ super(record);
+ this.setEndpoint(endpoint);
+ init(record);
+ }
+
+ /**
+ * Get the service's DNS name for registration.
+ *
+ * @return the service DNS name.
+ * @throws TextParseException
+ */
+ protected Name getServiceName() throws TextParseException {
+ String user = getUsername(getPath());
+ String service =
+ String.format("%s.%s.%s",
+ RegistryPathUtils.lastPathEntry(getPath()),
+ user,
+ domain);
+ return Name.fromString(service);
+ }
+
+ /**
+ * Get the host from the provided endpoint record.
+ *
+ * @param endpoint the endpoint info.
+ * @return the host name.
+ */
+ protected String getHost(Endpoint endpoint) {
+ String host = null;
+ // assume one address for now
+ Map<String, String> address = endpoint.addresses.get(0);
+ if (endpoint.addressType.equals(AddressTypes.ADDRESS_HOSTNAME_AND_PORT)) {
+ host = address.get(AddressTypes.ADDRESS_HOSTNAME_FIELD);
+ } else if (endpoint.addressType.equals(AddressTypes.ADDRESS_URI)) {
+ URI uri = URI.create(address.get("uri"));
+ host = uri.getHost();
+ }
+ return host;
+ }
+
+ /**
+ * Get the post from the provided endpoint record.
+ *
+ * @param endpoint the endpoint info.
+ * @return the port.
+ */
+ protected int getPort(Endpoint endpoint) {
+ int port = -1;
+ // assume one address for now
+ Map<String, String> address = endpoint.addresses.get(0);
+ if (endpoint.addressType.equals(AddressTypes.ADDRESS_HOSTNAME_AND_PORT)) {
+ port = Integer.parseInt(address.get(AddressTypes.ADDRESS_PORT_FIELD));
+ } else if (endpoint.addressType.equals(AddressTypes.ADDRESS_URI)) {
+ URI uri = URI.create(address.get("uri"));
+ port = uri.getPort();
+ }
+ return port;
+ }
+
+ /**
+ * Get the list of strings that can be related in a TXT record for the given
+ * endpoint.
+ *
+ * @param endpoint the endpoint information.
+ * @return the list of strings relating endpoint info.
+ */
+ protected List<String> getTextRecords(Endpoint endpoint) {
+ Map<String, String> address = endpoint.addresses.get(0);
+ List<String> txtRecs = new ArrayList<String>();
+ txtRecs.add("api=" + getDNSApiFragment(endpoint.api));
+ if (endpoint.addressType.equals(AddressTypes.ADDRESS_URI)) {
+ URI uri = URI.create(address.get("uri"));
+ txtRecs.add("path=" + uri.getPath());
+ }
+ return txtRecs;
+ }
+
+ /**
+ * Get an API name that is compatible with DNS standards (and shortened).
+ *
+ * @param api the api indicator.
+ * @return the shortened and compatible api name.
+ */
+ protected String getDNSApiFragment(String api) {
+ String dnsApi = null;
+ if (api.startsWith(SLIDER_API_PREFIX)) {
+ dnsApi = api.substring(SLIDER_API_PREFIX.length());
+ } else if (api.startsWith(HTTP_API_TYPE)) {
+ dnsApi = "http";
+ }
+ assert dnsApi != null;
+ dnsApi = dnsApi.replace('.', '-');
+ return dnsApi;
+ }
+
+ /**
+ * Return the DNS name associated with the API endpoint.
+ *
+ * @return the name.
+ * @throws TextParseException
+ */
+ protected Name getEndpointName() throws TextParseException {
+ return Name.fromString(String.format("%s-api.%s",
+ getDNSApiFragment(
+ getEndpoint().api),
+ getServiceName()));
+ }
+
+ /**
+ * Returns the endpoint.
+ * @return the endpoint.
+ */
+ public Endpoint getEndpoint() {
+ return srEndpoint;
+ }
+
+ /**
+ * Sets the endpoint.
+ * @param endpoint the endpoint.
+ */
+ public void setEndpoint(
+ Endpoint endpoint) {
+ this.srEndpoint = endpoint;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ContainerServiceRecordProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ContainerServiceRecordProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ContainerServiceRecordProcessor.java
new file mode 100644
index 0000000..75873d7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ContainerServiceRecordProcessor.java
@@ -0,0 +1,278 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.registry.server.dns;
+
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.xbill.DNS.Name;
+import org.xbill.DNS.TextParseException;
+import org.xbill.DNS.Type;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A processor for generating container DNS records from registry service
+ * records.
+ */
+public class ContainerServiceRecordProcessor extends
+ BaseServiceRecordProcessor {
+
+ /**
+ * Create a container service record processor.
+ * @param record the service record
+ * @param path the service record registry node path
+ * @param domain the DNS zone/domain name
+ * @param zoneSelector returns the zone associated with the provided name.
+ * @throws Exception if an issue is generated during instantiation.
+ */
+ public ContainerServiceRecordProcessor(
+ ServiceRecord record, String path, String domain,
+ ZoneSelector zoneSelector) throws Exception {
+ super(record, path, domain, zoneSelector);
+ }
+
+ /**
+ * Initializes the DNS record type to descriptor mapping based on the
+ * provided service record.
+ * @param serviceRecord the registry service record.
+ * @throws Exception if an issue arises.
+ */
+ @Override public void initTypeToInfoMapping(ServiceRecord serviceRecord)
+ throws Exception {
+ if (serviceRecord.get(YarnRegistryAttributes.YARN_IP) != null) {
+ for (int type : getRecordTypes()) {
+ switch (type) {
+ case Type.A:
+ createAInfo(serviceRecord);
+ break;
+ case Type.AAAA:
+ createAAAAInfo(serviceRecord);
+ break;
+ case Type.PTR:
+ createPTRInfo(serviceRecord);
+ break;
+ case Type.TXT:
+ createTXTInfo(serviceRecord);
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown type " + type);
+
+ }
+ }
+ }
+ }
+
+ /**
+ * Create a container TXT record descriptor.
+ * @param serviceRecord the service record.
+ * @throws Exception if the descriptor creation yields an issue.
+ */
+ protected void createTXTInfo(ServiceRecord serviceRecord) throws Exception {
+ TXTContainerRecordDescriptor txtInfo =
+ new TXTContainerRecordDescriptor(getPath(), serviceRecord);
+ registerRecordDescriptor(Type.TXT, txtInfo);
+ }
+
+ /**
+ * Creates a container PTR record descriptor.
+ * @param record the service record.
+ * @throws Exception if the descriptor creation yields an issue.
+ */
+ protected void createPTRInfo(ServiceRecord record) throws Exception {
+ PTRContainerRecordDescriptor
+ ptrInfo = new PTRContainerRecordDescriptor(getPath(), record);
+ registerRecordDescriptor(Type.PTR, ptrInfo);
+ }
+
+ /**
+ * Creates a container AAAA (IPv6) record descriptor.
+ * @param record the service record
+ * @throws Exception if the descriptor creation yields an issue.
+ */
+ protected void createAAAAInfo(ServiceRecord record)
+ throws Exception {
+ AAAAContainerRecordDescriptor
+ recordInfo = new AAAAContainerRecordDescriptor(
+ getPath(), record);
+ registerRecordDescriptor(Type.AAAA, recordInfo);
+ }
+
+ /**
+ * Creates a container A (IPv4) record descriptor.
+ * @param record service record.
+ * @throws Exception if the descriptor creation yields an issue.
+ */
+ protected void createAInfo(ServiceRecord record) throws Exception {
+ AContainerRecordDescriptor recordInfo = new AContainerRecordDescriptor(
+ getPath(), record);
+ registerRecordDescriptor(Type.A, recordInfo);
+ }
+
+ /**
+ * Returns the record types associated with a container service record.
+ * @return the record type array
+ */
+ @Override public int[] getRecordTypes() {
+ return new int[] {Type.A, Type.AAAA, Type.PTR, Type.TXT};
+ }
+
+ /**
+ * A container TXT record descriptor.
+ */
+ class TXTContainerRecordDescriptor
+ extends ContainerRecordDescriptor<List<String>> {
+
+ /**
+ * Creates a container TXT record descriptor.
+ * @param path registry path for service record
+ * @param record service record
+ * @throws Exception
+ */
+ public TXTContainerRecordDescriptor(String path,
+ ServiceRecord record) throws Exception {
+ super(path, record);
+ }
+
+ /**
+ * Initializes the descriptor parameters.
+ * @param serviceRecord the service record.
+ */
+ @Override protected void init(ServiceRecord serviceRecord) {
+ try {
+ this.setNames(new Name[] {getContainerIDName()});
+ } catch (TextParseException e) {
+ // log
+ }
+ List<String> txts = new ArrayList<>();
+ txts.add("id=" + serviceRecord.get(YarnRegistryAttributes.YARN_ID));
+ this.setTarget(txts);
+ }
+
+ }
+
+ /**
+ * A container PTR record descriptor.
+ */
+ class PTRContainerRecordDescriptor extends ContainerRecordDescriptor<Name> {
+
+ /**
+ * Creates a container PTR record descriptor.
+ * @param path registry path for service record
+ * @param record service record
+ * @throws Exception
+ */
+ public PTRContainerRecordDescriptor(String path,
+ ServiceRecord record) throws Exception {
+ super(path, record);
+ }
+
+ /**
+ * Initializes the descriptor parameters.
+ * @param serviceRecord the service record.
+ */
+ @Override protected void init(ServiceRecord serviceRecord) {
+ String host = serviceRecord.get(YarnRegistryAttributes.YARN_HOSTNAME);
+ String ip = serviceRecord.get(YarnRegistryAttributes.YARN_IP);
+ Name reverseLookupName = null;
+ if (host != null && ip != null) {
+ try {
+ reverseLookupName = reverseIP(ip);
+ } catch (UnknownHostException e) {
+ //LOG
+ }
+ }
+ this.setNames(new Name[] {reverseLookupName});
+ try {
+ this.setTarget(getContainerIDName());
+ } catch (TextParseException e) {
+ //LOG
+ }
+ }
+
+ }
+
+
+ /**
+ * A container A record descriptor.
+ */
+ class AContainerRecordDescriptor
+ extends ContainerRecordDescriptor<InetAddress> {
+
+ /**
+ * Creates a container A record descriptor.
+ * @param path registry path for service record
+ * @param record service record
+ * @throws Exception
+ */
+ public AContainerRecordDescriptor(String path,
+ ServiceRecord record) throws Exception {
+ super(path, record);
+ }
+
+ /**
+ * Initializes the descriptor parameters.
+ * @param serviceRecord the service record.
+ */
+ @Override protected void init(ServiceRecord serviceRecord) {
+ String ip = serviceRecord.get(YarnRegistryAttributes.YARN_IP);
+ if (ip == null) {
+ throw new IllegalArgumentException("No IP specified");
+ }
+ try {
+ this.setTarget(InetAddress.getByName(ip));
+ this.setNames(new Name[] {getContainerName(), getContainerIDName()});
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+
+ }
+
+ }
+
+ /**
+ * A container AAAA record descriptor.
+ */
+ class AAAAContainerRecordDescriptor extends AContainerRecordDescriptor {
+
+ /**
+ * Creates a container AAAA record descriptor.
+ * @param path registry path for service record
+ * @param record service record
+ * @throws Exception
+ */
+ public AAAAContainerRecordDescriptor(String path,
+ ServiceRecord record) throws Exception {
+ super(path, record);
+ }
+
+ /**
+ * Initializes the descriptor parameters.
+ * @param serviceRecord the service record.
+ */
+ @Override protected void init(ServiceRecord serviceRecord) {
+ super.init(serviceRecord);
+ try {
+ this.setTarget(getIpv6Address(getTarget()));
+ } catch (UnknownHostException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RecordCreatorFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RecordCreatorFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RecordCreatorFactory.java
new file mode 100644
index 0000000..23f9501
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RecordCreatorFactory.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.registry.server.dns;
+
+import org.xbill.DNS.AAAARecord;
+import org.xbill.DNS.ARecord;
+import org.xbill.DNS.CNAMERecord;
+import org.xbill.DNS.DClass;
+import org.xbill.DNS.Name;
+import org.xbill.DNS.PTRRecord;
+import org.xbill.DNS.Record;
+import org.xbill.DNS.SRVRecord;
+import org.xbill.DNS.TXTRecord;
+
+import java.net.InetAddress;
+import java.util.List;
+
+import static org.xbill.DNS.Type.*;
+
+/**
+ * A factory for creating DNS records.
+ */
+public final class RecordCreatorFactory {
+ private static long ttl;
+
+ /**
+ * Private constructor.
+ */
+ private RecordCreatorFactory() {
+ }
+
+ /**
+ * Returns the DNS record creator for the provided type.
+ *
+ * @param type the DNS record type.
+ * @return the record creator.
+ */
+ static RecordCreator getRecordCreator(int type) {
+ switch (type) {
+ case A:
+ return new ARecordCreator();
+ case CNAME:
+ return new CNAMERecordCreator();
+ case TXT:
+ return new TXTRecordCreator();
+ case AAAA:
+ return new AAAARecordCreator();
+ case PTR:
+ return new PTRRecordCreator();
+ case SRV:
+ return new SRVRecordCreator();
+ default:
+ throw new IllegalArgumentException("No type " + type);
+
+ }
+ }
+
+ /**
+ * Set the TTL value for the records created by the factory.
+ *
+ * @param ttl the ttl value, in seconds.
+ */
+ public static void setTtl(long ttl) {
+ RecordCreatorFactory.ttl = ttl;
+ }
+
+ /**
+ * A DNS Record creator.
+ *
+ * @param <R> the record type
+ * @param <T> the record's target type
+ */
+ public interface RecordCreator<R extends Record, T> {
+ R create(Name name, T target);
+ }
+
+ /**
+ * An A Record creator.
+ */
+ static class ARecordCreator implements RecordCreator<ARecord, InetAddress> {
+ /**
+ * Creates an A record creator.
+ */
+ public ARecordCreator() {
+ }
+
+ /**
+ * Creates a DNS A record.
+ *
+ * @param name the record name.
+ * @param target the record target/value.
+ * @return an A record.
+ */
+ @Override public ARecord create(Name name, InetAddress target) {
+ return new ARecord(name, DClass.IN, ttl, target);
+ }
+ }
+
+ /**
+ * An AAAA Record creator.
+ */
+ static class AAAARecordCreator
+ implements RecordCreator<AAAARecord, InetAddress> {
+ /**
+ * Creates an AAAA record creator.
+ */
+ public AAAARecordCreator() {
+ }
+
+ /**
+ * Creates a DNS AAAA record.
+ *
+ * @param name the record name.
+ * @param target the record target/value.
+ * @return an A record.
+ */
+ @Override public AAAARecord create(Name name, InetAddress target) {
+ return new AAAARecord(name, DClass.IN, ttl, target);
+ }
+ }
+
+ static class CNAMERecordCreator implements RecordCreator<CNAMERecord, Name> {
+ /**
+ * Creates a CNAME record creator.
+ */
+ public CNAMERecordCreator() {
+ }
+
+ /**
+ * Creates a DNS CNAME record.
+ *
+ * @param name the record name.
+ * @param target the record target/value.
+ * @return an A record.
+ */
+ @Override public CNAMERecord create(Name name, Name target) {
+ return new CNAMERecord(name, DClass.IN, ttl, target);
+ }
+ }
+
+ /**
+ * A TXT Record creator.
+ */
+ static class TXTRecordCreator
+ implements RecordCreator<TXTRecord, List<String>> {
+ /**
+ * Creates a TXT record creator.
+ */
+ public TXTRecordCreator() {
+ }
+
+ /**
+ * Creates a DNS TXT record.
+ *
+ * @param name the record name.
+ * @param target the record target/value.
+ * @return an A record.
+ */
+ @Override public TXTRecord create(Name name, List<String> target) {
+ return new TXTRecord(name, DClass.IN, ttl, target);
+ }
+ }
+
+ /**
+ * A PTR Record creator.
+ */
+ static class PTRRecordCreator implements RecordCreator<PTRRecord, Name> {
+ /**
+ * Creates a PTR record creator.
+ */
+ public PTRRecordCreator() {
+ }
+
+ /**
+ * Creates a DNS PTR record.
+ *
+ * @param name the record name.
+ * @param target the record target/value.
+ * @return an A record.
+ */
+ @Override public PTRRecord create(Name name, Name target) {
+ return new PTRRecord(name, DClass.IN, ttl, target);
+ }
+ }
+
+ /**
+ * A SRV Record creator.
+ */
+ static class SRVRecordCreator
+ implements RecordCreator<SRVRecord, HostPortInfo> {
+ /**
+ * Creates a SRV record creator.
+ */
+ public SRVRecordCreator() {
+ }
+
+ /**
+ * Creates a DNS SRV record.
+ *
+ * @param name the record name.
+ * @param target the record target/value.
+ * @return an A record.
+ */
+ @Override public SRVRecord create(Name name, HostPortInfo target) {
+ return new SRVRecord(name, DClass.IN, ttl, 1, 1, target.getPort(),
+ target.getHost());
+ }
+ }
+
+ /**
+ * An object for storing the host and port info used to generate SRV records.
+ */
+ public static class HostPortInfo {
+ private Name host;
+ private int port;
+
+ /**
+ * Creates an object with a host and port pair.
+ *
+ * @param host the hostname/ip
+ * @param port the port value
+ */
+ public HostPortInfo(Name host, int port) {
+ this.setHost(host);
+ this.setPort(port);
+ }
+
+ /**
+ * Return the host name.
+ * @return the host name.
+ */
+ Name getHost() {
+ return host;
+ }
+
+ /**
+ * Set the host name.
+ * @param host the host name.
+ */
+ void setHost(Name host) {
+ this.host = host;
+ }
+
+ /**
+ * Get the port.
+ * @return the port.
+ */
+ int getPort() {
+ return port;
+ }
+
+ /**
+ * Set the port.
+ * @param port the port.
+ */
+ void setPort(int port) {
+ this.port = port;
+ }
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/51] [abbrv] hadoop git commit: YARN-5909. Remove agent related
code in slider AM. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
deleted file mode 100644
index 78bb8c1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.codehaus.jackson.annotate.JsonProperty;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Component defined in master package metainfo.json
- */
-public class Component extends AbstractComponent {
-
- String category = CATEGORY_MASTER;
- String publishConfig = Boolean.FALSE.toString();
- String minInstanceCount = "0";
- String maxInstanceCount;
- String autoStartOnFailure = Boolean.FALSE.toString();
- String appExports;
- String compExports;
- String type = TYPE_STANDARD;
- List<ComponentExport> componentExports = new ArrayList<>();
- List<DockerContainer> dockerContainers = new ArrayList<>();
- List<ConfigFile> configFiles = new ArrayList<>();
-
- public Component() {
- }
-
- public String getType() {
- return type;
- }
-
- public void setType(String type) {
- this.type = type;
- }
-
- public String getCategory() {
- return category;
- }
-
- public void setCategory(String category) {
- this.category = category;
- }
-
- public String getPublishConfig() {
- return publishConfig;
- }
-
- public void setPublishConfig(String publishConfig) {
- this.publishConfig = publishConfig;
- }
-
- public String getAutoStartOnFailure() {
- return autoStartOnFailure;
- }
-
- public void setAutoStartOnFailure(String autoStartOnFailure) {
- this.autoStartOnFailure = autoStartOnFailure;
- }
-
- public String getAppExports() {
- return appExports;
- }
-
- public void setAppExports(String appExports) {
- this.appExports = appExports;
- }
-
- public String getCompExports() {
- return compExports;
- }
-
- public void setCompExports(String compExports) {
- this.compExports = compExports;
- }
-
- public String getMinInstanceCount() {
- return minInstanceCount;
- }
-
- @JsonProperty("dockerContainers")
- public List<DockerContainer> getDockerContainers() {
- return this.dockerContainers;
- }
-
- public Boolean getAutoStartOnFailureBoolean() {
- if (SliderUtils.isUnset(getAutoStartOnFailure())) {
- return Boolean.FALSE;
- }
-
- return Boolean.parseBoolean(getAutoStartOnFailure());
- }
-
- public int getMinInstanceCountInt() throws BadConfigException {
- if (SliderUtils.isUnset(minInstanceCount)) {
- return 0;
- }
-
- try {
- return Integer.parseInt(minInstanceCount);
- } catch (NumberFormatException nfe) {
- throw new BadConfigException(nfe, "Invalid value for minInstanceCount for %s", name);
- }
- }
-
- public int getMaxInstanceCountInt() throws BadConfigException {
- if (SliderUtils.isUnset(maxInstanceCount)) {
- return Integer.MAX_VALUE;
- }
-
- try {
- return Integer.parseInt(maxInstanceCount);
- } catch (NumberFormatException nfe) {
- throw new BadConfigException(nfe, "Invalid value for maxInstanceCount for %s", name);
- }
- }
-
- public void setMinInstanceCount(String minInstanceCount) {
- this.minInstanceCount = minInstanceCount;
- }
-
- public String getMaxInstanceCount() {
- return maxInstanceCount;
- }
-
- public void setMaxInstanceCount(String maxInstanceCount) {
- this.maxInstanceCount = maxInstanceCount;
- }
-
- public void addComponentExport(ComponentExport export) {
- componentExports.add(export);
- }
-
- public List<ComponentExport> getComponentExports() {
- return componentExports;
- }
-
- public Boolean getRequiresAutoRestart() {
- return Boolean.parseBoolean(this.autoStartOnFailure);
- }
-
- public void addConfigFile(ConfigFile configFile) {
- this.configFiles.add(configFile);
- }
-
- @JsonProperty("configFiles")
- public List<ConfigFile> getConfigFiles() {
- return configFiles;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("{");
- sb.append("\n\"name\": ").append(name);
- sb.append(",\n\"category\": ").append(category);
- sb.append(",\n\"commandScript\" :").append(commandScript);
- for(DockerContainer dc : dockerContainers){
- sb.append(",\n\"container\" :").append(dc.toString());
- }
- sb.append('}');
- return sb.toString();
- }
-
- public void validate(String version) throws SliderException {
- Metainfo.checkNonNull(getName(), "name", "component");
- Metainfo.checkNonNull(getCategory(), "category", "component");
- if (!getCategory().equals(CATEGORY_MASTER)
- && !getCategory().equals(CATEGORY_SLAVE)
- && !getCategory().equals(CATEGORY_CLIENT)) {
- throw new SliderException("Invalid category for the component " + getCategory());
- }
-
- Metainfo.checkNonNull(getType(), "type", "component");
- if (!getType().equals(TYPE_DOCKER)
- && !getType().equals(TYPE_STANDARD)) {
- throw new SliderException("Invalid type for the component " + getType());
- }
-
- if (version.equals(Metainfo.VERSION_TWO_ZERO)) {
- if (getType().equals(TYPE_DOCKER)) {
- throw new SliderException(TYPE_DOCKER + " is not supported in version " + Metainfo.VERSION_TWO_ZERO);
- }
-
- if (getCommands().size() > 0) {
- throw new SliderException("commands are not supported in version " + Metainfo.VERSION_TWO_ZERO);
- }
- }
-
- if (commandScript != null) {
- commandScript.validate(version);
- }
-
- if (version.equals(Metainfo.VERSION_TWO_ONE)) {
- for (ComponentCommand cc : getCommands()) {
- cc.validate(version);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java
deleted file mode 100644
index 52117c5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Represents the metadata associated with the application.
- */
-public class ComponentCommand implements Validate {
- protected static final Logger
- log = LoggerFactory.getLogger(ComponentCommand.class);
-
-
- private String exec;
- private String name = "START";
- private String type = "SHELL";
-
- /**
- * Creator.
- */
- public ComponentCommand() {
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getName() {
- return name;
- }
-
- public void setExec(String exec) {
- this.exec = exec;
- }
-
- public String getExec() {
- return exec;
- }
-
- public void setType(String type) {
- this.type = type;
- }
-
- public String getType() {
- return type;
- }
-
- public void validate(String version) throws SliderException {
- Metainfo.checkNonNull(getName(), "name", "componentCommand");
-
- Metainfo.checkNonNull(getType(), "version", "application");
- }
-
- public static ComponentCommand getDefaultComponentCommand() {
- ComponentCommand cc = new ComponentCommand();
- cc.setExec("DEFAULT");
- return cc;
- }
-
- public static ComponentCommand getDefaultComponentCommand(String commandName) {
- ComponentCommand cc = new ComponentCommand();
- cc.setExec("DEFAULT");
- cc.setName(commandName);
- return cc;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java
deleted file mode 100644
index a18854c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentExport.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-/**
- *
- */
-public class ComponentExport {
- String name;
- String value;
-
- public ComponentExport() {
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getValue() {
- return value;
- }
-
- public void setValue(String value) {
- this.value = value;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("{");
- sb.append(",\n\"name\": ").append(name);
- sb.append(",\n\"value\": ").append(value);
- sb.append('}');
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java
deleted file mode 100644
index 855e5b6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentsInAddonPackage.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-
-public class ComponentsInAddonPackage extends AbstractComponent {
-
- @Override
- public void validate(String version) throws SliderException {
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java
deleted file mode 100644
index cb47512..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-
-/**
- *
- */
-public class ConfigFile implements Validate {
- String type;
- String fileName;
- String dictionaryName;
-
- public ConfigFile() {
- }
-
- public String getType() {
- return type;
- }
-
- public void setType(String type) {
- this.type = type;
- }
-
- public String getFileName() {
- return fileName;
- }
-
- public void setFileName(String fileName) {
- this.fileName = fileName;
- }
-
- public String getDictionaryName() {
- return dictionaryName;
- }
-
- public void setDictionaryName(String dictionaryName) {
- this.dictionaryName = dictionaryName;
- }
-
- public void validate(String version) throws SliderException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java
deleted file mode 100644
index 46c8836..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfig.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Application default config
- */
-public class DefaultConfig {
- List<PropertyInfo> propertyInfos;
-
- public DefaultConfig() {
- propertyInfos = new ArrayList<PropertyInfo>();
- }
-
- public void addPropertyInfo(PropertyInfo propertyInfo) {
- propertyInfos.add(propertyInfo);
- }
-
- public List<PropertyInfo> getPropertyInfos() {
- return propertyInfos;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java
deleted file mode 100644
index e136775..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DefaultConfigParser.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.commons.digester.Digester;
-import org.xml.sax.SAXException;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- *
- */
-public class DefaultConfigParser {
-
- public DefaultConfig parse(InputStream configFileStream) throws IOException {
- Digester digester = new Digester();
- digester.setValidating(false);
-
- digester.addObjectCreate("configuration", DefaultConfig.class);
-
- digester.addObjectCreate("*/property", PropertyInfo.class);
- digester.addBeanPropertySetter("*/property/name");
- digester.addBeanPropertySetter("*/property/value");
- digester.addBeanPropertySetter("*/property/description");
- digester.addSetNext("*/property", "addPropertyInfo");
-
- try {
- return (DefaultConfig) digester.parse(configFileStream);
- } catch (IOException e) {
-
- } catch (SAXException e) {
-
- } finally {
- configFileStream.close();
- }
-
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java
deleted file mode 100644
index 4c61e7f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Represents a docker container
- */
-public class DockerContainer implements Validate {
- protected static final Logger
- log = LoggerFactory.getLogger(DockerContainer.class);
-
- private String name;
- private String image;
- private String network;
- private String useNetworkScript;
- private String options;
- private List<DockerContainerMount> mounts = new ArrayList<>();
- private List<DockerContainerPort> ports = new ArrayList<>();
- private String statusCommand;
- private String startCommand;
- private String commandPath;
- private String additionalParam;
- private String runPrivilegedContainer;
- private List<DockerContainerInputFile> inputFiles = new ArrayList<>();
- private List<ConfigFile> configFiles = new ArrayList<>();
-
- public DockerContainer() {
- }
-
- @JsonProperty("mounts")
- public List<DockerContainerMount> getMounts() { return this.mounts; }
-
- @JsonProperty("ports")
- public List<DockerContainerPort> getPorts() {
- return this.ports;
- }
-
- @JsonProperty("inputFiles")
- public List<DockerContainerInputFile> getInputFiles() {
- return this.inputFiles;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getImage() {
- return image;
- }
-
- public void setImage(String image) {
- this.image = image;
- }
-
- public String getNetwork() {
- return network;
- }
-
- public void setNetwork(String network) {
- this.network = network;
- }
-
- public String getUseNetworkScript() {
- return useNetworkScript;
- }
-
- public void setUseNetworkScript(String useNetworkScript) {
- this.useNetworkScript = useNetworkScript;
- }
-
- public String getOptions() {
- return options;
- }
-
- public void setOptions(String options) {
- this.options = options;
- }
-
- @Override
- public void validate(String version) throws SliderException {
- Metainfo.checkNonNull(getName(), "name", "dockerContainer");
- Metainfo.checkNonNull(getImage(), "image", "dockerContainer");
- for (DockerContainerMount dcm : getMounts()) {
- dcm.validate(version);
- }
- for (DockerContainerPort dcp : getPorts()) {
- dcp.validate(version);
- }
- }
-
- @JsonProperty("statusCommand")
- public String getStatusCommand() {
- return statusCommand;
- }
-
- @JsonProperty("statusCommand")
- public void setStatusCommand(String statusCommand) {
- this.statusCommand = statusCommand;
- }
-
- public String getCommandPath() {
- return commandPath;
- }
-
- public void setCommandPath(String commandPath) {
- this.commandPath = commandPath;
- }
-
- public String getAdditionalParam() {
- return additionalParam;
- }
-
- public void setAdditionalParam(String additionalParam) {
- this.additionalParam = additionalParam;
- }
-
- @JsonProperty("startCommand")
- public String getStartCommand() {
- return startCommand;
- }
-
- @JsonProperty("startCommand")
- public void setStartCommand(String startCommand) {
- this.startCommand = startCommand;
- }
-
- @JsonProperty("runPrivilegedContainer")
- public String getRunPrivilegedContainer() {
- return runPrivilegedContainer;
- }
-
- @JsonProperty("runPrivilegedContainer")
- public void setRunPrivilegedContainer(String runPrivilegedContainer) {
- this.runPrivilegedContainer = runPrivilegedContainer;
- }
-
- public List<ConfigFile> getConfigFiles() {
- return configFiles;
- }
-
- public void setConfigFiles(List<ConfigFile> configFiles) {
- this.configFiles = configFiles;
- }
-
- @Override
- public String toString() {
- StringBuilder result = new StringBuilder("DockerContainer [name=")
- .append(name).append(", image=").append(image).append(", options=")
- .append(options).append(", mounts=").append(mounts).append(", ports=")
- .append(ports).append(", statusCommand=").append(statusCommand)
- .append(", commandPath=").append(commandPath)
- .append(", additionalParam=").append(additionalParam)
- .append(", inputFiles=").append(inputFiles).append(", startCommand=")
- .append(startCommand).append(", runPriviledgedContainer=")
- .append(runPrivilegedContainer).append(", configFiles=")
- .append(configFiles).append("]");
- return result.toString();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerInputFile.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerInputFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerInputFile.java
deleted file mode 100644
index 0faceb9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerInputFile.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class DockerContainerInputFile {
- protected static final Logger log = LoggerFactory
- .getLogger(DockerContainerInputFile.class);
-
- private String containerPath;
- private String fileLocalPath;
-
- public DockerContainerInputFile() {
- }
-
- public String getContainerMount() {
- return containerPath;
- }
-
- public void setContainerMount(String containerMount) {
- this.containerPath = containerMount;
- }
-
- public String getFileLocalPath() {
- return fileLocalPath;
- }
-
- public void setFileLocalPath(String fileLocalPath) {
- this.fileLocalPath = fileLocalPath;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerMount.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerMount.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerMount.java
deleted file mode 100644
index 61f07f4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerMount.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Represents a docker container mount
- */
-public class DockerContainerMount implements Validate {
- protected static final Logger
- log = LoggerFactory.getLogger(DockerContainerMount.class);
-
-
- private String containerMount;
- private String hostMount;
-
- public DockerContainerMount() {
- }
-
- public String getContainerMount() {
- return containerMount;
- }
-
- public void setContainerMount(String containerMount) {
- this.containerMount = containerMount;
- }
-
- public String getHostMount() {
- return hostMount;
- }
-
- public void setHostMount(String hostMount) {
- this.hostMount = hostMount;
- }
-
- @Override
- public void validate(String version) throws SliderException {
- Metainfo.checkNonNull(getContainerMount(), "containerMount", "dockerContainerMount");
- Metainfo.checkNonNull(getHostMount(), "hostMount", "dockerContainerMount");
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerPort.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerPort.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerPort.java
deleted file mode 100644
index 0629d9d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainerPort.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Represents a docker container port
- */
-public class DockerContainerPort implements Validate {
- protected static final Logger
- log = LoggerFactory.getLogger(DockerContainerPort.class);
-
-
- private String containerPort;
- private String hostPort;
-
- public DockerContainerPort() {
- }
-
- public String getContainerPort() {
- return containerPort;
- }
-
- public void setContainerPort(String containerPort) {
- this.containerPort = containerPort;
- }
-
- public String getHostPort() {
- return hostPort;
- }
-
- public void setHostPort(String hostPort) {
- this.hostPort = hostPort;
- }
-
- @Override
- public void validate(String version) throws SliderException {
- Metainfo.checkNonNull(getContainerPort(), "containerPort", "dockerContainerPort");
- Metainfo.checkNonNull(getHostPort(), "hostPort", "dockerContainerPort");
- }
-
- @Override
- public String toString() {
- return "DockerContainerPort [containerPort=" + containerPort
- + ", hostPort=" + hostPort + "]";
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java
deleted file mode 100644
index 5e0fb24..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-
-/**
- *
- */
-public class Export implements Validate {
- String name;
- String value;
-
- public Export() {
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getValue() {
- return value;
- }
-
- public void setValue(String value) {
- this.value = value;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("{");
- sb.append(",\n\"name\": ").append(name);
- sb.append(",\n\"value\": ").append(value);
- sb.append('}');
- return sb.toString();
- }
-
- public void validate(String version) throws SliderException {
- Metainfo.checkNonNull(getName(), "name", "export");
- Metainfo.checkNonNull(getValue(), "value", "export");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java
deleted file mode 100644
index 3d9f34c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- *
- */
-public class ExportGroup implements Validate {
- String name;
- List<Export> exports;
-
- public ExportGroup() {
- exports = new ArrayList<Export>();
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public void addExport(Export export) {
- exports.add(export);
- }
-
- public List<Export> getExports() {
- return exports;
- }
-
- @Override
- public String toString() {
- final StringBuilder sb =
- new StringBuilder("{");
- sb.append(",\n\"name\": ").append(name);
- sb.append(",\n\"exports\" : {");
- for (Export export : exports) {
- sb.append("\n").append(export);
- }
- sb.append("\n},");
- sb.append('}');
- return sb.toString();
- }
-
- public void validate(String version) throws SliderException {
- Metainfo.checkNonNull(getName(), "name", "exportGroup");
- for(Export exp : getExports()) {
- exp.validate(version);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
deleted file mode 100644
index 10c497f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.SliderException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Application metainfo uber class
- */
-public class Metainfo {
- protected static final Logger log =
- LoggerFactory.getLogger(Metainfo.class);
- public static String VERSION_TWO_ZERO = "2.0";
- public static String VERSION_TWO_ONE = "2.1";
-
- String schemaVersion;
- ApplicationPackage applicationPackage;
- Application application;
-
- public String getSchemaVersion() {
- return schemaVersion;
- }
-
- public void setSchemaVersion(String schemaVersion) {
- this.schemaVersion = schemaVersion;
- }
-
- public ApplicationPackage getApplicationPackage() {
- return applicationPackage;
- }
-
- public void setApplicationPackage(ApplicationPackage pkg) {
- this.applicationPackage = pkg;
- }
-
- public Application getApplication() {
- return application;
- }
-
- public void setApplication(Application application) {
- this.application = application;
- }
-
- public Component getApplicationComponent(String roleGroup) {
- if (application == null) {
- log.error("Malformed app definition: Expect application as the top level element for metainfo");
- } else {
- for (Component component : application.getComponents()) {
- if (component.getName().equals(roleGroup)) {
- return component;
- }
- }
- }
- return null;
- }
-
- public List<ConfigFile> getComponentConfigFiles(String roleGroup) {
- List<ConfigFile> componentConfigFiles = new ArrayList<>();
- componentConfigFiles.addAll(application.getConfigFiles());
- Component component = getApplicationComponent(roleGroup);
- if (component != null) {
- componentConfigFiles.addAll(component.getConfigFiles());
- }
- return componentConfigFiles;
- }
-
- public void validate() throws SliderException {
- if (!VERSION_TWO_ONE.equals(schemaVersion) &&
- !VERSION_TWO_ZERO.equals(schemaVersion)) {
- throw new SliderException("Unsupported version " + getSchemaVersion());
- }
- if (application != null) {
- application.validate(schemaVersion);
- }
- if (applicationPackage != null) {
- applicationPackage.validate(schemaVersion);
- }
- }
-
- public static void checkNonNull(String value, String field, String type) throws SliderException {
- if (SliderUtils.isUnset(value)) {
- throw new SliderException(type + "." + field + " cannot be null");
- }
- }
-
- @Override
- public String toString() {
- StringBuilder builder = new StringBuilder();
- builder.append("Metainfo [schemaVersion=");
- builder.append(schemaVersion);
- builder.append(", applicationPackage=");
- builder.append(applicationPackage);
- builder.append(", application=");
- builder.append(application);
- builder.append("]");
- return builder.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
deleted file mode 100644
index 8b520eb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.commons.digester.Digester;
-
-/**
- *
- */
-public class MetainfoParser extends AbstractMetainfoParser{
-
- protected void composeSchema(Digester digester){
- digester.addObjectCreate("metainfo", Metainfo.class);
- digester.addBeanPropertySetter("metainfo/schemaVersion");
-
- digester.addObjectCreate("*/application", Application.class);
- digester.addBeanPropertySetter("*/application/name");
- digester.addBeanPropertySetter("*/application/comment");
- digester.addBeanPropertySetter("*/application/version");
- digester.addBeanPropertySetter("*/application/exportedConfigs");
-
- digester.addObjectCreate("*/commandOrder", CommandOrder.class);
- digester.addBeanPropertySetter("*/commandOrder/command");
- digester.addBeanPropertySetter("*/commandOrder/requires");
- digester.addSetNext("*/commandOrder", "addCommandOrder");
-
- digester.addObjectCreate("*/exportGroup", ExportGroup.class);
- digester.addBeanPropertySetter("*/exportGroup/name");
- digester.addObjectCreate("*/export", Export.class);
- digester.addBeanPropertySetter("*/export/name");
- digester.addBeanPropertySetter("*/export/value");
- digester.addSetNext("*/export", "addExport");
- digester.addSetNext("*/exportGroup", "addExportGroup");
-
- digester.addObjectCreate("*/component", Component.class);
- digester.addBeanPropertySetter("*/component/name");
- digester.addBeanPropertySetter("*/component/category");
- digester.addBeanPropertySetter("*/component/publishConfig");
- digester.addBeanPropertySetter("*/component/minInstanceCount");
- digester.addBeanPropertySetter("*/component/maxInstanceCount");
- digester.addBeanPropertySetter("*/component/autoStartOnFailure");
- digester.addBeanPropertySetter("*/component/appExports");
- digester.addBeanPropertySetter("*/component/compExports");
- digester.addObjectCreate("*/componentExport", ComponentExport.class);
- digester.addBeanPropertySetter("*/componentExport/name");
- digester.addBeanPropertySetter("*/componentExport/value");
- digester.addSetNext("*/componentExport", "addComponentExport");
- digester.addSetNext("*/component", "addComponent");
-
- digester.addObjectCreate("*/commandScript", CommandScript.class);
- digester.addBeanPropertySetter("*/commandScript/script");
- digester.addBeanPropertySetter("*/commandScript/scriptType");
- digester.addBeanPropertySetter("*/commandScript/timeout");
- digester.addSetNext("*/commandScript", "addCommandScript");
-
- digester.addObjectCreate("*/command", ComponentCommand.class);
- digester.addBeanPropertySetter("*/command/exec");
- digester.addBeanPropertySetter("*/command/name");
- digester.addBeanPropertySetter("*/command/type");
- digester.addSetNext("*/command", "addCommand");
-
- digester.addObjectCreate("*/osSpecific", OSSpecific.class);
- digester.addBeanPropertySetter("*/osSpecific/osType");
- digester.addObjectCreate("*/osSpecific/packages/package", OSPackage.class);
- digester.addBeanPropertySetter("*/osSpecific/packages/package/type");
- digester.addBeanPropertySetter("*/osSpecific/packages/package/name");
- digester.addSetNext("*/osSpecific/packages/package", "addOSPackage");
- digester.addSetNext("*/osSpecific", "addOSSpecific");
-
- digester.addObjectCreate("*/application/packages/package", Package.class);
- digester.addBeanPropertySetter("*/application/packages/package/type");
- digester.addBeanPropertySetter("*/application/packages/package/name");
- digester.addSetNext("*/application/packages/package", "addPackage");
-
- digester.addObjectCreate("*/configFile", ConfigFile.class);
- digester.addBeanPropertySetter("*/configFile/type");
- digester.addBeanPropertySetter("*/configFile/fileName");
- digester.addBeanPropertySetter("*/configFile/dictionaryName");
- digester.addSetNext("*/configFile", "addConfigFile");
-
- digester.addSetRoot("*/application", "setApplication");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java
deleted file mode 100644
index 32b4890..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-
-/**
- *
- */
-public class OSPackage implements Validate {
- String type;
- String name;
-
- public OSPackage() {
- }
-
- public String getType() {
- return type;
- }
-
- public void setType(String type) {
- this.type = type;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public void validate(String version) throws SliderException {
- Metainfo.checkNonNull(getName(), "name", "osPackage");
- Metainfo.checkNonNull(getType(), "type", "osPackage");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java
deleted file mode 100644
index c06d498..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- *
- */
-public class OSSpecific implements Validate {
- String osType;
- List<OSPackage> packages;
-
- public OSSpecific() {
- packages = new ArrayList<OSPackage>();
- }
-
- public String getOsType() {
- return osType;
- }
-
- public void setOsType(String osType) {
- this.osType = osType;
- }
-
- public void addOSPackage(OSPackage osPackage) {
- packages.add(osPackage);
- }
-
- public List<OSPackage> getPackages() {
- return packages;
- }
-
- public void validate(String version) throws SliderException {
- Metainfo.checkNonNull(getOsType(), "osType", "osSpecific");
- for (OSPackage opkg : getPackages()) {
- opkg.validate(version);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java
deleted file mode 100644
index b88f77d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Represents package description.
- */
-public class Package implements Validate {
- protected static final Logger
- log = LoggerFactory.getLogger(Package.class);
-
-
- private String name;
- private String type;
-
- public Package() {
- }
-
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getName() {
- return name;
- }
-
- public String getType() {
- return type;
- }
-
- public void setType(String type) {
- this.type = type;
- }
-
- public void validate(String version) throws SliderException {
- Metainfo.checkNonNull(getName(), "name", "package");
- Metainfo.checkNonNull(getType(), "type", "package");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java
deleted file mode 100644
index 62ee0f5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/PropertyInfo.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-/**
- * Application config property info
- */
-public class PropertyInfo {
- String name;
- String value;
- String description;
-
- public PropertyInfo() {
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getValue() {
- return value;
- }
-
- public void setValue(String value) {
- this.value = value;
- }
-
- public String getDescription() {
- return description;
- }
-
- public void setDescription(String description) {
- this.description = description;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java
deleted file mode 100644
index ef03dcd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.core.exceptions.SliderException;
-
-/**
- * Implementer provides a validate method
- */
-public interface Validate {
-
- public void validate(String version) throws SliderException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/todo.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/todo.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/todo.md
deleted file mode 100644
index dfd1373..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/todo.md
+++ /dev/null
@@ -1,22 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-# todo list
-
-* Retry on failure
- * Agent can toleate a configurable number of failures (e.g. 3) before giving up
-* Agent should separate out hostname and label that is received for registration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 57ec218..34b6a7d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -20,12 +20,8 @@ package org.apache.slider.server.appmaster;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.health.HealthCheckRegistry;
-import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
-import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
-import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
import com.google.common.base.Preconditions;
import com.google.protobuf.BlockingService;
-
import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -36,8 +32,14 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -66,18 +68,9 @@ import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
-import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG;
-
import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
@@ -122,18 +115,16 @@ import org.apache.slider.providers.ProviderCompleted;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.providers.SliderProviderFactory;
-import org.apache.slider.providers.agent.AgentKeys;
-import org.apache.slider.providers.agent.AgentProviderService;
import org.apache.slider.providers.slideram.SliderAMClientProvider;
import org.apache.slider.providers.slideram.SliderAMProviderService;
import org.apache.slider.server.appmaster.actions.ActionRegisterServiceInstance;
-import org.apache.slider.server.appmaster.actions.EscalateOutstandingRequests;
-import org.apache.slider.server.appmaster.actions.RegisterComponentInstance;
-import org.apache.slider.server.appmaster.actions.QueueExecutor;
-import org.apache.slider.server.appmaster.actions.QueueService;
import org.apache.slider.server.appmaster.actions.ActionStopSlider;
import org.apache.slider.server.appmaster.actions.ActionUpgradeContainers;
import org.apache.slider.server.appmaster.actions.AsyncAction;
+import org.apache.slider.server.appmaster.actions.EscalateOutstandingRequests;
+import org.apache.slider.server.appmaster.actions.QueueExecutor;
+import org.apache.slider.server.appmaster.actions.QueueService;
+import org.apache.slider.server.appmaster.actions.RegisterComponentInstance;
import org.apache.slider.server.appmaster.actions.RenewingAction;
import org.apache.slider.server.appmaster.actions.ResetFailureWindow;
import org.apache.slider.server.appmaster.actions.ReviewAndFlexApplicationSize;
@@ -143,26 +134,24 @@ import org.apache.slider.server.appmaster.management.YarnServiceHealthCheck;
import org.apache.slider.server.appmaster.monkey.ChaosKillAM;
import org.apache.slider.server.appmaster.monkey.ChaosKillContainer;
import org.apache.slider.server.appmaster.monkey.ChaosMonkeyService;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
import org.apache.slider.server.appmaster.operations.AsyncRMOperationHandler;
import org.apache.slider.server.appmaster.operations.ProviderNotifyingOperationHandler;
+import org.apache.slider.server.appmaster.operations.RMOperationHandler;
import org.apache.slider.server.appmaster.rpc.RpcBinder;
import org.apache.slider.server.appmaster.rpc.SliderAMPolicyProvider;
import org.apache.slider.server.appmaster.rpc.SliderClusterProtocolPBImpl;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
import org.apache.slider.server.appmaster.rpc.SliderIPCService;
import org.apache.slider.server.appmaster.security.SecurityConfiguration;
import org.apache.slider.server.appmaster.state.AppState;
import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
import org.apache.slider.server.appmaster.state.ContainerAssignment;
import org.apache.slider.server.appmaster.state.ProviderAppState;
-import org.apache.slider.server.appmaster.operations.RMOperationHandler;
import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.web.AgentService;
-import org.apache.slider.server.appmaster.web.rest.InsecureAmFilterInitializer;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentWebApp;
import org.apache.slider.server.appmaster.web.SliderAMWebApp;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.WebAppApiImpl;
+import org.apache.slider.server.appmaster.web.rest.InsecureAmFilterInitializer;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
import org.apache.slider.server.appmaster.web.rest.application.ApplicationResouceContentCacheFactory;
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
@@ -181,7 +170,6 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URL;
-import java.net.URLClassLoader;
import java.nio.ByteBuffer;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
@@ -199,6 +187,9 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
+import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG;
+
/**
* This is the AM, which directly implements the callbacks from the AM and NM
*/
@@ -765,11 +756,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
appMasterContainerID.toString(),
clustername);
certificateManager.setPassphrase(instanceDefinition.getPassphrase());
-
- if (component.getOptionBool(
- AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED, false)) {
- uploadServerCertForLocalization(clustername, fs);
- }
// Web service endpoints: initialize
WebAppApiImpl webAppApi =
@@ -784,10 +770,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
contentCache);
initAMFilterOptions(serviceConf);
- if (providerService instanceof AgentProviderService) {
- // start the agent web app
- startAgentWebApp(appInformation, serviceConf, webAppApi);
- }
int webAppPort = deployWebApplication(webAppApi);
String scheme = WebAppUtils.HTTP_PREFIX;
@@ -1165,26 +1147,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
return portScanner.getAvailablePort();
}
- private void uploadServerCertForLocalization(String clustername,
- SliderFileSystem fs)
- throws IOException {
- Path certsDir = fs.buildClusterSecurityDirPath(clustername);
- if (!fs.getFileSystem().exists(certsDir)) {
- fs.getFileSystem().mkdirs(certsDir,
- new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
- }
- Path destPath = new Path(certsDir, SliderKeys.CRT_FILE_NAME);
- if (!fs.getFileSystem().exists(destPath)) {
- fs.getFileSystem().copyFromLocalFile(
- new Path(CertificateManager.getServerCertficateFilePath().getAbsolutePath()),
- destPath);
- log.info("Uploaded server cert to localization path {}", destPath);
- }
-
- fs.getFileSystem().setPermission(destPath,
- new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE));
- }
-
protected void login(String principal, File localKeytabFile)
throws IOException, SliderException {
log.info("Logging in as {} with keytab {}", principal, localKeytabFile);
@@ -1221,53 +1183,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
/**
- * Set up and start the agent web application
- * @param appInformation application information
- * @param serviceConf service configuration
- * @param webAppApi web app API instance to bind to
- * @throws IOException
- */
- private void startAgentWebApp(MapOperations appInformation,
- Configuration serviceConf, WebAppApiImpl webAppApi) throws IOException, SliderException {
- URL[] urls = ((URLClassLoader) AgentWebApp.class.getClassLoader() ).getURLs();
- StringBuilder sb = new StringBuilder("AM classpath:");
- for (URL url : urls) {
- sb.append("\n").append(url.toString());
- }
- LOG_YARN.debug(sb.append("\n").toString());
- initAMFilterOptions(serviceConf);
-
-
- // Start up the agent web app and track the URL for it
- MapOperations appMasterConfig = getInstanceDefinition()
- .getAppConfOperations().getComponent(SliderKeys.COMPONENT_AM);
- AgentWebApp agentWebApp = AgentWebApp.$for(AgentWebApp.BASE_PATH,
- webAppApi,
- RestPaths.AGENT_WS_CONTEXT)
- .withComponentConfig(appMasterConfig)
- .withPort(getPortToRequest())
- .withSecuredPort(getPortToRequest())
- .start();
- agentOpsUrl =
- "https://" + appMasterHostname + ":" + agentWebApp.getSecuredPort();
- agentStatusUrl =
- "https://" + appMasterHostname + ":" + agentWebApp.getPort();
- AgentService agentService =
- new AgentService("slider-agent", agentWebApp);
-
- agentService.init(serviceConf);
- agentService.start();
- addService(agentService);
-
- appInformation.put(StatusKeys.INFO_AM_AGENT_OPS_URL, agentOpsUrl + "/");
- appInformation.put(StatusKeys.INFO_AM_AGENT_STATUS_URL, agentStatusUrl + "/");
- appInformation.set(StatusKeys.INFO_AM_AGENT_STATUS_PORT,
- agentWebApp.getPort());
- appInformation.set(StatusKeys.INFO_AM_AGENT_OPS_PORT,
- agentWebApp.getSecuredPort());
- }
-
- /**
* Set up the AM filter
* @param serviceConf configuration to patch
*/
@@ -1327,17 +1242,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
serviceRecord);
// provider service dynamic definitions.
- if (providerService instanceof AgentProviderService) {
- URL agentOpsURI = new URL(agentOpsUrl);
- URL agentStatusURI = new URL(agentStatusUrl);
- ((AgentProviderService)providerService).applyInitialRegistryDefinitions(
- amWebURI,
- agentOpsURI,
- agentStatusURI,
- serviceRecord);
- } else {
- providerService.applyInitialRegistryDefinitions(amWebURI, serviceRecord);
- }
+ providerService.applyInitialRegistryDefinitions(amWebURI, serviceRecord);
+
// set any provided attributes
setProvidedServiceRecordAttributes(
@@ -1800,11 +1706,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
LOG_YARN.info("Final list of containers to be upgraded (total {}) : {}",
containers.size(), containers);
- if (providerService instanceof AgentProviderService) {
- AgentProviderService agentProviderService = (AgentProviderService) providerService;
- agentProviderService.setInUpgradeMode(true);
- agentProviderService.addUpgradeContainers(containers);
- }
}
// create a reverse map of roles -> set of all live containers
@@ -1968,11 +1869,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* Shutdown operation: release all containers
*/
private void releaseAllContainers() {
- if (providerService instanceof AgentProviderService) {
- log.info("Setting stopInitiated flag to true");
- AgentProviderService agentProviderService = (AgentProviderService) providerService;
- agentProviderService.setAppStopInitiated(true);
- }
// Add the sleep here (before releasing containers) so that applications get
// time to perform graceful shutdown
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java
deleted file mode 100644
index f840035..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/AgentService.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web;
-
-import org.apache.slider.server.appmaster.web.rest.agent.AgentWebApp;
-import org.apache.slider.server.services.workflow.ClosingService;
-import org.apache.slider.server.services.workflow.WorkflowCompositeService;
-
-/**
- * agent service gives the agent webapp lifecycle integration
- */
-public class AgentService extends ClosingService<AgentWebApp> {
-
-
- public AgentService(String name) {
- super(name);
- }
-
- public AgentService(String name, AgentWebApp app) {
- super(name, app);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
index 0f99d6d..65a3591 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
@@ -17,7 +17,6 @@
package org.apache.slider.server.appmaster.web;
import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.slider.api.SliderClusterProtocol;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.AppMasterActionOperations;
import org.apache.slider.server.appmaster.actions.QueueAccess;
@@ -25,7 +24,6 @@ import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.state.AppState;
import org.apache.slider.server.appmaster.state.RoleStatus;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
import org.apache.slider.server.services.security.CertificateManager;
@@ -57,11 +55,6 @@ public interface WebAppApi {
* is a computed value and not just a getter
*/
Map<String, RoleStatus> getRoleStatusByName();
-
- /**
- * Returns an interface that can support the agent-based REST operations.
- */
- AgentRestOperations getAgentRestOperations();
/**
* Registry operations accessor
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/51] [abbrv] hadoop git commit: YARN-5967. Fix slider core module
findbugs warnings. Contributed by Jian He
Posted by ji...@apache.org.
YARN-5967. Fix slider core module findbugs warnings. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/562c8263
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/562c8263
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/562c8263
Branch: refs/heads/yarn-native-services
Commit: 562c8263e033ffd74d0dc1177992007bec7986da
Parents: 0cb7071
Author: Billie Rinaldi <bi...@apache.org>
Authored: Mon Dec 19 12:09:53 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../dev-support/findbugs-exclude.xml | 96 +++++
.../dev-support/findbugs-exclude.xml | 20 -
.../hadoop-yarn-slider-core/pom.xml | 5 +
.../apache/hadoop/security/KerberosDiags.java | 5 +-
.../slider/api/proto/RestTypeMarshalling.java | 283 -------------
.../slider/api/types/ComponentInformation.java | 2 +-
.../slider/api/types/RestTypeMarshalling.java | 284 +++++++++++++
.../org/apache/slider/client/SliderClient.java | 56 +--
.../apache/slider/client/TokensOperation.java | 9 +-
.../client/ipc/SliderClusterOperations.java | 2 +-
.../rest/SliderApplicationApiRestClient.java | 14 -
.../org/apache/slider/common/SliderKeys.java | 8 +-
.../common/params/AbstractActionArgs.java | 5 +-
.../AbstractClusterBuildingActionArgs.java | 1 -
.../apache/slider/common/params/CommonArgs.java | 9 +-
.../apache/slider/common/tools/Comparators.java | 17 +-
.../slider/common/tools/ConfigHelper.java | 2 +-
.../slider/common/tools/CoreFileSystem.java | 31 --
.../apache/slider/common/tools/SliderUtils.java | 102 +----
.../slider/core/buildutils/InstanceBuilder.java | 1 -
.../apache/slider/core/conf/AggregateConf.java | 2 +-
.../org/apache/slider/core/conf/ConfTree.java | 15 +-
.../core/persist/AppDefinitionPersister.java | 15 +-
.../core/registry/retrieve/AMWebClient.java | 51 ---
.../core/restclient/HttpOperationResponse.java | 34 --
.../restclient/UrlConnectionOperations.java | 120 ------
.../slider/core/zk/MiniZooKeeperCluster.java | 27 +-
.../apache/slider/core/zk/ZKIntegration.java | 20 +-
.../apache/slider/providers/ProviderUtils.java | 21 +-
.../providers/docker/DockerProviderService.java | 12 +-
.../server/appmaster/management/BoolMetric.java | 15 -
.../management/MetricsAndMonitoring.java | 25 --
.../management/RangeLimitedCounter.java | 85 ----
.../appmaster/management/RecordedEvent.java | 58 ---
.../server/appmaster/rpc/SliderIPCService.java | 2 +-
.../server/appmaster/state/NodeInstance.java | 2 +-
.../state/OutstandingRequestTracker.java | 2 +-
.../server/appmaster/state/RoleStatus.java | 4 -
.../appmaster/web/rest/InsecureAmFilter.java | 1 -
.../web/rest/publisher/PublisherResource.java | 5 +-
.../appmaster/web/view/ContainerStatsBlock.java | 3 +-
.../server/servicemonitor/ProbeStatus.java | 1 +
.../utility/AbstractSliderLaunchedService.java | 2 -
.../services/workflow/LongLivedProcess.java | 5 +-
.../slider/common/tools/TestSliderUtils.java | 2 +-
.../slider/test/MiniZooKeeperCluster.java | 395 -------------------
46 files changed, 501 insertions(+), 1375 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
new file mode 100644
index 0000000..d253762
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
@@ -0,0 +1,96 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<FindBugsFilter>
+ <Match>
+ <Package name="org.apache.slider.api.proto" />
+ </Match>
+ <Match>
+ <class name="~org\.apache\.slider\.*" />
+ <Bug pattern="IS2_INCONSISTENT_SYNC" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.core.zk.BlockingZKWatcher" />
+ <Bug pattern="JLM_JSR166_UTILCONCURRENT_MONITORENTER" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.server.appmaster.state.ProviderAppState" />
+ <Bug pattern="JLM_JSR166_UTILCONCURRENT_MONITORENTER" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.providers.ProviderUtils" />
+ <Bug pattern="SWL_SLEEP_WITH_LOCK_HELD" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.server.appmaster.SliderAppMaster" />
+ <Bug pattern="WA_AWAIT_NOT_IN_LOOP" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.core.zk.ZKIntegration" />
+ <Bug pattern="SBSC_USE_STRINGBUFFER_CONCATENATION" />
+ </Match>
+ <Match>
+ <Bug pattern="NP_PARAMETER_MUST_BE_NONNULL_BUT_MARKED_AS_NULLABLE" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.server.servicemonitor.ProbeWorker" />
+ <Bug pattern="SF_SWITCH_FALLTHROUGH" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.core.persist.JsonSerDeser" />
+ <Bug pattern="UI_INHERITANCE_UNSAFE_GETRESOURCE" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.server.appmaster.rpc.SliderAMPolicyProvider" />
+ <Bug pattern="EI_EXPOSE_REP" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.server.appmaster.state.OutstandingRequest" />
+ <Bug pattern="EQ_DOESNT_OVERRIDE_EQUALS" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.server.appmaster.web.SliderAMWebApp" />
+ <Bug pattern="LG_LOST_LOGGER_DUE_TO_WEAK_REFERENCE" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.server.servicemonitor.Probe" />
+ <Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.server.services.workflow.ForkedProcessService" />
+ <Bug pattern="JLM_JSR166_UTILCONCURRENT_MONITORENTER" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.server.appmaster.state.RoleInstance"/>
+ <Bug pattern="UWF_UNWRITTEN_PUBLIC_OR_PROTECTED_FIELD"/>
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.client.SliderClient" />
+ <Method name="actionRegistryListConfigsYarn" />
+ <Bug pattern="OS_OPEN_STREAM" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.client.SliderClient" />
+ <Method name="actionRegistryListExports" />
+ <Bug pattern="OS_OPEN_STREAM" />
+ </Match>
+ <Match>
+ <Class name="org.apache.slider.common.tools.SliderUtils" />
+ <Method name="getApplicationResourceInputStream" />
+ <Bug pattern="OS_OPEN_STREAM" />
+ </Match>
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml
deleted file mode 100644
index b89146a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml
+++ /dev/null
@@ -1,20 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<FindBugsFilter>
-
-</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index 7453d12..81e607b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -26,6 +26,11 @@
<packaging>jar</packaging>
<name>Apache Hadoop YARN Slider Core</name>
+ <properties>
+ <!-- Needed for generating FindBugs warnings using parent pom -->
+ <yarn.basedir>${project.parent.basedir}</yarn.basedir>
+ </properties>
+
<build>
<!-- resources are filtered for dynamic updates. This gets build info in-->
<resources>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java
index 8c572b3..905d4b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java
@@ -35,6 +35,7 @@ import java.io.Closeable;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
+import java.io.PrintStream;
import java.io.PrintWriter;
import java.lang.reflect.InvocationTargetException;
import java.net.InetAddress;
@@ -82,7 +83,7 @@ public class KerberosDiags implements Closeable {
private final Configuration conf;
private final List<String> services;
- private final PrintWriter out;
+ private final PrintStream out;
private final File keytab;
private final String principal;
private final long minKeyLength;
@@ -97,7 +98,7 @@ public class KerberosDiags implements Closeable {
@SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
public KerberosDiags(Configuration conf,
- PrintWriter out,
+ PrintStream out,
List<String> services,
File keytab,
String principal,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
deleted file mode 100644
index ec35028..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
+++ /dev/null
@@ -1,283 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.proto;
-
-import org.apache.slider.api.types.ApplicationLivenessInformation;
-import org.apache.slider.api.types.ComponentInformation;
-import org.apache.slider.api.types.ContainerInformation;
-import org.apache.slider.api.types.NodeEntryInformation;
-import org.apache.slider.api.types.NodeInformation;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.ConfTree;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.persist.AggregateConfSerDeser;
-import org.apache.slider.core.persist.ConfTreeSerDeser;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Class to handle marshalling of REST
- * types to/from Protobuf records.
- */
-public class RestTypeMarshalling {
-
- public static Messages.ApplicationLivenessInformationProto
- marshall(ApplicationLivenessInformation info) {
-
- Messages.ApplicationLivenessInformationProto.Builder builder =
- Messages.ApplicationLivenessInformationProto.newBuilder();
- builder.setAllRequestsSatisfied(info.allRequestsSatisfied);
- builder.setRequestsOutstanding(info.requestsOutstanding);
- return builder.build();
- }
-
- public static ApplicationLivenessInformation
- unmarshall(Messages.ApplicationLivenessInformationProto wire) {
- ApplicationLivenessInformation info = new ApplicationLivenessInformation();
- info.allRequestsSatisfied = wire.getAllRequestsSatisfied();
- info.requestsOutstanding = wire.getRequestsOutstanding();
- return info;
- }
-
- public static ComponentInformation
- unmarshall(Messages.ComponentInformationProto wire) {
- ComponentInformation info = new ComponentInformation();
- info.name = wire.getName();
- info.priority = wire.getPriority();
- info.placementPolicy = wire.getPlacementPolicy();
-
- info.actual = wire.getActual();
- info.completed = wire.getCompleted();
- info.desired = wire.getDesired();
- info.failed = wire.getFailed();
- info.releasing = wire.getReleasing();
- info.requested = wire.getRequested();
- info.started = wire.getStarted();
- info.startFailed = wire.getStartFailed();
- info.totalRequested = wire.getTotalRequested();
- info.containers = new ArrayList<>(wire.getContainersList());
- if (wire.hasFailureMessage()) {
- info.failureMessage = wire.getFailureMessage();
- }
- if (wire.hasPendingAntiAffineRequestCount()) {
- info.pendingAntiAffineRequestCount = wire.getPendingAntiAffineRequestCount();
- }
- if (wire.hasIsAARequestOutstanding()) {
- info.isAARequestOutstanding = wire.getIsAARequestOutstanding();
- }
- return info;
- }
- public static Messages.ComponentInformationProto marshall(ComponentInformation info) {
-
- Messages.ComponentInformationProto.Builder builder =
- Messages.ComponentInformationProto.newBuilder();
- builder.setName(info.name);
- builder.setPriority(info.priority);
- builder.setPlacementPolicy(info.placementPolicy);
-
- builder.setActual(info.actual);
- builder.setCompleted(info.completed);
- builder.setDesired(info.desired);
- builder.setFailed(info.failed);
- builder.setReleasing(info.releasing);
- builder.setRequested(info.requested);
- builder.setStarted(info.started);
- builder.setStartFailed(info.startFailed);
- builder.setTotalRequested(info.totalRequested);
- builder.setNodeFailed(info.nodeFailed);
- builder.setPreempted(info.preempted);
- builder.setFailedRecently(info.failedRecently);
- if (info.failureMessage != null) {
- builder.setFailureMessage(info.failureMessage);
- }
- if (info.containers != null) {
- builder.addAllContainers(info.containers);
- }
- builder.setPendingAntiAffineRequestCount(info.pendingAntiAffineRequestCount);
- builder.setIsAARequestOutstanding(info.isAARequestOutstanding);
- return builder.build();
- }
-
- public static Messages.NodeInformationProto marshall(NodeInformation info) {
-
- Messages.NodeInformationProto.Builder builder =
- Messages.NodeInformationProto.newBuilder();
- builder.setHostname(info.hostname);
- builder.setLastUpdated(info.lastUpdated);
- builder.setState(info.state != null? info.state : "unknown");
- builder.setRackName(info.rackName != null ? info.rackName : "");
- builder.setHealthReport(info.healthReport != null ? info.healthReport : "");
- builder.setHttpAddress(info.httpAddress != null ? info.httpAddress : "");
- builder.setLabels(info.labels != null ? info.labels: "");
-
-
- if (info.entries != null) {
- for (Map.Entry<String, NodeEntryInformation> elt : info.entries.entrySet()) {
- NodeEntryInformation entry = elt.getValue();
- Messages.NodeEntryInformationProto.Builder node =
- Messages.NodeEntryInformationProto.newBuilder();
- node.setPriority(entry.priority);
- node.setName(elt.getKey());
- node.setFailed(entry.failed);
- node.setFailedRecently(entry.failedRecently);
- node.setLive(entry.live);
- node.setLastUsed(entry.lastUsed);
- node.setPreempted(entry.preempted);
- node.setRequested(entry.requested);
- node.setReleasing(entry.releasing);
- node.setStartFailed(entry.startFailed);
- node.setStarting(entry.starting);
- builder.addEntries(node.build());
- }
- }
- return builder.build();
- }
-
- public static NodeInformation unmarshall(Messages.NodeInformationProto wire) {
- NodeInformation info = new NodeInformation();
- info.healthReport = wire.getHealthReport();
- info.hostname = wire.getHostname();
- info.httpAddress = wire.getHttpAddress();
- info.labels = wire.getLabels();
- info.lastUpdated = wire.getLastUpdated();
- info.rackName = wire.getRackName();
- info.state = wire.getState();
- List<Messages.NodeEntryInformationProto> entriesList = wire.getEntriesList();
- if (entriesList != null) {
- info.entries = new HashMap<>(entriesList.size());
- for (Messages.NodeEntryInformationProto entry : entriesList) {
- NodeEntryInformation nei = new NodeEntryInformation();
- nei.failed = entry.getFailed();
- nei.failedRecently = entry.getFailedRecently();
- nei.lastUsed = entry.getLastUsed();
- nei.live = entry.getLive();
- nei.preempted = entry.getPreempted();
- nei.priority = entry.getPriority();
- nei.requested = entry.getRequested();
- nei.releasing = entry.getReleasing();
- nei.startFailed = entry.getStartFailed();
- nei.starting = entry.getStarting();
- info.entries.put(entry.getName(), nei);
- }
- }
- return info;
- }
-
- public static ContainerInformation unmarshall(Messages.ContainerInformationProto wire) {
- ContainerInformation info = new ContainerInformation();
- info.containerId = wire.getContainerId();
- info.component = wire.getComponent();
- info.appVersion = wire.getAppVersion();
- info.state = wire.getState();
- if (wire.hasReleased()) {
- info.released = wire.getReleased();
- }
- if (wire.hasExitCode()) {
- info.exitCode = wire.getExitCode();
- }
- if (wire.hasDiagnostics()) {
- info.diagnostics = wire.getDiagnostics();
- }
- if (wire.hasHost()) {
- info.host = wire.getHost();
- }
- if (wire.hasHostURL()) {
- info.host = wire.getHostURL();
- }
- info.createTime = wire.getCreateTime();
- info.startTime = wire.getStartTime();
- info.output = wire.getOutputList().toArray(
- new String[wire.getOutputCount()]
- );
- if (wire.hasPlacement()) {
- info.placement = wire.getPlacement();
- }
- return info;
- }
-
- public static List<ContainerInformation> unmarshall(Messages.GetLiveContainersResponseProto wire) {
- List<ContainerInformation> infoList = new ArrayList<>(wire.getContainersList().size());
- for (Messages.ContainerInformationProto container : wire.getContainersList()) {
- infoList.add(unmarshall(container));
- }
- return infoList;
- }
-
- public static Messages.ContainerInformationProto marshall(ContainerInformation info) {
-
- Messages.ContainerInformationProto.Builder builder =
- Messages.ContainerInformationProto.newBuilder();
- if (info.containerId != null) {
- builder.setContainerId(info.containerId);
- }
- if (info.component != null) {
- builder.setComponent(info.component);
- }
- if (info.appVersion != null) {
- builder.setAppVersion(info.appVersion);
- }
- builder.setCreateTime(info.createTime);
- if (info.diagnostics != null) {
- builder.setDiagnostics(info.diagnostics);
- }
- if (info.host != null) {
- builder.setHost(info.host);
- }
- if (info.hostURL != null) {
- builder.setHostURL(info.hostURL);
- }
- if (info.output != null) {
- builder.addAllOutput(Arrays.asList(info.output));
- }
- if (info.released != null) {
- builder.setReleased(info.released);
- }
- if (info.placement != null) {
- builder.setPlacement(info.placement);
- }
- builder.setStartTime(info.startTime);
- builder.setState(info.state);
- return builder.build();
- }
-
- public static String unmarshall(Messages.WrappedJsonProto wire) {
- return wire.getJson();
- }
-
- public static ConfTree unmarshallToConfTree(Messages.WrappedJsonProto wire) throws
- IOException {
- return new ConfTreeSerDeser().fromJson(wire.getJson());
- }
-
- public static ConfTreeOperations unmarshallToCTO(Messages.WrappedJsonProto wire) throws
- IOException {
- return new ConfTreeOperations(new ConfTreeSerDeser().fromJson(wire.getJson()));
- }
-
- public static AggregateConf unmarshallToAggregateConf(Messages.WrappedJsonProto wire) throws
- IOException {
- return new AggregateConfSerDeser().fromJson(wire.getJson());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
index c46a59f..d2fdd62 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
@@ -31,7 +31,7 @@ import java.util.Map;
* Serializable version of component data.
* <p>
* This is sent in REST calls as a JSON object \u2014but is also marshalled into
- * a protobuf structure. Look at {@link org.apache.slider.api.proto.RestTypeMarshalling}
+ * a protobuf structure. Look at {@link RestTypeMarshalling}
* for the specifics there.
* <p>
* This means that if any fields are added here. they must be added to
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RestTypeMarshalling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RestTypeMarshalling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RestTypeMarshalling.java
new file mode 100644
index 0000000..713cffd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/RestTypeMarshalling.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api.types;
+
+import org.apache.slider.api.proto.Messages;
+import org.apache.slider.api.types.ApplicationLivenessInformation;
+import org.apache.slider.api.types.ComponentInformation;
+import org.apache.slider.api.types.ContainerInformation;
+import org.apache.slider.api.types.NodeEntryInformation;
+import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.persist.AggregateConfSerDeser;
+import org.apache.slider.core.persist.ConfTreeSerDeser;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Class to handle marshalling of REST
+ * types to/from Protobuf records.
+ */
+public class RestTypeMarshalling {
+
+ public static Messages.ApplicationLivenessInformationProto
+ marshall(ApplicationLivenessInformation info) {
+
+ Messages.ApplicationLivenessInformationProto.Builder builder =
+ Messages.ApplicationLivenessInformationProto.newBuilder();
+ builder.setAllRequestsSatisfied(info.allRequestsSatisfied);
+ builder.setRequestsOutstanding(info.requestsOutstanding);
+ return builder.build();
+ }
+
+ public static ApplicationLivenessInformation
+ unmarshall(Messages.ApplicationLivenessInformationProto wire) {
+ ApplicationLivenessInformation info = new ApplicationLivenessInformation();
+ info.allRequestsSatisfied = wire.getAllRequestsSatisfied();
+ info.requestsOutstanding = wire.getRequestsOutstanding();
+ return info;
+ }
+
+ public static ComponentInformation
+ unmarshall(Messages.ComponentInformationProto wire) {
+ ComponentInformation info = new ComponentInformation();
+ info.name = wire.getName();
+ info.priority = wire.getPriority();
+ info.placementPolicy = wire.getPlacementPolicy();
+
+ info.actual = wire.getActual();
+ info.completed = wire.getCompleted();
+ info.desired = wire.getDesired();
+ info.failed = wire.getFailed();
+ info.releasing = wire.getReleasing();
+ info.requested = wire.getRequested();
+ info.started = wire.getStarted();
+ info.startFailed = wire.getStartFailed();
+ info.totalRequested = wire.getTotalRequested();
+ info.containers = new ArrayList<>(wire.getContainersList());
+ if (wire.hasFailureMessage()) {
+ info.failureMessage = wire.getFailureMessage();
+ }
+ if (wire.hasPendingAntiAffineRequestCount()) {
+ info.pendingAntiAffineRequestCount = wire.getPendingAntiAffineRequestCount();
+ }
+ if (wire.hasIsAARequestOutstanding()) {
+ info.isAARequestOutstanding = wire.getIsAARequestOutstanding();
+ }
+ return info;
+ }
+ public static Messages.ComponentInformationProto marshall(ComponentInformation info) {
+
+ Messages.ComponentInformationProto.Builder builder =
+ Messages.ComponentInformationProto.newBuilder();
+ builder.setName(info.name);
+ builder.setPriority(info.priority);
+ builder.setPlacementPolicy(info.placementPolicy);
+
+ builder.setActual(info.actual);
+ builder.setCompleted(info.completed);
+ builder.setDesired(info.desired);
+ builder.setFailed(info.failed);
+ builder.setReleasing(info.releasing);
+ builder.setRequested(info.requested);
+ builder.setStarted(info.started);
+ builder.setStartFailed(info.startFailed);
+ builder.setTotalRequested(info.totalRequested);
+ builder.setNodeFailed(info.nodeFailed);
+ builder.setPreempted(info.preempted);
+ builder.setFailedRecently(info.failedRecently);
+ if (info.failureMessage != null) {
+ builder.setFailureMessage(info.failureMessage);
+ }
+ if (info.containers != null) {
+ builder.addAllContainers(info.containers);
+ }
+ builder.setPendingAntiAffineRequestCount(info.pendingAntiAffineRequestCount);
+ builder.setIsAARequestOutstanding(info.isAARequestOutstanding);
+ return builder.build();
+ }
+
+ public static Messages.NodeInformationProto marshall(NodeInformation info) {
+
+ Messages.NodeInformationProto.Builder builder =
+ Messages.NodeInformationProto.newBuilder();
+ builder.setHostname(info.hostname);
+ builder.setLastUpdated(info.lastUpdated);
+ builder.setState(info.state != null? info.state : "unknown");
+ builder.setRackName(info.rackName != null ? info.rackName : "");
+ builder.setHealthReport(info.healthReport != null ? info.healthReport : "");
+ builder.setHttpAddress(info.httpAddress != null ? info.httpAddress : "");
+ builder.setLabels(info.labels != null ? info.labels: "");
+
+
+ if (info.entries != null) {
+ for (Map.Entry<String, NodeEntryInformation> elt : info.entries.entrySet()) {
+ NodeEntryInformation entry = elt.getValue();
+ Messages.NodeEntryInformationProto.Builder node =
+ Messages.NodeEntryInformationProto.newBuilder();
+ node.setPriority(entry.priority);
+ node.setName(elt.getKey());
+ node.setFailed(entry.failed);
+ node.setFailedRecently(entry.failedRecently);
+ node.setLive(entry.live);
+ node.setLastUsed(entry.lastUsed);
+ node.setPreempted(entry.preempted);
+ node.setRequested(entry.requested);
+ node.setReleasing(entry.releasing);
+ node.setStartFailed(entry.startFailed);
+ node.setStarting(entry.starting);
+ builder.addEntries(node.build());
+ }
+ }
+ return builder.build();
+ }
+
+ public static NodeInformation unmarshall(Messages.NodeInformationProto wire) {
+ NodeInformation info = new NodeInformation();
+ info.healthReport = wire.getHealthReport();
+ info.hostname = wire.getHostname();
+ info.httpAddress = wire.getHttpAddress();
+ info.labels = wire.getLabels();
+ info.lastUpdated = wire.getLastUpdated();
+ info.rackName = wire.getRackName();
+ info.state = wire.getState();
+ List<Messages.NodeEntryInformationProto> entriesList = wire.getEntriesList();
+ if (entriesList != null) {
+ info.entries = new HashMap<>(entriesList.size());
+ for (Messages.NodeEntryInformationProto entry : entriesList) {
+ NodeEntryInformation nei = new NodeEntryInformation();
+ nei.failed = entry.getFailed();
+ nei.failedRecently = entry.getFailedRecently();
+ nei.lastUsed = entry.getLastUsed();
+ nei.live = entry.getLive();
+ nei.preempted = entry.getPreempted();
+ nei.priority = entry.getPriority();
+ nei.requested = entry.getRequested();
+ nei.releasing = entry.getReleasing();
+ nei.startFailed = entry.getStartFailed();
+ nei.starting = entry.getStarting();
+ info.entries.put(entry.getName(), nei);
+ }
+ }
+ return info;
+ }
+
+ public static ContainerInformation unmarshall(Messages.ContainerInformationProto wire) {
+ ContainerInformation info = new ContainerInformation();
+ info.containerId = wire.getContainerId();
+ info.component = wire.getComponent();
+ info.appVersion = wire.getAppVersion();
+ info.state = wire.getState();
+ if (wire.hasReleased()) {
+ info.released = wire.getReleased();
+ }
+ if (wire.hasExitCode()) {
+ info.exitCode = wire.getExitCode();
+ }
+ if (wire.hasDiagnostics()) {
+ info.diagnostics = wire.getDiagnostics();
+ }
+ if (wire.hasHost()) {
+ info.host = wire.getHost();
+ }
+ if (wire.hasHostURL()) {
+ info.host = wire.getHostURL();
+ }
+ info.createTime = wire.getCreateTime();
+ info.startTime = wire.getStartTime();
+ info.output = wire.getOutputList().toArray(
+ new String[wire.getOutputCount()]
+ );
+ if (wire.hasPlacement()) {
+ info.placement = wire.getPlacement();
+ }
+ return info;
+ }
+
+ public static List<ContainerInformation> unmarshall(Messages.GetLiveContainersResponseProto wire) {
+ List<ContainerInformation> infoList = new ArrayList<>(wire.getContainersList().size());
+ for (Messages.ContainerInformationProto container : wire.getContainersList()) {
+ infoList.add(unmarshall(container));
+ }
+ return infoList;
+ }
+
+ public static Messages.ContainerInformationProto marshall(ContainerInformation info) {
+
+ Messages.ContainerInformationProto.Builder builder =
+ Messages.ContainerInformationProto.newBuilder();
+ if (info.containerId != null) {
+ builder.setContainerId(info.containerId);
+ }
+ if (info.component != null) {
+ builder.setComponent(info.component);
+ }
+ if (info.appVersion != null) {
+ builder.setAppVersion(info.appVersion);
+ }
+ builder.setCreateTime(info.createTime);
+ if (info.diagnostics != null) {
+ builder.setDiagnostics(info.diagnostics);
+ }
+ if (info.host != null) {
+ builder.setHost(info.host);
+ }
+ if (info.hostURL != null) {
+ builder.setHostURL(info.hostURL);
+ }
+ if (info.output != null) {
+ builder.addAllOutput(Arrays.asList(info.output));
+ }
+ if (info.released != null) {
+ builder.setReleased(info.released);
+ }
+ if (info.placement != null) {
+ builder.setPlacement(info.placement);
+ }
+ builder.setStartTime(info.startTime);
+ builder.setState(info.state);
+ return builder.build();
+ }
+
+ public static String unmarshall(Messages.WrappedJsonProto wire) {
+ return wire.getJson();
+ }
+
+ public static ConfTree unmarshallToConfTree(Messages.WrappedJsonProto wire) throws
+ IOException {
+ return new ConfTreeSerDeser().fromJson(wire.getJson());
+ }
+
+ public static ConfTreeOperations unmarshallToCTO(Messages.WrappedJsonProto wire) throws
+ IOException {
+ return new ConfTreeOperations(new ConfTreeSerDeser().fromJson(wire.getJson()));
+ }
+
+ public static AggregateConf unmarshallToAggregateConf(Messages.WrappedJsonProto wire) throws
+ IOException {
+ return new AggregateConfSerDeser().fromJson(wire.getJson());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 1c126ac..1a959d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -192,10 +192,10 @@ import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.io.Writer;
-import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -1322,9 +1322,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
private void initializeOutputStream(String outFile)
- throws FileNotFoundException {
+ throws IOException {
if (outFile != null) {
- clientOutputStream = new PrintStream(new FileOutputStream(outFile));
+ clientOutputStream = new PrintStream(outFile, "UTF-8");
} else {
clientOutputStream = System.out;
}
@@ -3299,7 +3299,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
int updateCount = Integer.parseInt(updateCountStr);
// if component was specified before, get the current count
if (component.get(COMPONENT_INSTANCES) != null) {
- currentCount = Integer.valueOf(component.get(COMPONENT_INSTANCES));
+ currentCount = Integer.parseInt(component.get(COMPONENT_INSTANCES));
if (currentCount + updateCount < 0) {
throw new BadCommandArgumentsException("The requested count " +
"of \"%s\" for role %s makes the total number of " +
@@ -3610,16 +3610,16 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// as this is an API entry point, validate
// the arguments
args.validate();
- RegistryOperations operations = getRegistryOperations();
String path = SliderRegistryUtils.resolvePath(args.path);
ServiceRecordMarshal serviceRecordMarshal = new ServiceRecordMarshal();
try {
if (args.list) {
File destDir = args.destdir;
- if (destDir != null) {
- destDir.mkdirs();
+ if (destDir != null && !destDir.exists() && !destDir.mkdirs()) {
+ throw new IOException("Failed to create directory: " + destDir);
}
+
Map<String, ServiceRecord> recordMap;
Map<String, RegistryPathStatus> znodes;
try {
@@ -3656,9 +3656,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
} else {
String filename = RegistryPathUtils.lastPathEntry(name) + ".json";
File jsonFile = new File(destDir, filename);
- write(jsonFile,
- serviceRecordMarshal.toBytes(instance),
- true);
+ write(jsonFile, serviceRecordMarshal.toBytes(instance));
}
}
} else {
@@ -3669,7 +3667,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
outFile = new File(args.destdir, RegistryPathUtils.lastPathEntry(path));
}
if (outFile != null) {
- write(outFile, serviceRecordMarshal.toBytes(instance), true);
+ write(outFile, serviceRecordMarshal.toBytes(instance));
} else {
println(serviceRecordMarshal.toJson(instance));
}
@@ -4062,11 +4060,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
@SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
private int actionKDiag(ActionKDiagArgs args)
throws Exception {
- PrintWriter out = new PrintWriter(System.err);
+ PrintStream out;
boolean closeStream = false;
if (args.out != null) {
- out = new PrintWriter(new FileOutputStream(args.out));
+ out = new PrintStream(args.out, "UTF-8");
closeStream = true;
+ } else {
+ out = System.err;
}
try {
KerberosDiags kdiags = new KerberosDiags(getConfig(),
@@ -4137,7 +4137,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
PrintStream out = null;
try {
if (registryArgs.out != null) {
- out = new PrintStream(new FileOutputStream(registryArgs.out));
+ out = new PrintStream(registryArgs.out, "UTF-8");
} else {
out = System.out;
}
@@ -4145,11 +4145,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
if (!registryArgs.verbose) {
out.println(configName);
} else {
- PublishedConfiguration published =
- configurations.get(configName);
- out.printf("%s: %s\n",
- configName,
- published.description);
+ PublishedConfiguration published = configurations.get(configName);
+ out.printf("%s: %s%n", configName, published.description);
}
}
} finally {
@@ -4178,7 +4175,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
boolean streaming = false;
try {
if (registryArgs.out != null) {
- out = new PrintStream(new FileOutputStream(registryArgs.out));
+ out = new PrintStream(registryArgs.out, "UTF-8");
streaming = true;
log.debug("Saving output to {}", registryArgs.out);
} else {
@@ -4193,9 +4190,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
out.println(exportName);
} else {
PublishedExports published = exports.get(exportName);
- out.printf("%s: %s\n",
- exportName,
- published.description);
+ out.printf("%s: %s%n", exportName, published.description);
}
}
} finally {
@@ -4401,9 +4396,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* Output to standard out/stderr (implementation specific detail)
* @param src source
*/
- @SuppressWarnings("UseOfSystemOutOrSystemErr")
private static void print(CharSequence src) {
- clientOutputStream.append(src);
+ clientOutputStream.print(src);
}
/**
@@ -4411,8 +4405,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @param message message
*/
private static void println(String message) {
- print(message);
- print("\n");
+ clientOutputStream.println(message);
}
/**
* Output to standard out/stderr with a newline after, formatted
@@ -4420,8 +4413,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @param args arguments for string formatting
*/
private static void println(String message, Object ... args) {
- print(String.format(message, args));
- print("\n");
+ clientOutputStream.println(String.format(message, args));
}
/**
@@ -4497,12 +4489,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
throw new UsageException(CommonArgs.usage(serviceArgs, actionName));
}
- private int actionHelp(String errMsg, String actionName)
- throws YarnException, IOException {
- throw new UsageException("%s %s", errMsg, CommonArgs.usage(serviceArgs,
- actionName));
- }
-
/**
* List the nodes in the cluster, possibly filtering by node state or label.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/TokensOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/TokensOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/TokensOperation.java
index 9b9c141..84c65b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/TokensOperation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/TokensOperation.java
@@ -75,11 +75,10 @@ public class TokensOperation {
}
saveTokens(output, credentials);
String filename = output.getCanonicalPath();
- footnote = String.format("%d tokens saved to %s\n" +
- "To use these in the environment:\n" +
- "export %s=%s",
- credentials.numberOfTokens(),
- filename, UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION, filename);
+ footnote = String.format(
+ "%d tokens saved to %s%n" + "To use these in the environment:%n"
+ + "export %s=%s", credentials.numberOfTokens(), filename,
+ UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION, filename);
} else if (args.source != null) {
File source = args.source;
log.info("Reading credentials from file {}", source);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
index eaf15e6..702233a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
@@ -51,7 +51,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import static org.apache.slider.api.proto.RestTypeMarshalling.*;
+import static org.apache.slider.api.types.RestTypeMarshalling.*;
/**
* Cluster operations at a slightly higher level than the RPC code
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
index 4283ee8..573ef64 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
@@ -297,20 +297,6 @@ public class SliderApplicationApiRestClient extends BaseRestClient
.type(MediaType.APPLICATION_JSON_TYPE)
.post(PingInformation.class, f);
}
-
- /**
- * Ping as a POST
- * @param text text to include
- * @return the response
- * @throws IOException on any failure
- */
- public PingInformation pingPut(String text) throws IOException {
- WebResource pingResource = applicationResource(ACTION_PING);
- Form f = new Form();
- return pingResource
- .type(MediaType.TEXT_PLAIN)
- .put(PingInformation.class, text);
- }
@Override
public void stop(String text) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index adf40ce..4bf1b5b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -18,6 +18,9 @@
package org.apache.slider.common;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
/**
* Keys and various constants for Slider
@@ -79,8 +82,9 @@ public interface SliderKeys extends SliderXmlConfKeys {
*/
String COMPONENT_TYPE_EXTERNAL_APP = "external_app";
String COMPONENT_SEPARATOR = "-";
- String[] COMPONENT_KEYS_TO_SKIP = {"zookeeper.", "env.MALLOC_ARENA_MAX",
- "site.fs.", "site.dfs."};
+ List<String> COMPONENT_KEYS_TO_SKIP = Collections.unmodifiableList(Arrays
+ .asList("zookeeper.", "env.MALLOC_ARENA_MAX", "site.fs.", "site.dfs."));
+
/**
* A component type for a client component
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
index e3cb288..63ccff8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
@@ -142,11 +142,12 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
log.error(message);
int index = 1;
+ StringBuilder buf = new StringBuilder(message);
for (String actionArg : parameters) {
log.error("[{}] \"{}\"", index++, actionArg);
- message += " \"" + actionArg + "\" ";
+ buf.append(" \"" + actionArg + "\" ");
}
- throw new BadCommandArgumentsException(message);
+ throw new BadCommandArgumentsException(buf.toString());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
index 20a9989..1d28c78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
@@ -214,7 +214,6 @@ public abstract class AbstractClusterBuildingActionArgs extends
protected ConfTree buildConfTree(Map<String, String> optionsMap) throws
BadCommandArgumentsException {
ConfTree confTree = new ConfTree();
- ConfTreeOperations ops = new ConfTreeOperations(confTree);
confTree.global.putAll(optionsMap);
return confTree;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
index 162a87d..9c8e65d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
@@ -154,12 +154,12 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
public void parse() throws SliderException {
addActionArguments();
try {
- commander.parse(getArgs());
+ commander.parse(args);
} catch (ParameterException e) {
throw new BadCommandArgumentsException(e, "%s in %s",
e.toString(),
- (getArgs() != null
- ? (SliderUtils.join(getArgs(),
+ (args != null
+ ? (SliderUtils.join(args,
" ", false))
: "[]"));
}
@@ -297,7 +297,4 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
return coreAction.parameters;
}
- public String[] getArgs() {
- return args;
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
index 6380d0c..a83901b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
@@ -29,25 +29,18 @@ public class Comparators {
public static class LongComparator implements Comparator<Long>, Serializable {
@Override
public int compare(Long o1, Long o2) {
- long result = o1 - o2;
- // need to comparisons with a diff greater than integer size
- if (result < 0 ) {
- return -1;
- } else if (result > 0) {
- return 1;
- }
- return 0;
+ return o1.compareTo(o2);
}
}
-public static class InvertedLongComparator implements Comparator<Long>, Serializable {
- private static final LongComparator inner = new LongComparator();
+
+ public static class InvertedLongComparator
+ implements Comparator<Long>, Serializable {
@Override
public int compare(Long o1, Long o2) {
- return -inner.compare(o1, o2);
+ return o2.compareTo(o1);
}
}
-
/**
* Little template class to reverse any comparitor
* @param <CompareType> the type that is being compared
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
index 9db241d..02eba49 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
@@ -194,7 +194,7 @@ public class ConfigHelper {
byte[] data = loadBytes(fs, path);
//this is here to track down a parse issue
//related to configurations
- String s = new String(data, 0, data.length);
+ String s = new String(data, 0, data.length, "UTF-8");
log.debug("XML resource {} is \"{}\"", path, s);
/* JDK7
try (ByteArrayInputStream in = new ByteArrayInputStream(data)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index aa5edf1..c3d6d98 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -464,37 +464,6 @@ public class CoreFileSystem {
}
/**
- * Verify that a file exists in the zip file given by path
- * @param path path to zip file
- * @param file file expected to be in zip
- * @throws FileNotFoundException file not found or is not a zip file
- * @throws IOException trouble with FS
- */
- public void verifyFileExistsInZip(Path path, String file) throws IOException {
- fileSystem.copyToLocalFile(path, new Path("/tmp"));
- File dst = new File((new Path("/tmp", path.getName())).toString());
- Enumeration<? extends ZipEntry> entries;
- ZipFile zipFile = new ZipFile(dst);
- boolean found = false;
-
- try {
- entries = zipFile.entries();
- while (entries.hasMoreElements()) {
- ZipEntry entry = entries.nextElement();
- String nm = entry.getName();
- if (nm.endsWith(file)) {
- found = true;
- break;
- }
- }
- } finally {
- zipFile.close();
- }
- dst.delete();
- if (!found) throw new FileNotFoundException("file: " + file + " not found in " + path);
- log.info("Verification of " + path + " passed");
- }
- /**
* Create the application-instance specific temporary directory
* in the DFS
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 4457957..a107dfb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.GlobFilter;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
@@ -310,10 +309,6 @@ public final class SliderUtils {
}
String class_file = my_class.getName().replaceAll("\\.", "/") + ".class";
Enumeration<URL> urlEnumeration = loader.getResources(class_file);
- if (urlEnumeration == null) {
- throw new IOException("Unable to find resources for class " + my_class);
- }
-
for (; urlEnumeration.hasMoreElements(); ) {
URL url = urlEnumeration.nextElement();
if ("jar".equals(url.getProtocol())) {
@@ -756,10 +751,10 @@ public final class SliderUtils {
public static String containersToString(
List<ContainerInformation> containers, String version,
Set<String> components) {
- String containerf = " %-28s %30s %45s %s\n";
+ String containerf = " %-28s %30s %45s %s%n";
StringBuilder builder = new StringBuilder(512);
- builder.append("Containers:\n");
- builder.append(String.format(" %-28s %30s %45s %s\n", "Component Name",
+ builder.append("Containers:%n");
+ builder.append(String.format(" %-28s %30s %45s %s%n", "Component Name",
"App Version", "Container Id", "Container Info/Logs"));
for (ContainerInformation container : containers) {
if (filter(container.appVersion, version)
@@ -969,7 +964,7 @@ public final class SliderUtils {
*/
public static Map<String, String> mergeMapsIgnoreDuplicateKeysAndPrefixes(
Map<String, String> first, Map<String, String> second,
- String... prefixes) {
+ List<String> prefixes) {
Preconditions.checkArgument(first != null, "Null 'first' value");
Preconditions.checkArgument(second != null, "Null 'second' value");
Preconditions.checkArgument(prefixes != null, "Null 'prefixes' value");
@@ -2119,15 +2114,16 @@ public final class SliderUtils {
is = new ByteArrayInputStream(content);
} else {
log.debug("Size unknown. Reading {}", zipEntry.getName());
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- while (true) {
- int byteRead = zis.read();
- if (byteRead == -1) {
- break;
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
+ while (true) {
+ int byteRead = zis.read();
+ if (byteRead == -1) {
+ break;
+ }
+ baos.write(byteRead);
}
- baos.write(byteRead);
+ is = new ByteArrayInputStream(baos.toByteArray());
}
- is = new ByteArrayInputStream(baos.toByteArray());
}
done = true;
}
@@ -2205,90 +2201,26 @@ public final class SliderUtils {
}
/**
- * Look for the windows executable and check it has the right headers.
- * <code>File.canRead()</code> doesn't work on windows, so the reading
- * is mandatory.
- *
- * @param program program name for errors
- * @param exe executable
- * @throws IOException IOE
- */
- public static void verifyWindowsExe(String program, File exe)
- throws IOException {
- verifyIsFile(program, exe);
-
- verifyFileSize(program, exe, 0x100);
-
- // now read two bytes and verify the header.
- try(FileReader reader = new FileReader(exe)) {
- int[] header = new int[2];
- header[0] = reader.read();
- header[1] = reader.read();
- if ((header[0] != 'M' || header[1] != 'Z')) {
- throw new FileNotFoundException(program
- + " at " + exe
- + " is not a windows executable file");
- }
- }
- }
-
- /**
- * Verify that a Unix exe works
- * @param program program name for errors
- * @param exe executable
- * @throws IOException IOE
-
- */
- public static void verifyUnixExe(String program, File exe)
- throws IOException {
- verifyIsFile(program, exe);
-
- // read flag
- if (!exe.canRead()) {
- throw new IOException("Cannot read " + program + " at " + exe);
- }
- // exe flag
- if (!exe.canExecute()) {
- throw new IOException("Cannot execute " + program + " at " + exe);
- }
- }
-
- /**
- * Validate an executable
- * @param program program name for errors
- * @param exe program to look at
- * @throws IOException
- */
- public static void validateExe(String program, File exe) throws IOException {
- if (!Shell.WINDOWS) {
- verifyWindowsExe(program, exe);
- } else {
- verifyUnixExe(program, exe);
- }
- }
-
- /**
* Write bytes to a file
* @param outfile output file
* @param data data to write
- * @param createParent flag to indicate that the parent dir should
- * be created
* @throws IOException on any IO problem
*/
- public static void write(File outfile, byte[] data, boolean createParent)
+ public static void write(File outfile, byte[] data)
throws IOException {
File parentDir = outfile.getCanonicalFile().getParentFile();
if (parentDir == null) {
throw new IOException(outfile.getPath() + " has no parent dir");
}
- if (createParent) {
- parentDir.mkdirs();
+ if (!parentDir.exists()) {
+ if(!parentDir.mkdirs()) {
+ throw new IOException("Failed to create parent directory " + parentDir);
+ }
}
SliderUtils.verifyIsDir(parentDir, log);
try(FileOutputStream out = new FileOutputStream(outfile)) {
out.write(data);
}
-
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java
index f0686af..f14a07a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java
@@ -142,7 +142,6 @@ public class InstanceBuilder {
md.put(StatusKeys.INFO_CREATE_TIME_HUMAN, SliderUtils.toGMTString(time));
md.put(StatusKeys.INFO_CREATE_TIME_MILLIS, Long.toString(time));
- MapOperations globalOptions = internalOps.getGlobalOptions();
BuildHelper.addBuildMetadata(md, "create");
SliderUtils.setInfoTime(md,
StatusKeys.INFO_CREATE_TIME_HUMAN,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
index 18c3156..a272420 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/AggregateConf.java
@@ -163,7 +163,7 @@ public final class AggregateConf {
public String getPassphrase() {
if (passphrase == null) {
passphrase = RandomStringUtils.randomAlphanumeric(
- Integer.valueOf(SliderKeys.PASS_LEN));
+ Integer.parseInt(SliderKeys.PASS_LEN));
}
return passphrase;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
index be7c56f..74b6abb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/ConfTree.java
@@ -79,17 +79,6 @@ public final class ConfTree {
public Map<String, Map<String, String>> components =
new HashMap<>(INITAL_MAP_CAPACITY);
-
- /**
- * Shallow clone
- * @return a shallow clone
- * @throws CloneNotSupportedException
- */
- @Override
- public Object clone() throws CloneNotSupportedException {
- return super.clone();
- }
-
@Override
public String toString() {
try {
@@ -105,9 +94,7 @@ public final class ConfTree {
* @return a JSON string description
* @throws IOException Problems mapping/writing the object
*/
- public String toJson() throws IOException,
- JsonGenerationException,
- JsonMappingException {
+ public String toJson() throws IOException {
return ConfTreeSerDeser.toString(this);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
index 7fb3158..9eb7d5c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
@@ -147,7 +147,9 @@ public class AppDefinitionPersister {
File tempDir = Files.createTempDir();
File pkgSrcDir = new File(tempDir, "default");
- pkgSrcDir.mkdirs();
+ if (!pkgSrcDir.exists() && !pkgSrcDir.mkdirs()) {
+ throw new IOException("Failed to create directory: " + pkgSrcDir);
+ }
File destMetaInfo = new File(pkgSrcDir, "metainfo.json");
if (isFileUsed) {
if (buildInfo.appMetaInfo.getName().endsWith(".xml")) {
@@ -194,12 +196,13 @@ public class AppDefinitionPersister {
List<String> addons = new ArrayList<String>();
Map<String, String> addonMap = buildInfo.addonDelegate.getAddonMap();
- for (String key : addonMap.keySet()) {
- File defPath = new File(addonMap.get(key));
- if (SliderUtils.isUnset(addonMap.get(key))) {
+ for (Map.Entry<String, String > entry : addonMap.entrySet()) {
+ String key = entry.getKey();
+ String value = entry.getValue();
+ if (SliderUtils.isUnset(value)) {
throw new BadConfigException("Invalid path for addon package " + key);
}
-
+ File defPath = new File(value);
if (!defPath.exists()) {
throw new BadConfigException("addon folder or package path is not valid.");
}
@@ -234,7 +237,7 @@ public class AppDefinitionPersister {
}
// Helper class to hold details for the app and addon packages
- public class AppDefinition {
+ static class AppDefinition {
// The target folder where the package will be stored
public Path targetFolderInFs;
// The on disk location of the app def package or folder
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
index 40fa217..e204178 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/retrieve/AMWebClient.java
@@ -62,61 +62,10 @@ public class AMWebClient {
restClient = new BaseRestClient(binding.createJerseyClient());
}
-
-
- private static URLConnectionClientHandler getUrlConnectionClientHandler() {
- return new URLConnectionClientHandler(new HttpURLConnectionFactory() {
- @Override
- public HttpURLConnection getHttpURLConnection(URL url)
- throws IOException {
- HttpURLConnection connection = (HttpURLConnection) url.openConnection();
- if (connection.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP) {
- // is a redirect - are we changing schemes?
- String redirectLocation = connection.getHeaderField(HttpHeaders.LOCATION);
- String originalScheme = url.getProtocol();
- String redirectScheme = URI.create(redirectLocation).getScheme();
- if (!originalScheme.equals(redirectScheme)) {
- // need to fake it out by doing redirect ourselves
- log.info("Protocol change during redirect. Redirecting {} to URL {}",
- url, redirectLocation);
- URL redirectURL = new URL(redirectLocation);
- connection = (HttpURLConnection) redirectURL.openConnection();
- }
- }
- if (connection instanceof HttpsURLConnection) {
- log.debug("Attempting to configure HTTPS connection using client "
- + "configuration");
- final SSLFactory factory;
- final SSLSocketFactory sf;
- final HostnameVerifier hv;
-
- try {
- HttpsURLConnection c = (HttpsURLConnection) connection;
- factory = new SSLFactory(SSLFactory.Mode.CLIENT, new Configuration());
- factory.init();
- sf = factory.createSSLSocketFactory();
- hv = factory.getHostnameVerifier();
- c.setSSLSocketFactory(sf);
- c.setHostnameVerifier(hv);
- } catch (Exception e) {
- log.info("Unable to configure HTTPS connection from "
- + "configuration. Using JDK properties.");
- }
-
- }
- return connection;
- }
- });
- }
-
public WebResource resource(String url) {
return restClient.resource(url);
}
- public BaseRestClient getRestClient() {
- return restClient;
- }
-
/**
* Execute the operation. Failures are raised as IOException subclasses
* @param method method to execute
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java
deleted file mode 100644
index 0266223..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/HttpOperationResponse.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.restclient;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * A response for use as a return value from operations
- */
-public class HttpOperationResponse {
-
- public int responseCode;
- public long lastModified;
- public String contentType;
- public byte[] data;
- public Map<String, List<String>> headers;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
index 20ef198..46f0d02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java
@@ -87,124 +87,4 @@ public class UrlConnectionOperations extends Configured {
Preconditions.checkArgument(url.getPort() != 0, "no port");
return (HttpURLConnection) connectionFactory.openConnection(url, useSpnego);
}
-
- public HttpOperationResponse execGet(URL url) throws
- IOException,
- AuthenticationException {
- return execHttpOperation(HttpVerb.GET, url, null, "");
- }
-
- public HttpOperationResponse execHttpOperation(HttpVerb verb,
- URL url,
- byte[] payload,
- String contentType)
- throws IOException, AuthenticationException {
- HttpURLConnection conn = null;
- HttpOperationResponse outcome = new HttpOperationResponse();
- int resultCode;
- byte[] body = null;
- log.debug("{} {} spnego={}", verb, url, useSpnego);
-
- boolean doOutput = verb.hasUploadBody();
- if (doOutput) {
- Preconditions.checkArgument(payload !=null,
- "Null payload on a verb which expects one");
- }
- try {
- conn = openConnection(url);
- conn.setRequestMethod(verb.getVerb());
- conn.setDoOutput(doOutput);
- if (doOutput) {
- conn.setRequestProperty("Content-Type", contentType);
- }
-
- // now do the connection
- conn.connect();
-
- if (doOutput) {
- OutputStream output = conn.getOutputStream();
- IOUtils.write(payload, output);
- output.close();
- }
-
- resultCode = conn.getResponseCode();
- outcome.lastModified = conn.getLastModified();
- outcome.contentType = conn.getContentType();
- outcome.headers = conn.getHeaderFields();
- InputStream stream = conn.getErrorStream();
- if (stream == null) {
- stream = conn.getInputStream();
- }
- if (stream != null) {
- // read into a buffer.
- body = IOUtils.toByteArray(stream);
- } else {
- // no body:
- log.debug("No body in response");
-
- }
- } catch (SSLException e) {
- throw e;
- } catch (IOException e) {
- throw NetUtils.wrapException(url.toString(),
- url.getPort(), "localhost", 0, e);
-
- } catch (AuthenticationException e) {
- throw new AuthenticationException("From " + url + ": " + e, e);
-
- } finally {
- if (conn != null) {
- conn.disconnect();
- }
- }
- uprateFaults(HttpVerb.GET, url.toString(), resultCode, "", body);
- outcome.responseCode = resultCode;
- outcome.data = body;
- return outcome;
- }
-
- /**
- * Uprate error codes 400 and up into faults;
- * 404 is converted to a {@link NotFoundException},
- * 401 to {@link ForbiddenException}
- *
- * @param verb HTTP Verb used
- * @param url URL as string
- * @param resultCode response from the request
- * @param bodyAsString
- *@param body optional body of the request @throws IOException if the result was considered a failure
- */
- public static void uprateFaults(HttpVerb verb, String url,
- int resultCode, String bodyAsString, byte[] body)
- throws IOException {
-
- if (resultCode < 400) {
- //success
- return;
- }
- String msg = verb.toString() +" "+ url;
- if (resultCode == 404) {
- throw new NotFoundException(msg);
- }
- if (resultCode == 401) {
- throw new ForbiddenException(msg);
- }
- // all other error codes
-
- // get a string respnse
- if (bodyAsString == null) {
- if (body != null && body.length > 0) {
- bodyAsString = new String(body);
- } else {
- bodyAsString = "";
- }
- }
- String message = msg +
- " failed with exit code " + resultCode
- + ", body length " + bodyAsString.length()
- + ":\n" + bodyAsString;
- log.error(message);
- throw new IOException(message);
- }
-
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/51] [abbrv] hadoop git commit: YARN-5775. Convert enums in
swagger definition to uppercase. Contributed by Gour Saha
Posted by ji...@apache.org.
YARN-5775. Convert enums in swagger definition to uppercase. Contributed by Gour Saha
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c89c04af
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c89c04af
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c89c04af
Branch: refs/heads/yarn-native-services
Commit: c89c04af5d0b72a69a3fb5b8a880e03ac6301381
Parents: cff3b51
Author: Billie Rinaldi <bi...@apache.org>
Authored: Tue Oct 25 11:25:51 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
...RN-Simplified-V1-API-Layer-For-Services.yaml | 38 ++++++++++----------
1 file changed, 19 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c89c04af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
index 6169fcd..7eb3196 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -241,10 +241,10 @@ definitions:
type: string
description: Artifact type, like docker, tarball, etc. (optional).
enum:
- - docker
- - tarball
- - application
- default: docker
+ - DOCKER
+ - TARBALL
+ - APPLICATION
+ default: DOCKER
uri:
type: string
description: Artifact location to support multiple artifact stores (optional).
@@ -303,7 +303,7 @@ definitions:
type: string
description: E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content).
enum:
- - http
+ - HTTP
uri:
type: string
description: Fully qualified REST uri endpoint.
@@ -335,13 +335,13 @@ definitions:
type: string
description: Config file in the standard format like xml, properties, json, yaml, template.
enum:
- - xml
- - properties
- - json
- - yaml
- - template
- - env
- - hadoop_xml
+ - XML
+ - PROPERTIES
+ - JSON
+ - YAML
+ - TEMPLATE
+ - ENV
+ - HADOOP_XML
dest_file:
type: string
description: The absolute path that this configuration file should be mounted as, in the application container.
@@ -386,11 +386,11 @@ definitions:
type: string
description: enum of the state of the application
enum:
- - accepted
- - started
- - ready
- - stopped
- - failed
+ - ACCEPTED
+ - STARTED
+ - READY
+ - STOPPED
+ - FAILED
ContainerState:
description: The current state of the container of an application.
properties:
@@ -398,8 +398,8 @@ definitions:
type: string
description: enum of the state of the container
enum:
- - init
- - ready
+ - INIT
+ - READY
ApplicationStatus:
description: The current status of a submitted application, returned as a response to the GET API.
properties:
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/51] [abbrv] hadoop git commit: YARN-5610. Initial code for native
services REST API. Contributed by Gour Saha
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java
new file mode 100644
index 0000000..a3780cc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.resource;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Resource determines the amount of resources (vcores, memory, network, etc.)
+ * usable by a container. This field determines the resource to be applied for
+ * all the containers of a component or application. The resource specified at
+ * the app (or global) level can be overriden at the component level. Only one
+ * of profile OR cpu & memory are exepected. It raises a validation
+ * exception otherwise.
+ **/
+
+@ApiModel(description = "Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or application. The resource specified at the app (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are exepected. It raises a validation exception otherwise.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+public class Resource extends BaseResource {
+ private static final long serialVersionUID = -6431667797380250037L;
+
+ private String profile = null;
+ private Integer cpus = null;
+ private String memory = null;
+
+ /**
+ * Each resource profile has a unique id which is associated with a
+ * cluster-level predefined memory, cpus, etc.
+ **/
+ public Resource profile(String profile) {
+ this.profile = profile;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Each resource profile has a unique id which is associated with a cluster-level predefined memory, cpus, etc.")
+ @JsonProperty("profile")
+ public String getProfile() {
+ return profile;
+ }
+
+ public void setProfile(String profile) {
+ this.profile = profile;
+ }
+
+ /**
+ * Amount of vcores allocated to each container (optional but overrides cpus
+ * in profile if specified).
+ **/
+ public Resource cpus(Integer cpus) {
+ this.cpus = cpus;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Amount of vcores allocated to each container (optional but overrides cpus in profile if specified).")
+ @JsonProperty("cpus")
+ public Integer getCpus() {
+ return cpus;
+ }
+
+ public void setCpus(Integer cpus) {
+ this.cpus = cpus;
+ }
+
+ /**
+ * Amount of memory allocated to each container (optional but overrides memory
+ * in profile if specified). Currently accepts only an integer value and
+ * default unit is in MB.
+ **/
+ public Resource memory(String memory) {
+ this.memory = memory;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Amount of memory allocated to each container (optional but overrides memory in profile if specified). Currently accepts only an integer value and default unit is in MB.")
+ @JsonProperty("memory")
+ public String getMemory() {
+ return memory;
+ }
+
+ public void setMemory(String memory) {
+ this.memory = memory;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Resource resource = (Resource) o;
+ return Objects.equals(this.profile, resource.profile)
+ && Objects.equals(this.cpus, resource.cpus)
+ && Objects.equals(this.memory, resource.memory);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(profile, cpus, memory);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Resource {\n");
+
+ sb.append(" profile: ").append(toIndentedString(profile)).append("\n");
+ sb.append(" cpus: ").append(toIndentedString(cpus)).append("\n");
+ sb.append(" memory: ").append(toIndentedString(memory)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ return super.clone();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java
new file mode 100644
index 0000000..4c16546
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.utils;
+
+public interface RestApiConstants {
+ String CONTEXT_ROOT = "/services/v1";
+ String APPLICATIONS_API_RESOURCE_PATH = "/applications";
+ String CONTAINERS_API_RESOURCE_PATH = "/containers";
+ String SLIDER_APPMASTER_COMPONENT_NAME = "slider-appmaster";
+ String SLIDER_CONFIG_SCHEMA = "http://example.org/specification/v2.0.0";
+ String METAINFO_SCHEMA_VERSION = "2.1";
+ String COMPONENT_TYPE_YARN_DOCKER = "yarn_docker";
+
+ String DEFAULT_START_CMD = "/bootstrap/privileged-centos6-sshd";
+ String DEFAULT_COMPONENT_NAME = "DEFAULT";
+ String DEFAULT_IMAGE = "centos:centos6";
+ String DEFAULT_NETWORK = "bridge";
+ String DEFAULT_COMMAND_PATH = "/usr/bin/docker";
+ String DEFAULT_USE_NETWORK_SCRIPT = "yes";
+
+ String PLACEHOLDER_APP_NAME = "${APP_NAME}";
+ String PLACEHOLDER_APP_COMPONENT_NAME = "${APP_COMPONENT_NAME}";
+ String PLACEHOLDER_COMPONENT_ID = "${COMPONENT_ID}";
+
+ String PROPERTY_REST_SERVICE_HOST = "REST_SERVICE_HOST";
+ String PROPERTY_REST_SERVICE_PORT = "REST_SERVICE_PORT";
+ String PROPERTY_APP_LIFETIME = "docker.lifetime";
+ String PROPERTY_APP_RUNAS_USER = "APP_RUNAS_USER";
+ Long DEFAULT_UNLIMITED_LIFETIME = -1l;
+
+ Integer HTTP_STATUS_CODE_ACCEPTED = 202;
+ String ARTIFACT_TYPE_SLIDER_ZIP = "slider-zip";
+
+ Integer GET_APPLICATIONS_THREAD_POOL_SIZE = 200;
+
+ String PROPERTY_PYTHON_PATH = "python.path";
+ String PROPERTY_COMPONENT_TYPE = "site.global.component_type";
+ String PROPERTY_DNS_DEPENDENCY = "site.global.dns.dependency";
+
+ String COMPONENT_TYPE_EXTERNAL = "external";
+
+ String COMMAND_ORDER_SUFFIX_START = "-START";
+ String COMMAND_ORDER_SUFFIX_STARTED = "-STARTED";
+ String EXPORT_GROUP_NAME = "QuickLinks";
+
+ Integer ERROR_CODE_APP_DOES_NOT_EXIST = 404001;
+ Integer ERROR_CODE_APP_IS_NOT_RUNNING = 404002;
+ Integer ERROR_CODE_APP_SUBMITTED_BUT_NOT_RUNNING_YET = 404003;
+ Integer ERROR_CODE_APP_NAME_INVALID = 404004;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java
new file mode 100644
index 0000000..685f85a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiErrorMessages.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.utils;
+
+public interface RestApiErrorMessages {
+ String ERROR_APPLICATION_NAME_INVALID =
+ "Application name is either empty or not provided";
+ String ERROR_APPLICATION_NAME_INVALID_FORMAT =
+ "Application name is not valid - only lower case letters, digits,"
+ + " underscore and hyphen are allowed";
+
+ String ERROR_APPLICATION_NOT_RUNNING = "Application not running";
+ String ERROR_APPLICATION_DOES_NOT_EXIST = "Application not found";
+
+ String ERROR_SUFFIX_FOR_COMPONENT =
+ " for component %s (nor at the global level)";
+ String ERROR_ARTIFACT_INVALID = "Artifact is not provided";
+ String ERROR_ARTIFACT_FOR_COMP_INVALID =
+ ERROR_ARTIFACT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_ARTIFACT_ID_INVALID =
+ "Artifact id (like docker image name) is either empty or not provided";
+ String ERROR_ARTIFACT_ID_FOR_COMP_INVALID =
+ ERROR_ARTIFACT_ID_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+
+ String ERROR_RESOURCE_INVALID = "Resource is not provided";
+ String ERROR_RESOURCE_FOR_COMP_INVALID =
+ ERROR_RESOURCE_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_RESOURCE_MEMORY_INVALID =
+ "Application resource or memory not provided";
+ String ERROR_RESOURCE_CPUS_INVALID =
+ "Application resource or cpus not provided";
+ String ERROR_RESOURCE_CPUS_INVALID_RANGE =
+ "Unacceptable no of cpus specified, either zero or negative";
+ String ERROR_RESOURCE_MEMORY_FOR_COMP_INVALID =
+ ERROR_RESOURCE_MEMORY_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID =
+ ERROR_RESOURCE_CPUS_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+ String ERROR_RESOURCE_CPUS_FOR_COMP_INVALID_RANGE =
+ ERROR_RESOURCE_CPUS_INVALID_RANGE
+ + " for component %s (or at the global level)";
+ String ERROR_CONTAINERS_COUNT_INVALID =
+ "Required no of containers not specified";
+ String ERROR_CONTAINERS_COUNT_FOR_COMP_INVALID =
+ ERROR_CONTAINERS_COUNT_INVALID + ERROR_SUFFIX_FOR_COMPONENT;
+
+ String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED =
+ "Cannot specify" + " cpus/memory along with profile";
+ String ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_FOR_COMP_NOT_SUPPORTED =
+ ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED
+ + " for component %s";
+ String ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET =
+ "Resource profile is not " + "supported yet. Please specify cpus/memory.";
+
+ String ERROR_APPLICATION_IN_USE = "Application name is already in use";
+ String ERROR_NULL_ARTIFACT_ID =
+ "Artifact Id can not be null if artifact type is none";
+ String ERROR_ABSENT_NUM_OF_INSTANCE =
+ "Num of instances should appear either globally or per component";
+ String ERROR_ABSENT_LAUNCH_COMMAND =
+ "launch command should appear if type is slider-zip or none";
+
+ String ERROR_QUICKLINKS_FOR_COMP_INVALID =
+ "Quicklinks specified at component level, needs corresponding values set at application level";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java
new file mode 100644
index 0000000..b1b6d7c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/webapp/ApplicationApiWebApp.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.webapp;
+
+import static org.apache.hadoop.yarn.services.utils.RestApiConstants.*;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.URI;
+import java.util.Arrays;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.mortbay.jetty.webapp.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class launches the web application using Hadoop HttpServer2 (which uses
+ * an embedded Jetty container). This is the entry point to your application.
+ * The Java command used to launch this app should call the main method.
+ */
+public class ApplicationApiWebApp extends AbstractService {
+ private static final Logger logger = LoggerFactory
+ .getLogger(ApplicationApiWebApp.class);
+ private static final String SEP = ";";
+
+ // REST API server for YARN native services
+ private HttpServer2 applicationApiServer;
+
+ public static void main(String[] args) throws IOException {
+ ApplicationApiWebApp apiWebApp = new ApplicationApiWebApp();
+ try {
+ apiWebApp.startWebApp();
+ } catch (Exception e) {
+ if (apiWebApp != null) {
+ apiWebApp.close();
+ }
+ }
+ }
+
+ public ApplicationApiWebApp() {
+ super(ApplicationApiWebApp.class.getName());
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ startWebApp();
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (applicationApiServer != null) {
+ applicationApiServer.stop();
+ }
+ super.serviceStop();
+ }
+
+ protected void startWebApp() throws IOException {
+ // The port that we should run on can be set into an environment variable
+ // Look for that variable and default to 9191 if it isn't there.
+ String webPort = System.getenv(PROPERTY_REST_SERVICE_PORT);
+ if (StringUtils.isEmpty(webPort)) {
+ webPort = "9191";
+ }
+
+ String webHost = System.getenv(PROPERTY_REST_SERVICE_HOST);
+ if (StringUtils.isEmpty(webHost)) {
+ webHost = InetAddress.getLocalHost().getHostName();
+ }
+ logger.info("YARN native services REST API running on host {} and port {}",
+ webHost, webPort);
+ logger.info("Configuration = {}", getConfig());
+
+ applicationApiServer = new HttpServer2.Builder()
+ .setName("services-rest-api")
+ .addEndpoint(URI.create("http://" + webHost + ":" + webPort)).build();
+
+ String apiPackages = "org.apache.hadoop.yarn.services.api" + SEP
+ + "org.apache.hadoop.yarn.services.api.impl" + SEP
+ + "org.apache.hadoop.yarn.services.resource" + SEP
+ + "org.apache.hadoop.yarn.services.utils" + SEP
+ + "org.apache.hadoop.yarn.services.webapp" + SEP
+ + GenericExceptionHandler.class.getPackage().getName() + SEP
+ + YarnJacksonJaxbJsonProvider.class.getPackage().getName();
+ applicationApiServer.addJerseyResourcePackage(apiPackages, CONTEXT_ROOT
+ + "/*");
+
+ try {
+ logger.info("Application starting up. Logging start...");
+ applicationApiServer.start();
+ logger.info("Server status = {}", applicationApiServer.toString());
+ for (Configuration conf : applicationApiServer.getWebAppContext()
+ .getConfigurations()) {
+ logger.info("Configurations = {}", conf);
+ }
+ logger.info("Context Path = {}", Arrays.asList(applicationApiServer
+ .getWebAppContext().getContextPath()));
+ logger.info("ResourceBase = {}", Arrays.asList(applicationApiServer
+ .getWebAppContext().getResourceBase()));
+ logger.info("War = {}",
+ Arrays.asList(applicationApiServer.getWebAppContext().getWar()));
+ } catch (Exception ex) {
+ logger.error("Hadoop HttpServer2 App **failed**", ex);
+ throw ex;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/log4j-server.properties
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/log4j-server.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/log4j-server.properties
new file mode 100644
index 0000000..8c679b9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/log4j-server.properties
@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This is the log4j configuration for YARN Services REST API Server
+
+# Log rotation based on size (100KB) with a max of 10 backup files
+log4j.rootLogger=INFO, restservicelog
+log4j.threshhold=ALL
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n
+
+log4j.appender.restservicelog=org.apache.log4j.RollingFileAppender
+log4j.appender.restservicelog.layout=org.apache.log4j.PatternLayout
+log4j.appender.restservicelog.File=${REST_SERVICE_LOG_DIR}/restservice.log
+log4j.appender.restservicelog.MaxFileSize=1GB
+log4j.appender.restservicelog.MaxBackupIndex=10
+
+# log layout skips stack-trace creation operations by avoiding line numbers and method
+log4j.appender.restservicelog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+# debug edition is much more expensive
+#log4j.appender.restservicelog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
+
+# configure stderr
+# set the conversion pattern of stderr
+# Print the date in ISO 8601 format
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.Target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n
+
+log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
+log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
+log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
+
+# for debugging REST API Service
+#log4j.logger.org.apache.hadoop.yarn.services=DEBUG
+
+# uncomment to debug service lifecycle issues
+#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
+#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
+
+# uncomment for YARN operations
+#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
+
+# uncomment this to debug security problems
+#log4j.logger.org.apache.hadoop.security=DEBUG
+
+#crank back on some noise
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdfs=WARN
+log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR
+
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN
+log4j.logger.org.apache.zookeeper=WARN
+log4j.logger.org.apache.curator.framework.state=ERROR
+log4j.logger.org.apache.curator.framework.imps=WARN
+
+log4j.logger.org.mortbay.log=DEBUG
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/services-rest-api/app
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/services-rest-api/app b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/services-rest-api/app
new file mode 100644
index 0000000..6a077b1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/webapps/services-rest-api/app
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DON'T DELETE. REST WEBAPP RUN SCRIPT WILL STOP WORKING.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh
new file mode 100644
index 0000000..9f15b7e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/scripts/run_rest_service.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export SLIDER_VERSION=${project.version}
+export HDP_VERSION=${HDP_VERSION:-$SLIDER_VERSION}
+export SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
+export LIB_PARENT_DIR=`dirname $SCRIPT_DIR`
+export JAVA_HOME=${JAVA_HOME:-/usr/jdk64/jdk1.8.0_40}
+export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop/conf}
+export REST_SERVICE_PORT=${REST_SERVICE_PORT:-9191}
+export APP_RUNAS_USER=${APP_RUNAS_USER:-root}
+export REST_SERVICE_LOG_DIR=${REST_SERVICE_LOG_DIR:-/tmp/}
+export JAVA_OPTS="-Xms256m -Xmx1024m -XX:+PrintGC -Xloggc:$REST_SERVICE_LOG_DIR/gc.log"
+$JAVA_HOME/bin/java $JAVA_OPTS -cp .:$HADOOP_CONF_DIR:$LIB_PARENT_DIR/services-api/*:$LIB_PARENT_DIR/slider/* -DREST_SERVICE_LOG_DIR=$REST_SERVICE_LOG_DIR -Dlog4j.configuration=log4j-server.properties -Dslider.libdir=$LIB_PARENT_DIR/slider org.apache.hadoop.yarn.services.webapp.ApplicationApiWebApp 1>>$REST_SERVICE_LOG_DIR/restservice-out.log 2>&1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml
new file mode 100644
index 0000000..f2f8b5b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/webapp/WEB-INF/web.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<web-app xmlns="http://java.sun.com/xml/ns/javaee" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd"
+ version="3.0">
+
+ <servlet>
+ <servlet-name>Jersey REST API</servlet-name>
+ <servlet-class>com.sun.jersey.spi.container.servlet.ServletContainer</servlet-class>
+ <init-param>
+ <param-name>com.sun.jersey.config.property.packages</param-name>
+ <param-value>org.apache.hadoop.yarn.services.webapp,org.apache.hadoop.yarn.services.api,org.apache.hadoop.yarn.services.resource,org.apache.hadoop.yarn.services.api.impl</param-value>
+ </init-param>
+ <init-param>
+ <param-name>com.sun.jersey.api.json.POJOMappingFeature</param-name>
+ <param-value>true</param-value>
+ </init-param>
+ <load-on-startup>1</load-on-startup>
+ </servlet>
+ <servlet-mapping>
+ <servlet-name>Jersey REST API</servlet-name>
+ <url-pattern>/*</url-pattern>
+ </servlet-mapping>
+</web-app>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java
new file mode 100644
index 0000000..a03ab69
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.api.impl;
+
+import static org.apache.hadoop.yarn.services.utils.RestApiConstants.*;
+import static org.apache.hadoop.yarn.services.utils.RestApiErrorMessages.*;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.services.resource.Application;
+import org.apache.hadoop.yarn.services.resource.Artifact;
+import org.apache.hadoop.yarn.services.resource.Resource;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor;
+import org.powermock.modules.junit4.PowerMockRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test class for application life time monitor feature test.
+ */
+@RunWith(PowerMockRunner.class)
+@SuppressStaticInitializationFor("org.apache.hadoop.yarn.services.api.impl.ApplicationApiService")
+public class TestApplicationApiService {
+ private static final Logger logger = LoggerFactory
+ .getLogger(TestApplicationApiService.class);
+ private static String EXCEPTION_PREFIX = "Should have thrown exception: ";
+ private static String NO_EXCEPTION_PREFIX = "Should not have thrown exception: ";
+ private ApplicationApiService appApiService;
+
+ @Before
+ public void setup() throws Exception {
+ appApiService = new ApplicationApiService();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ }
+
+ @Test(timeout = 90000)
+ public void testValidateApplicationPostPayload() throws Exception {
+ Application app = new Application();
+ Map<String, String> compNameArtifactIdMap = new HashMap<>();
+
+ // no name
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ Assert.fail(EXCEPTION_PREFIX + "application with no name");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage());
+ }
+
+ // bad format name
+ String[] badNames = { "4finance", "Finance", "finance@home" };
+ for (String badName : badNames) {
+ app.setName(badName);
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ Assert.fail(EXCEPTION_PREFIX + "application with bad name " + badName);
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID_FORMAT,
+ e.getMessage());
+ }
+ }
+
+ // no artifact
+ app.setName("finance_home");
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ Assert.fail(EXCEPTION_PREFIX + "application with no artifact");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_ARTIFACT_INVALID, e.getMessage());
+ }
+
+ // no artifact id
+ Artifact artifact = new Artifact();
+ app.setArtifact(artifact);
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
+ }
+
+ // if artifact is of type APPLICATION then everything is valid here
+ artifact.setType(Artifact.TypeEnum.APPLICATION);
+ artifact.setId("app.io/hbase:facebook_0.2");
+ app.setNumberOfContainers(5l);
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ } catch (IllegalArgumentException e) {
+ logger.error("application attributes specified should be valid here", e);
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ // default-component, default-lifetime and the property component_type
+ // should get assigned here
+ Assert.assertEquals(app.getComponents().get(0).getName(),
+ DEFAULT_COMPONENT_NAME);
+ Assert.assertEquals(app.getLifetime(), DEFAULT_UNLIMITED_LIFETIME);
+ Assert.assertEquals("Property not set",
+ app.getConfiguration().getProperties().get(PROPERTY_COMPONENT_TYPE),
+ COMPONENT_TYPE_EXTERNAL);
+
+ // unset artifact type, default component and no of containers to test other
+ // validation logic
+ artifact.setType(null);
+ app.setComponents(null);
+ app.setNumberOfContainers(null);
+
+ // resource not specified
+ artifact.setId("docker.io/centos:centos7");
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ Assert.fail(EXCEPTION_PREFIX + "application with no resource");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_RESOURCE_INVALID, e.getMessage());
+ }
+
+ // memory not specified
+ Resource res = new Resource();
+ app.setResource(res);
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ Assert.fail(EXCEPTION_PREFIX + "application with no memory");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_RESOURCE_MEMORY_INVALID, e.getMessage());
+ }
+
+ // cpus not specified
+ res.setMemory("2gb");
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ Assert.fail(EXCEPTION_PREFIX + "application with no cpu");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_RESOURCE_CPUS_INVALID, e.getMessage());
+ }
+
+ // invalid no of cpus
+ res.setCpus(-2);
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ Assert.fail(
+ EXCEPTION_PREFIX + "application with invalid no of cpups");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_RESOURCE_CPUS_INVALID_RANGE, e.getMessage());
+ }
+
+ // number of containers not specified
+ res.setCpus(2);
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ Assert.fail(
+ EXCEPTION_PREFIX + "application with no container count");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_CONTAINERS_COUNT_INVALID, e.getMessage());
+ }
+
+ // specifying profile along with cpus/memory raises exception
+ res.setProfile("hbase_finance_large");
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ Assert.fail(EXCEPTION_PREFIX
+ + "application with resource profile along with cpus/memory");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_RESOURCE_PROFILE_MULTIPLE_VALUES_NOT_SUPPORTED,
+ e.getMessage());
+ }
+
+ // currently resource profile alone is not supported.
+ // TODO: remove the next test once it is supported.
+ res.setCpus(null);
+ res.setMemory(null);
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ Assert.fail(EXCEPTION_PREFIX
+ + "application with resource profile only - NOT SUPPORTED");
+ } catch (IllegalArgumentException e) {
+ Assert.assertEquals(ERROR_RESOURCE_PROFILE_NOT_SUPPORTED_YET,
+ e.getMessage());
+ }
+
+ // unset profile here and add cpus/memory back
+ res.setProfile(null);
+ res.setCpus(2);
+ res.setMemory("2gb");
+
+ // everything valid here
+ app.setNumberOfContainers(5l);
+ try {
+ appApiService.validateApplicationPostPayload(app,
+ compNameArtifactIdMap);
+ } catch (IllegalArgumentException e) {
+ logger.error("application attributes specified should be valid here", e);
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+
+ // Now test with components
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
index 493e03a..2cec8bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
@@ -37,7 +37,7 @@
<module>hadoop-yarn-applications-distributedshell</module>
<module>hadoop-yarn-applications-unmanaged-am-launcher</module>
<module>hadoop-yarn-slider</module>
-
+ <module>hadoop-yarn-services-api</module>
</modules>
<profiles>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/51] [abbrv] hadoop git commit: YARN-5610. Initial code for native
services REST API. Contributed by Gour Saha
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
new file mode 100644
index 0000000..cfcae95
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
@@ -0,0 +1,452 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.resource;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.slider.providers.PlacementPolicy;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+
+/**
+ * An Application resource has the following attributes.
+ **/
+
+@ApiModel(description = "An Application resource has the following attributes.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonPropertyOrder({ " name, state, resource, numberOfContainers, lifetime, containers " })
+public class Application extends BaseResource {
+ private static final long serialVersionUID = -4491694636566094885L;
+
+ private String id = null;
+ private String name = null;
+ private Artifact artifact = null;
+ private Resource resource = null;
+ private String launchCommand = null;
+ private Date launchTime = null;
+ private Long numberOfContainers = null;
+ private Long numberOfRunningContainers = null;
+ private Long lifetime = null;
+ private PlacementPolicy placementPolicy = null;
+ private List<Component> components = null;
+ private Configuration configuration = null;
+ private List<Container> containers = new ArrayList<>();
+ private ApplicationState state = null;
+ private Map<String, String> quicklinks = null;
+ private String queue;
+
+ /**
+ * A unique application id.
+ **/
+ public Application id(String id) {
+ this.id = id;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", required = true, value = "A unique application id.")
+ @JsonProperty("id")
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ /**
+ * A unique application name.
+ **/
+ public Application name(String name) {
+ this.name = name;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", required = true, value = "A unique application name.")
+ @JsonProperty("name")
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ /**
+ * Artifact of single-component applications. Mandatory if components
+ * attribute is not specified.
+ **/
+ public Application artifact(Artifact artifact) {
+ this.artifact = artifact;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact of single-component applications. Mandatory if components attribute is not specified.")
+ @JsonProperty("artifact")
+ public Artifact getArtifact() {
+ return artifact;
+ }
+
+ public void setArtifact(Artifact artifact) {
+ this.artifact = artifact;
+ }
+
+ /**
+ * Resource of single-component applications or the global default for
+ * multi-component applications. Mandatory if it is a single-component
+ * application and if cpus and memory are not specified at the Application
+ * level.
+ **/
+ public Application resource(Resource resource) {
+ this.resource = resource;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Resource of single-component applications or the global default for multi-component applications. Mandatory if it is a single-component application and if cpus and memory are not specified at the Application level.")
+ @JsonProperty("resource")
+ public Resource getResource() {
+ return resource;
+ }
+
+ public void setResource(Resource resource) {
+ this.resource = resource;
+ }
+
+ /**
+ * The custom launch command of an application component (optional). If not
+ * specified for applications with docker images say, it will default to the
+ * default start command of the image. If there is a single component in this
+ * application, you can specify this without the need to have a 'components'
+ * section.
+ **/
+ public Application launchCommand(String launchCommand) {
+ this.launchCommand = launchCommand;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The custom launch command of an application component (optional). If not specified for applications with docker images say, it will default to the default start command of the image. If there is a single component in this application, you can specify this without the need to have a 'components' section.")
+ @JsonProperty("launch_command")
+ public String getLaunchCommand() {
+ return launchCommand;
+ }
+
+ @XmlElement(name = "launch_command")
+ public void setLaunchCommand(String launchCommand) {
+ this.launchCommand = launchCommand;
+ }
+
+ /**
+ * The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.
+ **/
+ public Application launchTime(Date launchTime) {
+ this.launchTime = launchTime;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.")
+ @JsonProperty("launch_time")
+ public String getLaunchTime() {
+ return launchTime.toString();
+ }
+
+ @XmlElement(name = "launch_time")
+ public void setLaunchTime(Date launchTime) {
+ this.launchTime = launchTime;
+ }
+
+ /**
+ * Number of containers for each app-component in the application. Each
+ * app-component can further override this app-level global default.
+ **/
+ public Application numberOfContainers(Long numberOfContainers) {
+ this.numberOfContainers = numberOfContainers;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Number of containers for each app-component in the application. Each app-component can further override this app-level global default.")
+ @JsonProperty("number_of_containers")
+ public Long getNumberOfContainers() {
+ return numberOfContainers;
+ }
+
+ @XmlElement(name = "number_of_containers")
+ public void setNumberOfContainers(Long numberOfContainers) {
+ this.numberOfContainers = numberOfContainers;
+ }
+
+ /**
+ * In get response this provides the total number of running containers for
+ * this application (across all components) at the time of request. Note, a
+ * subsequent request can return a different number as and when more
+ * containers get allocated until it reaches the total number of containers or
+ * if a flex request has been made between the two requests.
+ **/
+ public Application numberOfRunningContainers(Long numberOfRunningContainers) {
+ this.numberOfRunningContainers = numberOfRunningContainers;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "In get response this provides the total number of running containers for this application (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.")
+ @JsonProperty("number_of_running_containers")
+ public Long getNumberOfRunningContainers() {
+ return numberOfRunningContainers;
+ }
+
+ @XmlElement(name = "number_of_running_containers")
+ public void setNumberOfRunningContainers(Long numberOfRunningContainers) {
+ this.numberOfRunningContainers = numberOfRunningContainers;
+ }
+
+ /**
+ * Life time (in seconds) of the application from the time it reaches the
+ * STARTED state (after which it is automatically destroyed by YARN). For
+ * unlimited lifetime do not set a lifetime value.
+ **/
+ public Application lifetime(Long lifetime) {
+ this.lifetime = lifetime;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Life time (in seconds) of the application from the time it reaches the STARTED state (after which it is automatically destroyed by YARN). For unlimited lifetime do not set a lifetime value.")
+ @JsonProperty("lifetime")
+ public Long getLifetime() {
+ return lifetime;
+ }
+
+ public void setLifetime(Long lifetime) {
+ this.lifetime = lifetime;
+ }
+
+ /**
+ * Advanced scheduling and placement policies (optional). If not specified, it
+ * defaults to the default placement policy of the app owner. The design of
+ * placement policies are in the works. It is not very clear at this point,
+ * how policies in conjunction with labels be exposed to application owners.
+ * This is a placeholder for now. The advanced structure of this attribute
+ * will be determined by YARN-4902.
+ **/
+ public Application placementPolicy(PlacementPolicy placementPolicy) {
+ this.placementPolicy = placementPolicy;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the app owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to application owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.")
+ @JsonProperty("placement_policy")
+ public PlacementPolicy getPlacementPolicy() {
+ return placementPolicy;
+ }
+
+ @XmlElement(name = "placement_policy")
+ public void setPlacementPolicy(PlacementPolicy placementPolicy) {
+ this.placementPolicy = placementPolicy;
+ }
+
+ /**
+ * Components of an application.
+ **/
+ public Application components(List<Component> components) {
+ this.components = components;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Components of an application.")
+ @JsonProperty("components")
+ public List<Component> getComponents() {
+ return components;
+ }
+
+ public void setComponents(List<Component> components) {
+ this.components = components;
+ }
+
+ /**
+ * Config properties of an application. Configurations provided at the
+ * application/global level are available to all the components. Specific
+ * properties can be overridden at the component level.
+ **/
+ public Application configuration(Configuration configuration) {
+ this.configuration = configuration;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Config properties of an application. Configurations provided at the application/global level are available to all the components. Specific properties can be overridden at the component level.")
+ @JsonProperty("configuration")
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ public void setConfiguration(Configuration configuration) {
+ this.configuration = configuration;
+ }
+
+ /**
+ * Containers of a started application. Specifying a value for this attribute
+ * for the POST payload raises a validation error. This blob is available only
+ * in the GET response of a started application.
+ **/
+ public Application containers(List<Container> containers) {
+ this.containers = containers;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Containers of a started application. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started application.")
+ @JsonProperty("containers")
+ public List<Container> getContainers() {
+ return containers;
+ }
+
+ public void setContainers(List<Container> containers) {
+ this.containers = containers;
+ }
+
+ public void addContainer(Container container) {
+ this.containers.add(container);
+ }
+
+ /**
+ * State of the application. Specifying a value for this attribute for the
+ * POST payload raises a validation error. This attribute is available only in
+ * the GET response of a started application.
+ **/
+ public Application state(ApplicationState state) {
+ this.state = state;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "State of the application. Specifying a value for this attribute for the POST payload raises a validation error. This attribute is available only in the GET response of a started application.")
+ @JsonProperty("state")
+ public ApplicationState getState() {
+ return state;
+ }
+
+ public void setState(ApplicationState state) {
+ this.state = state;
+ }
+
+ /**
+ * A blob of key-value pairs of quicklinks to be exported for an application.
+ **/
+ public Application quicklinks(Map<String, String> quicklinks) {
+ this.quicklinks = quicklinks;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A blob of key-value pairs of quicklinks to be exported for an application.")
+ @JsonProperty("quicklinks")
+ public Map<String, String> getQuicklinks() {
+ return quicklinks;
+ }
+
+ public void setQuicklinks(Map<String, String> quicklinks) {
+ this.quicklinks = quicklinks;
+ }
+
+ /**
+ * The YARN queue that this application should be submitted to.
+ **/
+ public Application queue(String queue) {
+ this.queue = queue;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The YARN queue that this application should be submitted to.")
+ @JsonProperty("queue")
+ public String getQueue() {
+ return queue;
+ }
+
+ public void setQueue(String queue) {
+ this.queue = queue;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Application application = (Application) o;
+ return Objects.equals(this.name, application.name);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Application {\n");
+
+ sb.append(" name: ").append(toIndentedString(name)).append("\n");
+ sb.append(" id: ").append(toIndentedString(id)).append("\n");
+ sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
+ sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
+ sb.append(" launchCommand: ").append(toIndentedString(launchCommand))
+ .append("\n");
+ sb.append(" launchTime: ").append(toIndentedString(launchTime))
+ .append("\n");
+ sb.append(" numberOfContainers: ")
+ .append(toIndentedString(numberOfContainers)).append("\n");
+ sb.append(" numberOfRunningContainers: ")
+ .append(toIndentedString(numberOfRunningContainers)).append("\n");
+ sb.append(" lifetime: ").append(toIndentedString(lifetime)).append("\n");
+ sb.append(" placementPolicy: ")
+ .append(toIndentedString(placementPolicy)).append("\n");
+ sb.append(" components: ").append(toIndentedString(components))
+ .append("\n");
+ sb.append(" configuration: ").append(toIndentedString(configuration))
+ .append("\n");
+ sb.append(" containers: ").append(toIndentedString(containers))
+ .append("\n");
+ sb.append(" state: ").append(toIndentedString(state)).append("\n");
+ sb.append(" quicklinks: ").append(toIndentedString(quicklinks))
+ .append("\n");
+ sb.append(" queue: ").append(toIndentedString(queue)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ApplicationState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ApplicationState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ApplicationState.java
new file mode 100644
index 0000000..ae96e8a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ApplicationState.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.resource;
+
+/**
+ * The current state of an application.
+ **/
+public enum ApplicationState {
+ ACCEPTED, STARTED, READY, STOPPED, FAILED;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ApplicationStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ApplicationStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ApplicationStatus.java
new file mode 100644
index 0000000..0166b48
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ApplicationStatus.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.resource;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * The current status of a submitted application, returned as a response to the
+ * GET API.
+ **/
+
+@ApiModel(description = "The current status of a submitted application, returned as a response to the GET api")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ApplicationStatus extends BaseResource {
+ private static final long serialVersionUID = -3469885905347851034L;
+
+ private String diagnostics = null;
+ private ApplicationState state = null;
+ private Integer code = null;
+
+ /**
+ * Diagnostic information (if any) for the reason of the current state of the
+ * application. It typically has a non-null value, if the application is in a
+ * non-running state.
+ **/
+ public ApplicationStatus diagnostics(String diagnostics) {
+ this.diagnostics = diagnostics;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Diagnostic information (if any) for the reason of the current state of the application. It typically has a non-null value, if the application is in a non-running state.")
+ @JsonProperty("diagnostics")
+ public String getDiagnostics() {
+ return diagnostics;
+ }
+
+ @XmlElement(name = "diagnostics")
+ public void setDiagnostics(String diagnostics) {
+ this.diagnostics = diagnostics;
+ }
+
+ /**
+ * Application state
+ **/
+ public ApplicationStatus state(ApplicationState state) {
+ this.state = state;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Application state")
+ @JsonProperty("state")
+ public ApplicationState getState() {
+ return state;
+ }
+
+ public void setState(ApplicationState state) {
+ this.state = state;
+ }
+
+ /**
+ * An error code specific to a scenario which app owners should be able to use
+ * to understand the failure in addition to the diagnostic information.
+ **/
+ public ApplicationStatus code(Integer code) {
+ this.code = code;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "An error code specific to a scenario which app owners should be able to use to understand the failure in addition to the diagnostic information.")
+ @JsonProperty("code")
+ public Integer getCode() {
+ return code;
+ }
+
+ public void setCode(Integer code) {
+ this.code = code;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ApplicationStatus applicationStatus = (ApplicationStatus) o;
+ return Objects.equals(this.diagnostics, applicationStatus.diagnostics)
+ && Objects.equals(this.state, applicationStatus.state)
+ && Objects.equals(this.code, applicationStatus.code);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(diagnostics, state, code);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class ApplicationStatus {\n");
+
+ sb.append(" diagnostics: ").append(toIndentedString(diagnostics))
+ .append("\n");
+ sb.append(" state: ").append(toIndentedString(state)).append("\n");
+ sb.append(" code: ").append(toIndentedString(code)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
new file mode 100644
index 0000000..aee4d11
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.resource;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonValue;
+
+/**
+ * Artifact of an application component.
+ **/
+
+@ApiModel(description = "Artifact of an application component")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Artifact {
+
+ private String id = null;
+
+ public enum TypeEnum {
+ DOCKER("docker"), TARBALL("tarball"), APPLICATION("application");
+
+ private String value;
+
+ TypeEnum(String value) {
+ this.value = value;
+ }
+
+ @Override
+ @JsonValue
+ public String toString() {
+ return value;
+ }
+ }
+
+ private TypeEnum type = TypeEnum.DOCKER;
+ private String uri = null;
+
+ /**
+ * Artifact id. Examples are package location uri for tarball based apps,
+ * image name for docker, etc.
+ **/
+ public Artifact id(String id) {
+ this.id = id;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", required = true, value = "Artifact id. Examples are package location uri for tarball based apps, image name for docker, etc.")
+ @JsonProperty("id")
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ /**
+ * Artifact type, like docker, tarball, etc. (optional)
+ **/
+ public Artifact type(TypeEnum type) {
+ this.type = type;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact type, like docker, tarball, etc. (optional)")
+ @JsonProperty("type")
+ public TypeEnum getType() {
+ return type;
+ }
+
+ public void setType(TypeEnum type) {
+ this.type = type;
+ }
+
+ /**
+ * Artifact location to support multiple artifact stores (optional).
+ **/
+ public Artifact uri(String uri) {
+ this.uri = uri;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact location to support multiple artifact stores (optional).")
+ @JsonProperty("uri")
+ public String getUri() {
+ return uri;
+ }
+
+ public void setUri(String uri) {
+ this.uri = uri;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Artifact artifact = (Artifact) o;
+ return Objects.equals(this.id, artifact.id)
+ && Objects.equals(this.type, artifact.type)
+ && Objects.equals(this.uri, artifact.uri);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id, type, uri);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Artifact {\n");
+
+ sb.append(" id: ").append(toIndentedString(id)).append("\n");
+ sb.append(" type: ").append(toIndentedString(type)).append("\n");
+ sb.append(" uri: ").append(toIndentedString(uri)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/BaseResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/BaseResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/BaseResource.java
new file mode 100644
index 0000000..3b2c8b1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/BaseResource.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.resource;
+
+import java.io.Serializable;
+
+public class BaseResource implements Serializable {
+ private static final long serialVersionUID = 1492603053176889431L;
+
+ private String uri;
+
+ /**
+ * Resource location, e.g. \
+ * "/applications/helloworld/containers/container_e3751_1458061340047_0008_01_000002\
+ * "
+ **/
+ public String getUri() {
+ return uri;
+ }
+
+ public void setUri(String uri) {
+ this.uri = uri;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("BaseResource [uri=");
+ builder.append(uri);
+ builder.append("]");
+ return builder.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Component.java
new file mode 100644
index 0000000..3ff6945
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Component.java
@@ -0,0 +1,377 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.resource;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * One or more components of the application. If the application is HBase say,
+ * then the component can be a simple role like master or regionserver. If the
+ * application is a complex business webapp then a component can be other
+ * applications say Kafka or Storm. Thereby it opens up the support for complex
+ * and nested applications.
+ **/
+
+@ApiModel(description = "One or more components of the application. If the application is HBase say, then the component can be a simple role like master or regionserver. If the application is a complex business webapp then a component can be other applications say Kafka or Storm. Thereby it opens up the support for complex and nested applications.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Component {
+
+ private String name = null;
+ private List<String> dependencies = new ArrayList<String>();
+ private ReadinessCheck readinessCheck = null;
+ private Artifact artifact = null;
+ private String launchCommand = null;
+ private Resource resource = null;
+ private Long numberOfContainers = null;
+ private Boolean uniqueComponentSupport = null;
+ private Boolean runPrivilegedContainer = null;
+ private PlacementPolicy placementPolicy = null;
+ private Configuration configuration = null;
+ private List<String> quicklinks = new ArrayList<String>();
+
+ /**
+ * Name of the application component (mandatory).
+ **/
+ public Component name(String name) {
+ this.name = name;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", required = true, value = "Name of the application component (mandatory).")
+ @JsonProperty("name")
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ /**
+ * An array of application components which should be in READY state (as
+ * defined by readiness check), before this component can be started. The
+ * dependencies across all components of an application should be represented
+ * as a DAG.
+ **/
+ public Component dependencies(List<String> dependencies) {
+ this.dependencies = dependencies;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "An array of application components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of an application should be represented as a DAG.")
+ @JsonProperty("dependencies")
+ public List<String> getDependencies() {
+ return dependencies;
+ }
+
+ public void setDependencies(List<String> dependencies) {
+ this.dependencies = dependencies;
+ }
+
+ /**
+ * Readiness check for this app-component.
+ **/
+ public Component readinessCheck(ReadinessCheck readinessCheck) {
+ this.readinessCheck = readinessCheck;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Readiness check for this app-component.")
+ @JsonProperty("readiness_check")
+ public ReadinessCheck getReadinessCheck() {
+ return readinessCheck;
+ }
+
+ @XmlElement(name = "readiness_check")
+ public void setReadinessCheck(ReadinessCheck readinessCheck) {
+ this.readinessCheck = readinessCheck;
+ }
+
+ /**
+ * Artifact of the component (optional). If not specified, the application
+ * level global artifact takes effect.
+ **/
+ public Component artifact(Artifact artifact) {
+ this.artifact = artifact;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact of the component (optional). If not specified, the application level global artifact takes effect.")
+ @JsonProperty("artifact")
+ public Artifact getArtifact() {
+ return artifact;
+ }
+
+ public void setArtifact(Artifact artifact) {
+ this.artifact = artifact;
+ }
+
+ /**
+ * The custom launch command of this component (optional). When specified at
+ * the component level, it overrides the value specified at the global level
+ * (if any).
+ **/
+ public Component launchCommand(String launchCommand) {
+ this.launchCommand = launchCommand;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The custom launch command of this component (optional). When specified at the component level, it overrides the value specified at the global level (if any).")
+ @JsonProperty("launch_command")
+ public String getLaunchCommand() {
+ return launchCommand;
+ }
+
+ @XmlElement(name = "launch_command")
+ public void setLaunchCommand(String launchCommand) {
+ this.launchCommand = launchCommand;
+ }
+
+ /**
+ * Resource of this component (optional). If not specified, the application
+ * level global resource takes effect.
+ **/
+ public Component resource(Resource resource) {
+ this.resource = resource;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Resource of this component (optional). If not specified, the application level global resource takes effect.")
+ @JsonProperty("resource")
+ public Resource getResource() {
+ return resource;
+ }
+
+ public void setResource(Resource resource) {
+ this.resource = resource;
+ }
+
+ /**
+ * Number of containers for this app-component (optional). If not specified,
+ * the application level global number_of_containers takes effect.
+ **/
+ public Component numberOfContainers(Long numberOfContainers) {
+ this.numberOfContainers = numberOfContainers;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Number of containers for this app-component (optional). If not specified, the application level global number_of_containers takes effect.")
+ @JsonProperty("number_of_containers")
+ public Long getNumberOfContainers() {
+ return numberOfContainers;
+ }
+
+ @XmlElement(name = "number_of_containers")
+ public void setNumberOfContainers(Long numberOfContainers) {
+ this.numberOfContainers = numberOfContainers;
+ }
+
+ /**
+ * Certain applications need to define multiple components using the same
+ * artifact and resource profile, differing only in configurations. In such
+ * cases, this field helps app owners to avoid creating multiple component
+ * definitions with repeated information. The number_of_containers field
+ * dictates the initial number of components created. Component names
+ * typically differ with a trailing id, but assumptions should not be made on
+ * that, as the algorithm can change at any time. Configurations section will
+ * be able to use placeholders like ${APP_COMPONENT_NAME} to get its component
+ * name at runtime, and thereby differing in value at runtime. The best part
+ * of this feature is that when the component is flexed up, entirely new
+ * components (with new trailing ids) are created.
+ **/
+ public Component uniqueComponentSupport(Boolean uniqueComponentSupport) {
+ this.uniqueComponentSupport = uniqueComponentSupport;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Certain applications need to define multiple components using the same artifact and resource profile, differing only in configurations. In such cases, this field helps app owners to avoid creating multiple component definitions with repeated information. The number_of_containers field dictates the initial number of components created. Component names typically differ with a trailing id, but assumptions should not be made on that, as the algorithm can change at any time. Configurations section will be able to use placeholders like ${APP_COMPONENT_NAME} to get its component name at runtime, and thereby differing in value at runtime. The best part of this feature is that when the component is flexed up, entirely new components (with new trailing ids) are created.")
+ @JsonProperty("unique_component_support")
+ public Boolean getUniqueComponentSupport() {
+ return uniqueComponentSupport;
+ }
+
+ @XmlElement(name = "unique_component_support")
+ public void setUniqueComponentSupport(Boolean uniqueComponentSupport) {
+ this.uniqueComponentSupport = uniqueComponentSupport;
+ }
+
+ /**
+ * Run all containers of this component in privileged mode (YARN-4262).
+ **/
+ public Component runPrivilegedContainer(Boolean runPrivilegedContainer) {
+ this.runPrivilegedContainer = runPrivilegedContainer;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Run all containers of this component in privileged mode (YARN-4262).")
+ @JsonProperty("run_privileged_container")
+ public Boolean getRunPrivilegedContainer() {
+ return runPrivilegedContainer;
+ }
+
+ @XmlElement(name = "run_privileged_container")
+ public void setRunPrivilegedContainer(Boolean runPrivilegedContainer) {
+ this.runPrivilegedContainer = runPrivilegedContainer;
+ }
+
+ /**
+ * Advanced scheduling and placement policies for all containers of this
+ * component (optional). If not specified, the app level placement_policy
+ * takes effect. Refer to the description at the global level for more
+ * details.
+ **/
+ public Component placementPolicy(PlacementPolicy placementPolicy) {
+ this.placementPolicy = placementPolicy;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the app level placement_policy takes effect. Refer to the description at the global level for more details.")
+ @JsonProperty("placement_policy")
+ public PlacementPolicy getPlacementPolicy() {
+ return placementPolicy;
+ }
+
+ @XmlElement(name = "placement_policy")
+ public void setPlacementPolicy(PlacementPolicy placementPolicy) {
+ this.placementPolicy = placementPolicy;
+ }
+
+ /**
+ * Config properties for this app-component.
+ **/
+ public Component configuration(Configuration configuration) {
+ this.configuration = configuration;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Config properties for this app-component.")
+ @JsonProperty("configuration")
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ public void setConfiguration(Configuration configuration) {
+ this.configuration = configuration;
+ }
+
+ /**
+ * A list of quicklink keys defined at the application level, and to be
+ * resolved by this component.
+ **/
+ public Component quicklinks(List<String> quicklinks) {
+ this.quicklinks = quicklinks;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A list of quicklink keys defined at the application level, and to be resolved by this component.")
+ @JsonProperty("quicklinks")
+ public List<String> getQuicklinks() {
+ return quicklinks;
+ }
+
+ public void setQuicklinks(List<String> quicklinks) {
+ this.quicklinks = quicklinks;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Component component = (Component) o;
+ return Objects.equals(this.name, component.name)
+ && Objects.equals(this.dependencies, component.dependencies)
+ && Objects.equals(this.readinessCheck, component.readinessCheck)
+ && Objects.equals(this.artifact, component.artifact)
+ && Objects.equals(this.launchCommand, component.launchCommand)
+ && Objects.equals(this.resource, component.resource)
+ && Objects
+ .equals(this.numberOfContainers, component.numberOfContainers)
+ && Objects.equals(this.uniqueComponentSupport,
+ component.uniqueComponentSupport)
+ && Objects.equals(this.runPrivilegedContainer,
+ component.runPrivilegedContainer)
+ && Objects.equals(this.placementPolicy, component.placementPolicy)
+ && Objects.equals(this.configuration, component.configuration)
+ && Objects.equals(this.quicklinks, component.quicklinks);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, dependencies, readinessCheck, artifact,
+ launchCommand, resource, numberOfContainers, uniqueComponentSupport,
+ runPrivilegedContainer, placementPolicy, configuration, quicklinks);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Component {\n");
+
+ sb.append(" name: ").append(toIndentedString(name)).append("\n");
+ sb.append(" dependencies: ").append(toIndentedString(dependencies))
+ .append("\n");
+ sb.append(" readinessCheck: ").append(toIndentedString(readinessCheck))
+ .append("\n");
+ sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
+ sb.append(" launchCommand: ").append(toIndentedString(launchCommand))
+ .append("\n");
+ sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
+ sb.append(" numberOfContainers: ")
+ .append(toIndentedString(numberOfContainers)).append("\n");
+ sb.append(" uniqueComponentSupport: ")
+ .append(toIndentedString(uniqueComponentSupport)).append("\n");
+ sb.append(" runPrivilegedContainer: ")
+ .append(toIndentedString(runPrivilegedContainer)).append("\n");
+ sb.append(" placementPolicy: ")
+ .append(toIndentedString(placementPolicy)).append("\n");
+ sb.append(" configuration: ").append(toIndentedString(configuration))
+ .append("\n");
+ sb.append(" quicklinks: ").append(toIndentedString(quicklinks))
+ .append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
new file mode 100644
index 0000000..d06c1b8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.resource;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonValue;
+
+/**
+ * A config file that needs to be created and made available as a volume in an
+ * application component container.
+ **/
+
+@ApiModel(description = "A config file that needs to be created and made available as a volume in an application component container.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ConfigFile {
+
+ public enum TypeEnum {
+ XML("xml"), PROPERTIES("properties"), JSON("json"), YAML("yaml"), TEMPLATE(
+ "template"), ENV("env"), HADOOP_XML("hadoop_xml");
+
+ private String value;
+
+ TypeEnum(String value) {
+ this.value = value;
+ }
+
+ @Override
+ @JsonValue
+ public String toString() {
+ return value;
+ }
+ }
+
+ private TypeEnum type = null;
+ private String destFile = null;
+ private String srcFile = null;
+ private Object props = null;
+
+ /**
+ * Config file in the standard format like xml, properties, json, yaml,
+ * template.
+ **/
+ public ConfigFile type(TypeEnum type) {
+ this.type = type;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Config file in the standard format like xml, properties, json, yaml, template.")
+ @JsonProperty("type")
+ public TypeEnum getType() {
+ return type;
+ }
+
+ public void setType(TypeEnum type) {
+ this.type = type;
+ }
+
+ /**
+ * The absolute path that this configuration file should be mounted as, in the
+ * application container.
+ **/
+ public ConfigFile destFile(String destFile) {
+ this.destFile = destFile;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The absolute path that this configuration file should be mounted as, in the application container.")
+ @JsonProperty("dest_file")
+ public String getDestFile() {
+ return destFile;
+ }
+
+ @XmlElement(name = "dest_file")
+ public void setDestFile(String destFile) {
+ this.destFile = destFile;
+ }
+
+ /**
+ * Required for type template. This provides the source location of the
+ * template which needs to be mounted as dest_file post property
+ * substitutions. Typically the src_file would point to a source controlled
+ * network accessible file maintained by tools like puppet, chef, etc.
+ **/
+ public ConfigFile srcFile(String srcFile) {
+ this.srcFile = srcFile;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Required for type template. This provides the source location of the template which needs to be mounted as dest_file post property substitutions. Typically the src_file would point to a source controlled network accessible file maintained by tools like puppet, chef, etc.")
+ @JsonProperty("src_file")
+ public String getSrcFile() {
+ return srcFile;
+ }
+
+ @XmlElement(name = "src_file")
+ public void setSrcFile(String srcFile) {
+ this.srcFile = srcFile;
+ }
+
+ /**
+ * A blob of key value pairs that will be dumped in the dest_file in the
+ * format as specified in type. If the type is template then the attribute
+ * src_file is mandatory and the src_file content is dumped to dest_file post
+ * property substitutions.
+ **/
+ public ConfigFile props(Object props) {
+ this.props = props;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A blob of key value pairs that will be dumped in the dest_file in the format as specified in type. If the type is template then the attribute src_file is mandatory and the src_file content is dumped to dest_file post property substitutions.")
+ @JsonProperty("props")
+ public Object getProps() {
+ return props;
+ }
+
+ public void setProps(Object props) {
+ this.props = props;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ConfigFile configFile = (ConfigFile) o;
+ return Objects.equals(this.type, configFile.type)
+ && Objects.equals(this.destFile, configFile.destFile)
+ && Objects.equals(this.srcFile, configFile.srcFile)
+ && Objects.equals(this.props, configFile.props);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(type, destFile, srcFile, props);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class ConfigFile {\n");
+
+ sb.append(" type: ").append(toIndentedString(type)).append("\n");
+ sb.append(" destFile: ").append(toIndentedString(destFile)).append("\n");
+ sb.append(" srcFile: ").append(toIndentedString(srcFile)).append("\n");
+ sb.append(" props: ").append(toIndentedString(props)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Configuration.java
new file mode 100644
index 0000000..05983db
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Configuration.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.resource;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Set of configuration properties that can be injected into the application
+ * components via envs, files and custom pluggable helper docker containers.
+ * Files of several standard formats like xml, properties, json, yaml and
+ * templates will be supported.
+ **/
+
+@ApiModel(description = "Set of configuration properties that can be injected into the application components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Configuration {
+
+ private Map<String, String> properties = new HashMap<String, String>();
+ private Map<String, String> env = new HashMap<String, String>();
+ private List<ConfigFile> files = new ArrayList<ConfigFile>();
+
+ /**
+ * A blob of key-value pairs of common application properties.
+ **/
+ public Configuration properties(Map<String, String> properties) {
+ this.properties = properties;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A blob of key-value pairs of common application properties.")
+ @JsonProperty("properties")
+ public Map<String, String> getProperties() {
+ return properties;
+ }
+
+ public void setProperties(Map<String, String> properties) {
+ this.properties = properties;
+ }
+
+ /**
+ * A blob of key-value pairs which will be appended to the default system
+ * properties and handed off to the application at start time. All placeholder
+ * references to properties will be substituted before injection.
+ **/
+ public Configuration env(Map<String, String> env) {
+ this.env = env;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "A blob of key-value pairs which will be appended to the default system properties and handed off to the application at start time. All placeholder references to properties will be substituted before injection.")
+ @JsonProperty("env")
+ public Map<String, String> getEnv() {
+ return env;
+ }
+
+ public void setEnv(Map<String, String> env) {
+ this.env = env;
+ }
+
+ /**
+ * Array of list of files that needs to be created and made available as
+ * volumes in the application component containers.
+ **/
+ public Configuration files(List<ConfigFile> files) {
+ this.files = files;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Array of list of files that needs to be created and made available as volumes in the application component containers.")
+ @JsonProperty("files")
+ public List<ConfigFile> getFiles() {
+ return files;
+ }
+
+ public void setFiles(List<ConfigFile> files) {
+ this.files = files;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Configuration configuration = (Configuration) o;
+ return Objects.equals(this.properties, configuration.properties)
+ && Objects.equals(this.env, configuration.env)
+ && Objects.equals(this.files, configuration.files);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(properties, env, files);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Configuration {\n");
+
+ sb.append(" properties: ").append(toIndentedString(properties))
+ .append("\n");
+ sb.append(" env: ").append(toIndentedString(env)).append("\n");
+ sb.append(" files: ").append(toIndentedString(files)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java
new file mode 100644
index 0000000..2faf6f2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.resource;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Date;
+import java.util.Objects;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@ApiModel(description = "An instance of a running application container")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Container extends BaseResource {
+ private static final long serialVersionUID = -8955788064529288L;
+
+ private String id = null;
+ private Date launchTime = null;
+ private String ip = null;
+ private String hostname = null;
+ private String bareHost = null;
+ private ContainerState state = null;
+ private String componentName = null;
+ private Resource resource = null;
+
+ /**
+ * Unique container id of a running application, e.g.
+ * container_e3751_1458061340047_0008_01_000002
+ **/
+ public Container id(String id) {
+ this.id = id;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Unique container id of a running application, e.g. container_e3751_1458061340047_0008_01_000002")
+ @JsonProperty("id")
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ /**
+ * The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.
+ **/
+ public Container launchTime(Date launchTime) {
+ this.launchTime = launchTime;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.")
+ @JsonProperty("launch_time")
+ public String getLaunchTime() {
+ return launchTime.toString();
+ }
+
+ @XmlElement(name = "launch_time")
+ public void setLaunchTime(Date launchTime) {
+ this.launchTime = launchTime;
+ }
+
+ /**
+ * IP address of a running container, e.g. 172.31.42.141. The IP address and
+ * hostname attribute values are dependent on the cluster/docker network setup
+ * as per YARN-4007.
+ **/
+ public Container ip(String ip) {
+ this.ip = ip;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "IP address of a running container, e.g. 172.31.42.141. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.")
+ @JsonProperty("ip")
+ public String getIp() {
+ return ip;
+ }
+
+ public void setIp(String ip) {
+ this.ip = ip;
+ }
+
+ /**
+ * Fully qualified hostname of a running container, e.g.
+ * ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and
+ * hostname attribute values are dependent on the cluster/docker network setup
+ * as per YARN-4007.
+ **/
+ public Container hostname(String hostname) {
+ this.hostname = hostname;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Fully qualified hostname of a running container, e.g. ctr-e3751-1458061340047-0008-01-000002.examplestg.site. The IP address and hostname attribute values are dependent on the cluster/docker network setup as per YARN-4007.")
+ @JsonProperty("hostname")
+ public String getHostname() {
+ return hostname;
+ }
+
+ public void setHostname(String hostname) {
+ this.hostname = hostname;
+ }
+
+ /**
+ * The bare node or host in which the container is running, e.g.
+ * cn008.example.com
+ **/
+ public Container bareHost(String bareHost) {
+ this.bareHost = bareHost;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "The bare node or host in which the container is running, e.g. cn008.example.com")
+ @JsonProperty("bare_host")
+ public String getBareHost() {
+ return bareHost;
+ }
+
+ @XmlElement(name = "bare_host")
+ public void setBareHost(String bareHost) {
+ this.bareHost = bareHost;
+ }
+
+ /**
+ * State of the container of an application.
+ **/
+ public Container state(ContainerState state) {
+ this.state = state;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "State of the container of an application.")
+ @JsonProperty("state")
+ public ContainerState getState() {
+ return state;
+ }
+
+ public void setState(ContainerState state) {
+ this.state = state;
+ }
+
+ /**
+ * Name of the component that this container instance belongs to.
+ **/
+ public Container componentName(String componentName) {
+ this.componentName = componentName;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Name of the component that this container instance belongs to.")
+ @JsonProperty("component_name")
+ public String getComponentName() {
+ return componentName;
+ }
+
+ @XmlElement(name = "component_name")
+ public void setComponentName(String componentName) {
+ this.componentName = componentName;
+ }
+
+ /**
+ * Resource used for this container.
+ **/
+ public Container resource(Resource resource) {
+ this.resource = resource;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Resource used for this container.")
+ @JsonProperty("resource")
+ public Resource getResource() {
+ return resource;
+ }
+
+ public void setResource(Resource resource) {
+ this.resource = resource;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Container container = (Container) o;
+ return Objects.equals(this.id, container.id)
+ && Objects.equals(this.launchTime, container.launchTime)
+ && Objects.equals(this.ip, container.ip)
+ && Objects.equals(this.hostname, container.hostname)
+ && Objects.equals(this.bareHost, container.bareHost)
+ && Objects.equals(this.state, container.state)
+ && Objects.equals(this.componentName, container.componentName)
+ && Objects.equals(this.resource, container.resource);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(id, launchTime, ip, hostname, bareHost, state,
+ componentName, resource);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Container {\n");
+
+ sb.append(" id: ").append(toIndentedString(id)).append("\n");
+ sb.append(" launchTime: ").append(toIndentedString(launchTime))
+ .append("\n");
+ sb.append(" ip: ").append(toIndentedString(ip)).append("\n");
+ sb.append(" hostname: ").append(toIndentedString(hostname)).append("\n");
+ sb.append(" bareHost: ").append(toIndentedString(bareHost)).append("\n");
+ sb.append(" state: ").append(toIndentedString(state)).append("\n");
+ sb.append(" componentName: ").append(toIndentedString(componentName))
+ .append("\n");
+ sb.append(" resource: ").append(toIndentedString(resource)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ContainerState.java
new file mode 100644
index 0000000..cb017fb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ContainerState.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.resource;
+
+/**
+ * The current state of the container of an application.
+ **/
+public enum ContainerState {
+ INIT, READY;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Error.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Error.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Error.java
new file mode 100644
index 0000000..91c4e3a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Error.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.resource;
+
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+public class Error {
+
+ private Integer code = null;
+ private String message = null;
+ private String fields = null;
+
+ /**
+ **/
+ public Error code(Integer code) {
+ this.code = code;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "")
+ @JsonProperty("code")
+ public Integer getCode() {
+ return code;
+ }
+
+ public void setCode(Integer code) {
+ this.code = code;
+ }
+
+ /**
+ **/
+ public Error message(String message) {
+ this.message = message;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "")
+ @JsonProperty("message")
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+
+ /**
+ **/
+ public Error fields(String fields) {
+ this.fields = fields;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "")
+ @JsonProperty("fields")
+ public String getFields() {
+ return fields;
+ }
+
+ public void setFields(String fields) {
+ this.fields = fields;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ Error error = (Error) o;
+ return Objects.equals(this.code, error.code)
+ && Objects.equals(this.message, error.message)
+ && Objects.equals(this.fields, error.fields);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(code, message, fields);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class Error {\n");
+
+ sb.append(" code: ").append(toIndentedString(code)).append("\n");
+ sb.append(" message: ").append(toIndentedString(message)).append("\n");
+ sb.append(" fields: ").append(toIndentedString(fields)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/PlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/PlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/PlacementPolicy.java
new file mode 100644
index 0000000..7541e2f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/PlacementPolicy.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.resource;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Placement policy of an instance of an application. This feature is in the
+ * works in YARN-4902.
+ **/
+
+@ApiModel(description = "Placement policy of an instance of an application. This feature is in the works in YARN-4902.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+public class PlacementPolicy {
+
+ private String label = null;
+
+ /**
+ * Assigns an app to a named partition of the cluster where the application
+ * desires to run (optional). If not specified all apps are submitted to a
+ * default label of the app owner. One or more labels can be setup for each
+ * application owner account with required constraints like no-preemption,
+ * sla-99999, preemption-ok, etc.
+ **/
+ public PlacementPolicy label(String label) {
+ this.label = label;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Assigns an app to a named partition of the cluster where the application desires to run (optional). If not specified all apps are submitted to a default label of the app owner. One or more labels can be setup for each application owner account with required constraints like no-preemption, sla-99999, preemption-ok, etc.")
+ @JsonProperty("label")
+ public String getLabel() {
+ return label;
+ }
+
+ public void setLabel(String label) {
+ this.label = label;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ PlacementPolicy placementPolicy = (PlacementPolicy) o;
+ return Objects.equals(this.label, placementPolicy.label);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(label);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class PlacementPolicy {\n");
+
+ sb.append(" label: ").append(toIndentedString(label)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07710758/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ReadinessCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ReadinessCheck.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ReadinessCheck.java
new file mode 100644
index 0000000..80fdf92
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ReadinessCheck.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.resource;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonValue;
+
+/**
+ * A custom command or a pluggable helper container to determine the readiness
+ * of a container of a component. Readiness for every application is different.
+ * Hence the need for a simple interface, with scope to support advanced
+ * usecases.
+ **/
+
+@ApiModel(description = "A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every application is different. Hence the need for a simple interface, with scope to support advanced usecases.")
+@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+public class ReadinessCheck {
+
+ public enum TypeEnum {
+ HTTP("http");
+
+ private String value;
+
+ TypeEnum(String value) {
+ this.value = value;
+ }
+
+ @Override
+ @JsonValue
+ public String toString() {
+ return value;
+ }
+ }
+
+ private TypeEnum type = null;
+ private String uri = null;
+ private Artifact artifact = null;
+
+ /**
+ * http (YARN will perform a simple REST call at a regular interval and expect
+ * a 204 No content).
+ **/
+ public ReadinessCheck type(TypeEnum type) {
+ this.type = type;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "http (YARN will perform a simple REST call at a regular interval and expect a 204 No content).")
+ @JsonProperty("type")
+ public TypeEnum getType() {
+ return type;
+ }
+
+ public void setType(TypeEnum type) {
+ this.type = type;
+ }
+
+ /**
+ * Fully qualified REST uri endpoint.
+ **/
+ public ReadinessCheck uri(String uri) {
+ this.uri = uri;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", required = true, value = "Fully qualified REST uri endpoint.")
+ @JsonProperty("uri")
+ public String getUri() {
+ return uri;
+ }
+
+ public void setUri(String uri) {
+ this.uri = uri;
+ }
+
+ /**
+ * Artifact of the pluggable readiness check helper container (optional). If
+ * specified, this helper container typically hosts the http uri and
+ * encapsulates the complex scripts required to perform actual container
+ * readiness check. At the end it is expected to respond a 204 No content just
+ * like the simplified use case. This pluggable framework benefits application
+ * owners who can run applications without any packaging modifications. Note,
+ * artifacts of type docker only is supported for now.
+ **/
+ public ReadinessCheck artifact(Artifact artifact) {
+ this.artifact = artifact;
+ return this;
+ }
+
+ @ApiModelProperty(example = "null", value = "Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits application owners who can run applications without any packaging modifications. Note, artifacts of type docker only is supported for now.")
+ @JsonProperty("artifact")
+ public Artifact getArtifact() {
+ return artifact;
+ }
+
+ public void setArtifact(Artifact artifact) {
+ this.artifact = artifact;
+ }
+
+ @Override
+ public boolean equals(java.lang.Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ ReadinessCheck readinessCheck = (ReadinessCheck) o;
+ return Objects.equals(this.type, readinessCheck.type)
+ && Objects.equals(this.uri, readinessCheck.uri)
+ && Objects.equals(this.artifact, readinessCheck.artifact);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(type, uri, artifact);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("class ReadinessCheck {\n");
+
+ sb.append(" type: ").append(toIndentedString(type)).append("\n");
+ sb.append(" uri: ").append(toIndentedString(uri)).append("\n");
+ sb.append(" artifact: ").append(toIndentedString(artifact)).append("\n");
+ sb.append("}");
+ return sb.toString();
+ }
+
+ /**
+ * Convert the given object to string with each line indented by 4 spaces
+ * (except the first line).
+ */
+ private String toIndentedString(java.lang.Object o) {
+ if (o == null) {
+ return "null";
+ }
+ return o.toString().replace("\n", "\n ");
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/51] [abbrv] hadoop git commit: YARN-5975. Remove the agent -
slider AM ssl related code. Contributed by Jian He
Posted by ji...@apache.org.
YARN-5975. Remove the agent - slider AM ssl related code. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cef1ce44
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cef1ce44
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cef1ce44
Branch: refs/heads/yarn-native-services
Commit: cef1ce44bc7a124ac311fc3fb121c328525e1294
Parents: a62a490
Author: Billie Rinaldi <bi...@apache.org>
Authored: Tue Dec 13 10:16:09 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../slider/api/SliderClusterProtocol.java | 3 -
.../slider/api/proto/RestTypeMarshalling.java | 36 --
.../org/apache/slider/client/SliderClient.java | 80 ---
.../client/ipc/SliderClusterOperations.java | 28 +-
.../slider/common/params/ActionClientArgs.java | 31 +-
.../apache/slider/providers/ProviderUtils.java | 141 -----
.../providers/docker/DockerProviderService.java | 5 -
.../server/appmaster/SliderAppMaster.java | 18 +-
.../rpc/SliderClusterProtocolPBImpl.java | 12 -
.../rpc/SliderClusterProtocolProxy.java | 18 +-
.../server/appmaster/rpc/SliderIPCService.java | 54 +-
.../slider/server/appmaster/web/WebAppApi.java | 23 -
.../server/appmaster/web/WebAppApiImpl.java | 37 +-
.../AbstractSecurityStoreGenerator.java | 98 ----
.../services/security/CertificateManager.java | 495 -----------------
.../services/security/KeystoreGenerator.java | 64 ---
.../server/services/security/SecurityStore.java | 66 ---
.../security/SecurityStoreGenerator.java | 40 --
.../server/services/security/SecurityUtils.java | 256 ---------
.../services/security/SignCertResponse.java | 67 ---
.../server/services/security/SignMessage.java | 54 --
.../services/security/StoresGenerator.java | 68 ---
.../services/security/TruststoreGenerator.java | 62 ---
.../src/main/proto/SliderClusterProtocol.proto | 6 -
.../security/TestCertificateManager.java | 540 -------------------
.../TestMultiThreadedStoreGeneration.java | 156 ------
26 files changed, 14 insertions(+), 2444 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
index 33fce22..893e706 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
@@ -173,7 +173,4 @@ public interface SliderClusterProtocol extends VersionedProtocol {
Messages.WrappedJsonProto getModelResolvedResources(Messages.EmptyPayloadProto request) throws IOException;
Messages.WrappedJsonProto getLiveResources(Messages.EmptyPayloadProto request) throws IOException;
-
- Messages.GetCertificateStoreResponseProto getClientCertificateStore(Messages.GetCertificateStoreRequestProto request)
- throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
index 17fd965..ec35028 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
@@ -18,8 +18,6 @@
package org.apache.slider.api.proto;
-import com.google.protobuf.ByteString;
-import org.apache.commons.io.IOUtils;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.ContainerInformation;
@@ -30,15 +28,10 @@ import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.persist.AggregateConfSerDeser;
import org.apache.slider.core.persist.ConfTreeSerDeser;
-import org.apache.slider.server.services.security.SecurityStore;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -95,35 +88,6 @@ public class RestTypeMarshalling {
}
return info;
}
-
- public static Messages.GetCertificateStoreResponseProto marshall(
- SecurityStore securityStore) throws IOException {
- Messages.GetCertificateStoreResponseProto.Builder builder =
- Messages.GetCertificateStoreResponseProto.newBuilder();
- builder.setStore(ByteString.copyFrom(getStoreBytes(securityStore)));
-
- return builder.build();
- }
-
- private static byte[] getStoreBytes(SecurityStore securityStore)
- throws IOException {
- InputStream is = null;
- byte[] storeBytes;
- try {
- is = new FileInputStream(securityStore.getFile());
- storeBytes = IOUtils.toByteArray(is);
- } finally {
- if (is != null) {
- is.close();
- }
- }
- return storeBytes;
- }
-
- public static byte[] unmarshall(Messages.GetCertificateStoreResponseProto response) {
- return response.getStore().toByteArray();
- }
-
public static Messages.ComponentInformationProto marshall(ComponentInformation info) {
Messages.ComponentInformationProto.Builder builder =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 12f7870..ef45d10 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -168,7 +168,6 @@ import org.apache.slider.providers.docker.DockerClientProvider;
import org.apache.slider.providers.slideram.SliderAMClientProvider;
import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.rpc.RpcBinder;
-import org.apache.slider.server.services.security.SecurityStore;
import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
@@ -1223,8 +1222,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
IOException {
if (clientInfo.install) {
return doClientInstall(clientInfo);
- } else if (clientInfo.getCertStore) {
- return doCertificateStoreRetrieval(clientInfo);
} else {
throw new BadCommandArgumentsException(
"Only install, keystore, and truststore commands are supported for the client.\n"
@@ -1233,83 +1230,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
}
- private int doCertificateStoreRetrieval(ActionClientArgs clientInfo)
- throws YarnException, IOException {
- if (clientInfo.keystore != null && clientInfo.truststore != null) {
- throw new BadCommandArgumentsException(
- "Only one of either keystore or truststore can be retrieved at one time. "
- + "Retrieval of both should be done separately\n"
- + CommonArgs.usage(serviceArgs, ACTION_CLIENT));
- }
-
- requireArgumentSet(Arguments.ARG_NAME, clientInfo.name);
-
- File storeFile = null;
- SecurityStore.StoreType type;
- if (clientInfo.keystore != null) {
- storeFile = clientInfo.keystore;
- type = SecurityStore.StoreType.keystore;
- } else {
- storeFile = clientInfo.truststore;
- type = SecurityStore.StoreType.truststore;
- }
-
- require (!storeFile.exists(),
- "File %s already exists. Please remove that file or select a different file name.",
- storeFile.getAbsolutePath());
- String hostname = null;
- if (type == SecurityStore.StoreType.keystore) {
- hostname = clientInfo.hostname;
- if (hostname == null) {
- hostname = InetAddress.getLocalHost().getCanonicalHostName();
- log.info("No hostname specified via command line. Using {}", hostname);
- }
- }
-
- String password = clientInfo.password;
- if (password == null) {
- String provider = clientInfo.provider;
- String alias = clientInfo.alias;
- if (provider != null && alias != null) {
- Configuration conf = new Configuration(getConfig());
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
- char[] chars = conf.getPassword(alias);
- if (chars == null) {
- CredentialProvider credentialProvider =
- CredentialProviderFactory.getProviders(conf).get(0);
- chars = readOnePassword(alias);
- credentialProvider.createCredentialEntry(alias, chars);
- credentialProvider.flush();
- }
- password = String.valueOf(chars);
- Arrays.fill(chars, ' ');
- } else {
- log.info("No password and no provider/alias pair were provided, " +
- "prompting for password");
- // get a password
- password = String.valueOf(readOnePassword(type.name()));
- }
- }
-
- byte[] keystore = createClusterOperations(clientInfo.name)
- .getClientCertificateStore(hostname, "client", password, type.name());
- // persist to file
- FileOutputStream storeFileOutputStream = null;
- try {
- storeFileOutputStream = new FileOutputStream(storeFile);
- IOUtils.write(keystore, storeFileOutputStream);
- } catch (Exception e) {
- log.error("Unable to persist to file {}", storeFile);
- throw e;
- } finally {
- if (storeFileOutputStream != null) {
- storeFileOutputStream.close();
- }
- }
-
- return EXIT_SUCCESS;
- }
-
private int doClientInstall(ActionClientArgs clientInfo)
throws IOException, SliderException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
index 392f451..eaf15e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
@@ -19,17 +19,12 @@
package org.apache.slider.client.ipc;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.SliderClusterProtocol;
import org.apache.slider.api.StateValues;
import org.apache.slider.api.proto.Messages;
-
-import static org.apache.slider.api.proto.RestTypeMarshalling.*;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.ContainerInformation;
@@ -37,7 +32,6 @@ import org.apache.slider.api.types.NodeInformation;
import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.api.types.PingInformation;
import org.apache.slider.common.tools.Duration;
-import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.conf.ConfTreeOperations;
@@ -45,8 +39,6 @@ import org.apache.slider.core.exceptions.NoSuchNodeException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.exceptions.WaitTimeoutException;
import org.apache.slider.core.persist.ConfTreeSerDeser;
-import org.apache.slider.server.services.security.SecurityStore;
-import org.apache.slider.server.services.security.SignCertResponse;
import org.codehaus.jackson.JsonParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -59,6 +51,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import static org.apache.slider.api.proto.RestTypeMarshalling.*;
+
/**
* Cluster operations at a slightly higher level than the RPC code
*/
@@ -508,22 +502,4 @@ public class SliderClusterOperations {
);
return unmarshall(proto);
}
-
- public byte[] getClientCertificateStore(String hostname, String clientId,
- String password, String type) throws IOException {
- Messages.GetCertificateStoreRequestProto.Builder
- builder = Messages.GetCertificateStoreRequestProto.newBuilder();
- if (hostname != null) {
- builder.setHostname(hostname);
- }
- Messages.GetCertificateStoreRequestProto requestProto =
- builder.setRequesterId(clientId)
- .setPassword(password)
- .setType(type)
- .build();
- Messages.GetCertificateStoreResponseProto response =
- appMaster.getClientCertificateStore(requestProto);
-
- return unmarshall(response);
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java
index 85d39ea..09e2b62 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java
@@ -37,39 +37,10 @@ public class ActionClientArgs extends AbstractActionArgs {
description = "Install client")
public boolean install;
- @Parameter(names = {ARG_GETCERTSTORE},
- description = "Get a certificate store")
- public boolean getCertStore;
-
- @Parameter(names = {ARG_KEYSTORE},
- description = "Retrieve keystore to specified location")
- public File keystore;
-
- @Parameter(names = {ARG_TRUSTSTORE},
- description = "Retrieve truststore to specified location")
- public File truststore;
-
- @Parameter(names = {ARG_HOSTNAME},
- description = "(Optional) Specify the hostname to use for generation of keystore certificate")
- public String hostname;
-
@Parameter(names = {ARG_NAME},
description = "The name of the application")
public String name;
- @Parameter(names = {ARG_PROVIDER},
- description = "The credential provider in which the password is stored")
- public String provider;
-
- @Parameter(names = {ARG_ALIAS},
- description = "The credential provider alias associated with the password")
- public String alias;
-
- @Parameter(names = {ARG_PASSWORD},
- description = "The certificate store password (alternative to " +
- "provider/alias; if password is specified, those will be ignored)")
- public String password;
-
@Parameter(names = {ARG_PACKAGE},
description = "Path to app package")
public String packageURI;
@@ -95,4 +66,4 @@ public class ActionClientArgs extends AbstractActionArgs {
public int getMaxParams() {
return 1;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index bc237f5..cff5ed8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.slider.api.ClusterNode;
@@ -38,7 +37,6 @@ import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.OptionKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.RoleKeys;
-import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.tools.SliderFileSystem;
@@ -59,9 +57,6 @@ import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
import org.apache.slider.core.registry.docstore.PublishedExports;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.services.security.CertificateManager;
-import org.apache.slider.server.services.security.SecurityStore;
-import org.apache.slider.server.services.security.StoresGenerator;
import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
import org.slf4j.Logger;
@@ -398,61 +393,6 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
}
}
- /**
- * Return whether two-way SSL is enabled for Agent / AM communication.
- * @param amComponent component specification
- * @return true if enabled
- */
- public boolean hasTwoWaySSLEnabled(MapOperations amComponent) {
- return amComponent != null ?
- amComponent.getOptionBool(TWO_WAY_SSL_ENABLED, false) : false;
- }
-
- /**
- * Generate and localize SSL certs for Agent / AM communication
- * @param launcher container launcher
- * @param container allocated container information
- * @param fileSystem file system
- * @param clusterName app name
- * @throws SliderException certs cannot be generated/uploaded
- */
- public void localizeContainerSSLResources(ContainerLauncher launcher,
- Container container, SliderFileSystem fileSystem, String clusterName)
- throws SliderException {
- try {
- // localize server cert
- Path certsDir = fileSystem.buildClusterSecurityDirPath(clusterName);
- LocalResource certResource = fileSystem.createAmResource(
- new Path(certsDir, CRT_FILE_NAME),
- LocalResourceType.FILE);
- launcher.addLocalResource(CERT_FILE_LOCALIZATION_PATH, certResource);
-
- // generate and localize agent cert
- CertificateManager certMgr = new CertificateManager();
- String hostname = container.getNodeId().getHost();
- String containerId = container.getId().toString();
- certMgr.generateContainerCertificate(hostname, containerId);
- LocalResource agentCertResource = fileSystem.createAmResource(
- uploadSecurityResource(
- CertificateManager.getAgentCertficateFilePath(containerId),
- fileSystem, clusterName), LocalResourceType.FILE);
- // still using hostname as file name on the agent side, but the files
- // do end up under the specific container's file space
- launcher.addLocalResource(INFRA_RUN_SECURITY_DIR + hostname +
- ".crt", agentCertResource);
- LocalResource agentKeyResource = fileSystem.createAmResource(
- uploadSecurityResource(
- CertificateManager.getAgentKeyFilePath(containerId), fileSystem,
- clusterName),
- LocalResourceType.FILE);
- launcher.addLocalResource(INFRA_RUN_SECURITY_DIR + hostname +
- ".key", agentKeyResource);
-
- } catch (Exception e) {
- throw new SliderException(SliderExitCodes.EXIT_DEPLOYMENT_FAILED, e,
- "Unable to localize certificates. Two-way SSL cannot be enabled");
- }
- }
/**
* Upload a local file to the cluster security dir in HDFS. If the file
@@ -707,87 +647,6 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
}
/**
- * Generate and localize security stores requested by the app. Also perform
- * last-minute substitution of cluster name into credentials strings.
- * @param launcher container launcher
- * @param container allocated container information
- * @param role component name
- * @param fileSystem file system
- * @param instanceDefinition app specification
- * @param compOps component specification
- * @param clusterName app name
- * @throws SliderException stores cannot be generated/uploaded
- * @throws IOException stores cannot be generated/uploaded
- */
- public void localizeContainerSecurityStores(ContainerLauncher launcher,
- Container container,
- String role,
- SliderFileSystem fileSystem,
- AggregateConf instanceDefinition,
- MapOperations compOps,
- String clusterName)
- throws SliderException, IOException {
- // substitute CLUSTER_NAME into credentials
- Map<String,List<String>> newcred = new HashMap<>();
- for (Entry<String,List<String>> entry :
- instanceDefinition.getAppConf().credentials.entrySet()) {
- List<String> resultList = new ArrayList<>();
- for (String v : entry.getValue()) {
- resultList.add(v.replaceAll(Pattern.quote("${CLUSTER_NAME}"),
- clusterName).replaceAll(Pattern.quote("${CLUSTER}"),
- clusterName));
- }
- newcred.put(entry.getKey().replaceAll(Pattern.quote("${CLUSTER_NAME}"),
- clusterName).replaceAll(Pattern.quote("${CLUSTER}"),
- clusterName),
- resultList);
- }
- instanceDefinition.getAppConf().credentials = newcred;
-
- // generate and localize security stores
- SecurityStore[] stores = generateSecurityStores(container, role,
- instanceDefinition, compOps);
- for (SecurityStore store : stores) {
- LocalResource keystoreResource = fileSystem.createAmResource(
- uploadSecurityResource(store.getFile(), fileSystem, clusterName),
- LocalResourceType.FILE);
- launcher.addLocalResource(String.format("secstores/%s-%s.p12",
- store.getType(), role),
- keystoreResource);
- }
- }
-
- /**
- * Generate security stores requested by the app.
- * @param container allocated container information
- * @param role component name
- * @param instanceDefinition app specification
- * @param compOps component specification
- * @return security stores
- * @throws SliderException stores cannot be generated
- * @throws IOException stores cannot be generated
- */
- private SecurityStore[] generateSecurityStores(Container container,
- String role,
- AggregateConf instanceDefinition,
- MapOperations compOps)
- throws SliderException, IOException {
- return StoresGenerator.generateSecurityStores(
- container.getNodeId().getHost(), container.getId().toString(),
- role, instanceDefinition, compOps);
- }
-
- /**
- * Return whether security stores are requested by the app.
- * @param compOps component specification
- * @return true if stores are requested
- */
- public boolean areStoresRequested(MapOperations compOps) {
- return compOps != null ? compOps.
- getOptionBool(COMP_STORES_REQUIRED_KEY, false) : false;
- }
-
- /**
* Localize application tarballs and other resources requested by the app.
* @param launcher container launcher
* @param fileSystem file system
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
index e4a7cdf..1482062 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
@@ -165,11 +165,6 @@ public class DockerProviderService extends AbstractProviderService implements
fileSystem, getClusterName());
}
- if (providerUtils.areStoresRequested(appComponent)) {
- providerUtils.localizeContainerSecurityStores(launcher, container,
- roleName, fileSystem, instanceDefinition, appComponent, getClusterName());
- }
-
if (appComponent.getOptionBool(AM_CONFIG_GENERATION, false)) {
// build and localize configuration files
Map<String, Map<String, String>> configurations =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 8c39343..74dbc88 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -26,8 +26,6 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.http.HttpConfig;
@@ -155,7 +153,6 @@ import org.apache.slider.server.appmaster.web.rest.InsecureAmFilterInitializer;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
import org.apache.slider.server.appmaster.web.rest.application.ApplicationResouceContentCacheFactory;
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
-import org.apache.slider.server.services.security.CertificateManager;
import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
import org.apache.slider.server.services.utility.WebAppService;
import org.apache.slider.server.services.workflow.ServiceThreadFactory;
@@ -373,7 +370,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
private InetSocketAddress rpcServiceAddress;
private SliderAMProviderService sliderAMProvider;
- private CertificateManager certificateManager;
/**
* Executor.
@@ -732,8 +728,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
}
- certificateManager = new CertificateManager();
-
//bring up the Slider RPC service
buildPortScanner(instanceDefinition);
startSliderRPCServer(instanceDefinition);
@@ -757,18 +751,12 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// Start up the WebApp and track the URL for it
MapOperations component = instanceDefinition.getAppConfOperations()
.getComponent(SliderKeys.COMPONENT_AM);
- certificateManager.initialize(component, appMasterHostname,
- appMasterContainerID.toString(),
- clustername);
- certificateManager.setPassphrase(instanceDefinition.getPassphrase());
// Web service endpoints: initialize
WebAppApiImpl webAppApi =
new WebAppApiImpl(
stateForProviders,
- providerService,
- certificateManager,
- registryOperations,
+ providerService, registryOperations,
metricsAndMonitoring,
actionQueues,
this,
@@ -1551,9 +1539,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
verifyIPCAccess();
sliderIPCService = new SliderIPCService(
- this,
- certificateManager,
- stateForProviders,
+ this, stateForProviders,
actionQueues,
metricsAndMonitoring,
contentCache);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java
index f0d9063..fbd408e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java
@@ -303,16 +303,4 @@ public class SliderClusterProtocolPBImpl implements SliderClusterProtocolPB {
throw wrap(e);
}
}
-
- @Override
- public Messages.GetCertificateStoreResponseProto getClientCertificateStore(
- RpcController controller,
- Messages.GetCertificateStoreRequestProto request)
- throws ServiceException {
- try {
- return real.getClientCertificateStore(request);
- } catch (Exception e) {
- throw wrap(e);
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
index b230816..448c6f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
@@ -88,10 +88,9 @@ public class SliderClusterProtocolProxy implements SliderClusterProtocol {
return ioe;
}
- @Override
- public Messages.StopClusterResponseProto stopCluster(Messages.StopClusterRequestProto request) throws
- IOException,
- YarnException {
+ @Override public Messages.StopClusterResponseProto stopCluster(
+ Messages.StopClusterRequestProto request)
+ throws IOException, YarnException {
try {
return endpoint.stopCluster(NULL_CONTROLLER, request);
} catch (ServiceException e) {
@@ -343,16 +342,5 @@ public class SliderClusterProtocolProxy implements SliderClusterProtocol {
} catch (ServiceException e) {
throw convert(e);
}
-
- }
-
- @Override
- public Messages.GetCertificateStoreResponseProto getClientCertificateStore(Messages.GetCertificateStoreRequestProto request) throws
- IOException {
- try {
- return endpoint.getClientCertificateStore(NULL_CONTROLLER, request);
- } catch (ServiceException e) {
- throw convert(e);
- }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java
index fda23aa..00910a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java
@@ -35,7 +35,6 @@ import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.exceptions.ServiceNotReadyException;
-import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.slider.core.persist.AggregateConfSerDeser;
import org.apache.slider.core.persist.ConfTreeSerDeser;
@@ -51,8 +50,6 @@ import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
-import org.apache.slider.server.services.security.CertificateManager;
-import org.apache.slider.server.services.security.SecurityStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,16 +60,7 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.apache.slider.api.proto.RestTypeMarshalling.marshall;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_COMPONENTS;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_CONTAINERS;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_NODES;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_RESOURCES;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_DESIRED;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_DESIRED_APPCONF;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_DESIRED_RESOURCES;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_RESOLVED;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_RESOLVED_APPCONF;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_RESOLVED_RESOURCES;
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
/**
* Implement the {@link SliderClusterProtocol}.
@@ -90,7 +78,6 @@ public class SliderIPCService extends AbstractService
private final MetricsAndMonitoring metricsAndMonitoring;
private final AppMasterActionOperations amOperations;
private final ContentCache cache;
- private final CertificateManager certificateManager;
/**
* This is the prefix used for metrics
@@ -107,11 +94,8 @@ public class SliderIPCService extends AbstractService
* @param cache
*/
public SliderIPCService(AppMasterActionOperations amOperations,
- CertificateManager certificateManager,
- StateAccessForProviders state,
- QueueAccess actionQueues,
- MetricsAndMonitoring metricsAndMonitoring,
- ContentCache cache) {
+ StateAccessForProviders state, QueueAccess actionQueues,
+ MetricsAndMonitoring metricsAndMonitoring, ContentCache cache) {
super("SliderIPCService");
Preconditions.checkArgument(amOperations != null, "null amOperations");
Preconditions.checkArgument(state != null, "null appState");
@@ -124,7 +108,6 @@ public class SliderIPCService extends AbstractService
this.metricsAndMonitoring = metricsAndMonitoring;
this.amOperations = amOperations;
this.cache = cache;
- this.certificateManager = certificateManager;
}
@Override //SliderClusterProtocol
@@ -517,35 +500,4 @@ public class SliderIPCService extends AbstractService
builder.setJson(json);
return builder.build();
}
-
- @Override
- public Messages.GetCertificateStoreResponseProto getClientCertificateStore(Messages.GetCertificateStoreRequestProto request) throws
- IOException {
- String hostname = request.getHostname();
- String clientId = request.getRequesterId();
- String password = request.getPassword();
- String type = request.getType();
-
- SecurityStore store = null;
- try {
- if ( SecurityStore.StoreType.keystore.equals(
- SecurityStore.StoreType.valueOf(type))) {
- store = certificateManager.generateContainerKeystore(hostname,
- clientId,
- null,
- password);
- } else if (SecurityStore.StoreType.truststore.equals(
- SecurityStore.StoreType.valueOf(type))) {
- store = certificateManager.generateContainerTruststore(clientId,
- null,
- password);
-
- } else {
- throw new IOException("Illegal store type");
- }
- } catch (SliderException e) {
- throw new IOException(e);
- }
- return marshall(store);
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
index 65a3591..ea07a8a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
@@ -18,16 +18,11 @@ package org.apache.slider.server.appmaster.web;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.slider.providers.ProviderService;
-import org.apache.slider.server.appmaster.AppMasterActionOperations;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.RoleStatus;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
-import org.apache.slider.server.services.security.CertificateManager;
-
-import java.util.Map;
/**
* Interface to pass information from the Slider AppMaster to the WebApp
@@ -43,18 +38,6 @@ public interface WebAppApi {
* The {@link ProviderService} for the current cluster
*/
ProviderService getProviderService();
-
-
- /**
- * The {@link CertificateManager} for the current cluster
- */
- CertificateManager getCertificateManager();
-
- /**
- * Generate a mapping from role name to its {@link RoleStatus}. Be aware that this
- * is a computed value and not just a getter
- */
- Map<String, RoleStatus> getRoleStatusByName();
/**
* Registry operations accessor
@@ -75,12 +58,6 @@ public interface WebAppApi {
QueueAccess getQueues();
/**
- * API for AM operations
- * @return current operations implementation
- */
- AppMasterActionOperations getAMOperations();
-
- /**
* Local cache of content
* @return the cache
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
index bd4d2bf..d20f1ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
@@ -21,17 +21,11 @@ import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.AppMasterActionOperations;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
-import org.apache.slider.server.appmaster.state.RoleStatus;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
-import org.apache.slider.server.services.security.CertificateManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
import static com.google.common.base.Preconditions.checkNotNull;
/**
@@ -42,7 +36,6 @@ public class WebAppApiImpl implements WebAppApi {
protected final StateAccessForProviders appState;
protected final ProviderService provider;
- protected final CertificateManager certificateManager;
private final RegistryOperations registryOperations;
private final MetricsAndMonitoring metricsAndMonitoring;
private final QueueAccess queues;
@@ -50,13 +43,9 @@ public class WebAppApiImpl implements WebAppApi {
private final ContentCache contentCache;
public WebAppApiImpl(StateAccessForProviders appState,
- ProviderService provider,
- CertificateManager certificateManager,
- RegistryOperations registryOperations,
- MetricsAndMonitoring metricsAndMonitoring,
- QueueAccess queues,
- AppMasterActionOperations appMasterOperations,
- ContentCache contentCache) {
+ ProviderService provider, RegistryOperations registryOperations,
+ MetricsAndMonitoring metricsAndMonitoring, QueueAccess queues,
+ AppMasterActionOperations appMasterOperations, ContentCache contentCache) {
this.appMasterOperations = appMasterOperations;
this.contentCache = contentCache;
checkNotNull(appState);
@@ -66,7 +55,6 @@ public class WebAppApiImpl implements WebAppApi {
this.registryOperations = registryOperations;
this.appState = appState;
this.provider = provider;
- this.certificateManager = certificateManager;
this.metricsAndMonitoring = metricsAndMonitoring;
}
@@ -81,21 +69,6 @@ public class WebAppApiImpl implements WebAppApi {
}
@Override
- public CertificateManager getCertificateManager() {
- return certificateManager;
- }
-
- @Override
- public Map<String,RoleStatus> getRoleStatusByName() {
- List<RoleStatus> roleStatuses = appState.cloneRoleStatusList();
- Map<String, RoleStatus> map = new TreeMap<>();
- for (RoleStatus status : roleStatuses) {
- map.put(status.getName(), status);
- }
- return map;
- }
-
- @Override
public RegistryOperations getRegistryOperations() {
return registryOperations;
}
@@ -110,10 +83,6 @@ public class WebAppApiImpl implements WebAppApi {
return queues;
}
- @Override
- public AppMasterActionOperations getAMOperations() {
- return appMasterOperations;
- }
@Override
public ContentCache getContentCache() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/AbstractSecurityStoreGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/AbstractSecurityStoreGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/AbstractSecurityStoreGenerator.java
deleted file mode 100644
index 11d3aa1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/AbstractSecurityStoreGenerator.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.services.security;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.alias.CredentialProvider;
-import org.apache.hadoop.security.alias.CredentialProviderFactory;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.core.conf.MapOperations;
-import org.apache.slider.core.exceptions.SliderException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-/**
- *
- */
-public abstract class AbstractSecurityStoreGenerator implements
- SecurityStoreGenerator {
- private static final Logger LOG =
- LoggerFactory.getLogger(AbstractSecurityStoreGenerator.class);
-
- protected CertificateManager certificateMgr;
-
- public AbstractSecurityStoreGenerator(CertificateManager certificateMgr) {
- this.certificateMgr = certificateMgr;
- }
-
- protected String getStorePassword(Map<String, List<String>> credentials,
- MapOperations compOps, String role)
- throws SliderException, IOException {
- String password = getPassword(compOps);
- if (password == null) {
- // need to leverage credential provider
- String alias = getAlias(compOps);
- LOG.debug("Alias {} found for role {}", alias, role);
- if (alias == null) {
- throw new SliderException("No store password or credential provider "
- + "alias found");
- }
- if (credentials.isEmpty()) {
- LOG.info("Credentials can not be retrieved for store generation since "
- + "no CP paths are configured");
- }
- synchronized (this) {
- for (Map.Entry<String, List<String>> cred : credentials.entrySet()) {
- String provider = cred.getKey();
- Configuration c = new Configuration();
- c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
- LOG.debug("Configured provider {}", provider);
- CredentialProvider cp =
- CredentialProviderFactory.getProviders(c).get(0);
- LOG.debug("Aliases: {}", cp.getAliases());
- char[] credential = c.getPassword(alias);
- if (credential != null) {
- LOG.info("Credential found for role {}", role);
- return String.valueOf(credential);
- }
- }
- }
-
- if (password == null) {
- LOG.info("No store credential found for alias {}. "
- + "Generation of store for {} is not possible.", alias, role);
-
- }
- }
-
- return password;
-
- }
-
- @Override
- public boolean isStoreRequested(MapOperations compOps) {
- return compOps.getOptionBool(SliderKeys.COMP_STORES_REQUIRED_KEY, false);
- }
-
- abstract String getPassword(MapOperations compOps);
-
- abstract String getAlias(MapOperations compOps);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
deleted file mode 100644
index e436ae9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
+++ /dev/null
@@ -1,495 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.services.security;
-
-import com.google.inject.Singleton;
-import org.apache.commons.io.FileUtils;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.core.conf.MapOperations;
-import org.apache.slider.core.exceptions.SliderException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.nio.charset.Charset;
-import java.text.MessageFormat;
-
-@Singleton
-public class CertificateManager {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(CertificateManager.class);
-
- private static final String GEN_SRVR_KEY = "openssl genrsa -des3 " +
- "-passout pass:{0} -out {1}" + File.separator + "{2} 4096 ";
- private static final String GEN_SRVR_REQ = "openssl req -passin pass:{0} " +
- "-new -key {1}" + File.separator + "{2} -out {1}" + File.separator +
- "{5} -config {1}" + File.separator + "ca.config " +
- "-subj {6} -batch";
- private static final String SIGN_SRVR_CRT = "openssl ca -create_serial " +
- "-out {1}" + File.separator + "{3} -days 365 -keyfile {1}" + File.separator
- + "{2} -key {0} -selfsign -extensions jdk7_ca -config {1}" + File.separator
- + "ca.config -batch -infiles {1}" + File.separator + "{5}";
- private static final String EXPRT_KSTR = "openssl pkcs12 -export" +
- " -in {2}" + File.separator + "{4} -inkey {2}" + File.separator +
- "{3} -certfile {2}" + File.separator + "{4} -out {2}" + File.separator +
- "{5} -password pass:{1} -passin pass:{0} \n";
- private static final String REVOKE_AGENT_CRT = "openssl ca " +
- "-config {0}" + File.separator + "ca.config -keyfile {0}" +
- File.separator + "{4} -revoke {0}" + File.separator + "{2} -batch " +
- "-passin pass:{3} -cert {0}" + File.separator + "{5}";
- private static final String SIGN_AGENT_CRT = "openssl ca -config " +
- "{0}" + File.separator + "ca.config -in {0}" + File.separator +
- "{1} -out {0}" + File.separator + "{2} -batch -passin pass:{3} " +
- "-keyfile {0}" + File.separator + "{4} -cert {0}" + File.separator + "{5}";
- private static final String GEN_AGENT_KEY="openssl req -new -newkey " +
- "rsa:1024 -nodes -keyout {0}" + File.separator +
- "{2}.key -subj {1} -out {0}" + File.separator + "{2}.csr " +
- "-config {3}" + File.separator + "ca.config ";
- private String passphrase;
- private String applicationName;
-
-
- public void initialize(MapOperations compOperations) throws SliderException {
- String hostname = null;
- try {
- hostname = InetAddress.getLocalHost().getCanonicalHostName();
- } catch (UnknownHostException e) {
- hostname = "localhost";
- }
- this.initialize(compOperations, hostname, null, null);
- }
-
- /**
- * Verify that root certificate exists, generate it otherwise.
- */
- public void initialize(MapOperations compOperations,
- String hostname, String containerId,
- String appName) throws SliderException {
- SecurityUtils.initializeSecurityParameters(compOperations);
-
- LOG.info("Initialization of root certificate");
- boolean certExists = isCertExists();
- LOG.info("Certificate exists:" + certExists);
-
- this.applicationName = appName;
-
- if (!certExists) {
- generateAMKeystore(hostname, containerId);
- }
-
- }
-
- /**
- * Checks root certificate state.
- * @return "true" if certificate exists
- */
- private boolean isCertExists() {
-
- String srvrKstrDir = SecurityUtils.getSecurityDir();
- String srvrCrtName = SliderKeys.CRT_FILE_NAME;
- File certFile = new File(srvrKstrDir + File.separator + srvrCrtName);
- LOG.debug("srvrKstrDir = " + srvrKstrDir);
- LOG.debug("srvrCrtName = " + srvrCrtName);
- LOG.debug("certFile = " + certFile.getAbsolutePath());
-
- return certFile.exists();
- }
-
- public void setPassphrase(String passphrase) {
- this.passphrase = passphrase;
- }
-
- class StreamConsumer extends Thread
- {
- InputStream is;
- boolean logOutput;
-
- StreamConsumer(InputStream is, boolean logOutput)
- {
- this.is = is;
- this.logOutput = logOutput;
- }
-
- StreamConsumer(InputStream is)
- {
- this(is, false);
- }
-
- public void run()
- {
- try
- {
- InputStreamReader isr = new InputStreamReader(is,
- Charset.forName("UTF8"));
- BufferedReader br = new BufferedReader(isr);
- String line;
- while ( (line = br.readLine()) != null)
- if (logOutput) {
- LOG.info(line);
- }
- } catch (IOException e)
- {
- LOG.error("Error during processing of process stream", e);
- }
- }
- }
-
-
- /**
- * Runs os command
- *
- * @return command execution exit code
- */
- private int runCommand(String command) throws SliderException {
- int exitCode = -1;
- String line = null;
- Process process = null;
- BufferedReader br= null;
- try {
- process = Runtime.getRuntime().exec(command);
- StreamConsumer outputConsumer =
- new StreamConsumer(process.getInputStream(), true);
- StreamConsumer errorConsumer =
- new StreamConsumer(process.getErrorStream(), true);
-
- outputConsumer.start();
- errorConsumer.start();
-
- try {
- process.waitFor();
- SecurityUtils.logOpenSslExitCode(command, process.exitValue());
- exitCode = process.exitValue();
- if (exitCode != 0) {
- throw new SliderException(exitCode, "Error running command %s", command);
- }
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- } catch (IOException e) {
- e.printStackTrace();
- } finally {
- if (br != null) {
- try {
- br.close();
- } catch (IOException ioe) {
- ioe.printStackTrace();
- }
- }
- }
-
- return exitCode;//some exception occurred
-
- }
-
- public synchronized void generateContainerCertificate(String hostname,
- String identifier) {
- LOG.info("Generation of certificate for {}", hostname);
-
- String srvrKstrDir = SecurityUtils.getSecurityDir();
- Object[] scriptArgs = {srvrKstrDir, getSubjectDN(hostname, identifier,
- this.applicationName), identifier, SecurityUtils.getSecurityDir()};
-
- try {
- String command = MessageFormat.format(GEN_AGENT_KEY, scriptArgs);
- runCommand(command);
-
- signAgentCertificate(identifier);
-
- } catch (SliderException e) {
- LOG.error("Error generating the agent certificate", e);
- }
- }
-
- public synchronized SecurityStore generateContainerKeystore(String hostname,
- String requesterId,
- String role,
- String keystorePass)
- throws SliderException {
- LOG.info("Generation of container keystore for container {} on {}",
- requesterId, hostname);
-
- generateContainerCertificate(hostname, requesterId);
-
- // come up with correct args to invoke keystore command
- String srvrCrtPass = SecurityUtils.getKeystorePass();
- String srvrKstrDir = SecurityUtils.getSecurityDir();
- String containerCrtName = requesterId + ".crt";
- String containerKeyName = requesterId + ".key";
- String kstrName = getKeystoreFileName(requesterId, role);
-
- Object[] scriptArgs = {srvrCrtPass, keystorePass, srvrKstrDir,
- containerKeyName, containerCrtName, kstrName};
-
- String command = MessageFormat.format(EXPRT_KSTR, scriptArgs);
- runCommand(command);
-
- return new SecurityStore(new File(srvrKstrDir, kstrName),
- SecurityStore.StoreType.keystore);
- }
-
- private static String getKeystoreFileName(String containerId,
- String role) {
- return String.format("keystore-%s-%s.p12", containerId,
- role != null ? role : "");
- }
-
- private void generateAMKeystore(String hostname, String containerId)
- throws SliderException {
- LOG.info("Generation of server certificate");
-
- String srvrKstrDir = SecurityUtils.getSecurityDir();
- String srvrCrtName = SliderKeys.CRT_FILE_NAME;
- String srvrCsrName = SliderKeys.CSR_FILE_NAME;
- String srvrKeyName = SliderKeys.KEY_FILE_NAME;
- String kstrName = SliderKeys.KEYSTORE_FILE_NAME;
- String srvrCrtPass = SecurityUtils.getKeystorePass();
-
- Object[] scriptArgs = {srvrCrtPass, srvrKstrDir, srvrKeyName,
- srvrCrtName, kstrName, srvrCsrName, getSubjectDN(hostname, containerId,
- this.applicationName)};
-
- String command = MessageFormat.format(GEN_SRVR_KEY, scriptArgs);
- runCommand(command);
-
- command = MessageFormat.format(GEN_SRVR_REQ, scriptArgs);
- runCommand(command);
-
- command = MessageFormat.format(SIGN_SRVR_CRT, scriptArgs);
- runCommand(command);
-
- Object[] keystoreArgs = {srvrCrtPass, srvrCrtPass, srvrKstrDir, srvrKeyName,
- srvrCrtName, kstrName, srvrCsrName};
- command = MessageFormat.format(EXPRT_KSTR, keystoreArgs);
- runCommand(command);
- }
-
- public SecurityStore generateContainerTruststore(String containerId,
- String role,
- String truststorePass)
- throws SliderException {
-
- String srvrKstrDir = SecurityUtils.getSecurityDir();
- String srvrCrtName = SliderKeys.CRT_FILE_NAME;
- String srvrCsrName = SliderKeys.CSR_FILE_NAME;
- String srvrKeyName = SliderKeys.KEY_FILE_NAME;
- String kstrName = getTruststoreFileName(role, containerId);
- String srvrCrtPass = SecurityUtils.getKeystorePass();
-
- Object[] scriptArgs = {srvrCrtPass, truststorePass, srvrKstrDir, srvrKeyName,
- srvrCrtName, kstrName, srvrCsrName};
-
- String command = MessageFormat.format(EXPRT_KSTR, scriptArgs);
- runCommand(command);
-
- return new SecurityStore(new File(srvrKstrDir, kstrName),
- SecurityStore.StoreType.truststore);
- }
-
- private static String getTruststoreFileName(String role, String containerId) {
- return String.format("truststore-%s-%s.p12", containerId,
- role != null ? role : "");
- }
-
- /**
- * Returns server certificate content
- * @return string with server certificate content
- */
- public String getServerCert() {
- File certFile = getServerCertficateFilePath();
- String srvrCrtContent = null;
- try {
- srvrCrtContent = FileUtils.readFileToString(certFile);
- } catch (IOException e) {
- LOG.error(e.getMessage());
- }
- return srvrCrtContent;
- }
-
- public static File getServerCertficateFilePath() {
- return new File(String.format("%s%s%s",
- SecurityUtils.getSecurityDir(),
- File.separator,
- SliderKeys.CRT_FILE_NAME));
- }
-
- public static File getAgentCertficateFilePath(String containerId) {
- return new File(String.format("%s%s%s.crt",
- SecurityUtils.getSecurityDir(),
- File.separator,
- containerId));
- }
-
- public static File getContainerKeystoreFilePath(String containerId,
- String role) {
- return new File(SecurityUtils.getSecurityDir(), getKeystoreFileName(
- containerId,
- role
- ));
- }
-
- public static File getContainerTruststoreFilePath(String role,
- String containerId) {
- return new File(SecurityUtils.getSecurityDir(),
- getTruststoreFileName(role, containerId));
- }
-
- public static File getAgentKeyFilePath(String containerId) {
- return new File(String.format("%s%s%s.key",
- SecurityUtils.getSecurityDir(),
- File.separator,
- containerId));
- }
-
- /**
- * Signs agent certificate
- * Adds agent certificate to server keystore
- * @return string with agent signed certificate content
- */
- public synchronized SignCertResponse signAgentCrt(String agentHostname,
- String agentCrtReqContent,
- String passphraseAgent) {
- SignCertResponse response = new SignCertResponse();
- LOG.info("Signing of agent certificate");
- LOG.info("Verifying passphrase");
-
- if (!this.passphrase.equals(passphraseAgent.trim())) {
- LOG.warn("Incorrect passphrase from the agent");
- response.setResult(SignCertResponse.ERROR_STATUS);
- response.setMessage("Incorrect passphrase from the agent");
- return response;
- }
-
- String srvrKstrDir = SecurityUtils.getSecurityDir();
- String srvrCrtPass = SecurityUtils.getKeystorePass();
- String srvrCrtName = SliderKeys.CRT_FILE_NAME;
- String srvrKeyName = SliderKeys.KEY_FILE_NAME;
- String agentCrtReqName = agentHostname + ".csr";
- String agentCrtName = agentHostname + ".crt";
-
- Object[] scriptArgs = {srvrKstrDir, agentCrtReqName, agentCrtName,
- srvrCrtPass, srvrKeyName, srvrCrtName};
-
- //Revoke previous agent certificate if exists
- File agentCrtFile = new File(srvrKstrDir + File.separator + agentCrtName);
-
- String command = null;
- if (agentCrtFile.exists()) {
- LOG.info("Revoking of " + agentHostname + " certificate.");
- command = MessageFormat.format(REVOKE_AGENT_CRT, scriptArgs);
- try {
- runCommand(command);
- } catch (SliderException e) {
- int commandExitCode = e.getExitCode();
- response.setResult(SignCertResponse.ERROR_STATUS);
- response.setMessage(
- SecurityUtils.getOpenSslCommandResult(command, commandExitCode));
- return response;
- }
- }
-
- File agentCrtReqFile = new File(srvrKstrDir + File.separator +
- agentCrtReqName);
- try {
- FileUtils.writeStringToFile(agentCrtReqFile, agentCrtReqContent);
- } catch (IOException e1) {
- // TODO Auto-generated catch block
- e1.printStackTrace();
- }
-
- command = MessageFormat.format(SIGN_AGENT_CRT, scriptArgs);
-
- LOG.debug(SecurityUtils.hideOpenSslPassword(command));
- try {
- runCommand(command);
- } catch (SliderException e) {
- int commandExitCode = e.getExitCode();
- response.setResult(SignCertResponse.ERROR_STATUS);
- response.setMessage(
- SecurityUtils.getOpenSslCommandResult(command, commandExitCode));
- return response;
- }
-
- String agentCrtContent = "";
- try {
- agentCrtContent = FileUtils.readFileToString(agentCrtFile);
- } catch (IOException e) {
- e.printStackTrace();
- LOG.error("Error reading signed agent certificate");
- response.setResult(SignCertResponse.ERROR_STATUS);
- response.setMessage("Error reading signed agent certificate");
- return response;
- }
- response.setResult(SignCertResponse.OK_STATUS);
- response.setSignedCa(agentCrtContent);
- //LOG.info(ShellCommandUtil.getOpenSslCommandResult(command, commandExitCode));
- return response;
- }
-
- private String signAgentCertificate (String containerId)
- throws SliderException {
- String srvrKstrDir = SecurityUtils.getSecurityDir();
- String srvrCrtPass = SecurityUtils.getKeystorePass();
- String srvrCrtName = SliderKeys.CRT_FILE_NAME;
- String srvrKeyName = SliderKeys.KEY_FILE_NAME;
- String agentCrtReqName = containerId + ".csr";
- String agentCrtName = containerId + ".crt";
-
- // server certificate must exist already
- if (!(new File(srvrKstrDir, srvrCrtName).exists())) {
- throw new SliderException("CA certificate not generated");
- }
-
- Object[] scriptArgs = {srvrKstrDir, agentCrtReqName, agentCrtName,
- srvrCrtPass, srvrKeyName, srvrCrtName};
-
- //Revoke previous agent certificate if exists
- File agentCrtFile = new File(srvrKstrDir + File.separator + agentCrtName);
-
- String command;
- if (agentCrtFile.exists()) {
- LOG.info("Revoking of " + containerId + " certificate.");
- command = MessageFormat.format(REVOKE_AGENT_CRT, scriptArgs);
- runCommand(command);
- }
-
- command = MessageFormat.format(SIGN_AGENT_CRT, scriptArgs);
-
- LOG.debug(SecurityUtils.hideOpenSslPassword(command));
- runCommand(command);
-
- return agentCrtName;
-
- }
-
- private String getSubjectDN(String hostname, String containerId,
- String appName) {
- return String.format("/CN=%s%s%s",
- hostname,
- containerId != null ? "/OU=" + containerId : "",
- appName != null ? "/OU=" + appName : "");
-
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/KeystoreGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/KeystoreGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/KeystoreGenerator.java
deleted file mode 100644
index e2339d5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/KeystoreGenerator.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.services.security;
-
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.MapOperations;
-import org.apache.slider.core.exceptions.SliderException;
-
-import java.io.File;
-import java.io.IOException;
-
-/**
- *
- */
-public class KeystoreGenerator extends AbstractSecurityStoreGenerator {
-
-
- public KeystoreGenerator(CertificateManager certificateMgr) {
- super(certificateMgr);
- }
-
- @Override
- public SecurityStore generate(String hostname, String containerId,
- AggregateConf instanceDefinition,
- MapOperations compOps, String role)
- throws SliderException, IOException {
- SecurityStore keystore = null;
- String password = getStorePassword(
- instanceDefinition.getAppConf().credentials, compOps, role);
- if (password != null) {
- keystore =
- certificateMgr.generateContainerKeystore(hostname, containerId, role,
- password);
- }
- return keystore;
- }
-
- @Override
- String getPassword(MapOperations compOps) {
- return compOps.get(
- compOps.get(SliderKeys.COMP_KEYSTORE_PASSWORD_PROPERTY_KEY));
- }
-
- @Override
- String getAlias(MapOperations compOps) {
- return compOps.getOption(SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_KEY,
- SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_DEFAULT);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStore.java
deleted file mode 100644
index fc54267..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStore.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.services.security;
-
-import java.io.File;
-
-/**
- *
- */
-public class SecurityStore {
- private File file;
-
- public enum StoreType {truststore, keystore}
-
- private StoreType type;
-
- public String getType() {
- return type.name();
- }
-
- public File getFile() {
- return file;
- }
-
- public SecurityStore(File file,
- StoreType type) {
-
- this.file = file;
- this.type = type;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- SecurityStore that = (SecurityStore) o;
-
- if (file != null ? !file.equals(that.file) : that.file != null)
- return false;
- if (type != that.type) return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = file != null ? file.hashCode() : 0;
- result = 31 * result + (type != null ? type.hashCode() : 0);
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStoreGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStoreGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStoreGenerator.java
deleted file mode 100644
index a814988..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStoreGenerator.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.services.security;
-
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.MapOperations;
-import org.apache.slider.core.exceptions.SliderException;
-
-import java.io.File;
-import java.io.IOException;
-
-/**
- *
- */
-public interface SecurityStoreGenerator {
-
- SecurityStore generate(String hostname,
- String containerId,
- AggregateConf instanceDefinition,
- MapOperations compOps,
- String role)
- throws SliderException, IOException;
-
- boolean isStoreRequested(MapOperations compOps);
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/51] [abbrv] hadoop git commit: YARN-5944. Native services AM
should remain up if RM is down. Contributed by Billie Rinaldi
Posted by ji...@apache.org.
YARN-5944. Native services AM should remain up if RM is down. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/03a32582
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/03a32582
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/03a32582
Branch: refs/heads/yarn-native-services
Commit: 03a3258229e7b3f4996f5658bde53ecc186c12bb
Parents: 23395f7
Author: Gour Saha <go...@apache.org>
Authored: Thu Dec 1 00:30:01 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../org/apache/slider/server/appmaster/SliderAppMaster.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03a32582/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 34b6a7d..8c39343 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -705,6 +705,11 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
synchronized (appState) {
int heartbeatInterval = HEARTBEAT_INTERVAL;
+ // configure AM to wait forever for RM
+ getConfig().setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
+ -1);
+ getConfig().unset(YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS);
+
// add the RM client -this brings the callbacks in
asyncRMClient = AMRMClientAsync.createAMRMClientAsync(heartbeatInterval, this);
addService(asyncRMClient);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/51] [abbrv] hadoop git commit: YARN-5778. Add .keep file for yarn
native services AM web app. Contributed by Billie Rinaldi
Posted by ji...@apache.org.
YARN-5778. Add .keep file for yarn native services AM web app. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cff3b517
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cff3b517
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cff3b517
Branch: refs/heads/yarn-native-services
Commit: cff3b51714a3dcb64e7bb13aa38e80cf2203c85b
Parents: 98e3497
Author: Gour Saha <go...@apache.org>
Authored: Tue Oct 25 10:06:16 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../src/main/resources/webapps/slideram/.keep | 0
1 file changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cff3b517/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/webapps/slideram/.keep
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/webapps/slideram/.keep b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/webapps/slideram/.keep
new file mode 100644
index 0000000..e69de29
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/51] [abbrv] hadoop git commit: YARN-5813. Slider should not try
to set a negative lifetime timeout value. Contributed by Jian He
Posted by ji...@apache.org.
YARN-5813. Slider should not try to set a negative lifetime timeout value. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f83fef4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f83fef4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f83fef4
Branch: refs/heads/yarn-native-services
Commit: 1f83fef4b8b41a3d88099006718a45c880744765
Parents: a79a070
Author: Gour Saha <go...@apache.org>
Authored: Tue Nov 1 17:39:54 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/slider/client/SliderClient.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f83fef4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index d1f88c5..ea10ed0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -2120,7 +2120,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
amLauncher.setKeepContainersOverRestarts(true);
// set lifetime in submission context;
Map<ApplicationTimeoutType, Long> appTimeout = new HashMap<>();
- appTimeout.put(ApplicationTimeoutType.LIFETIME, lifetime);
+ if (lifetime >= 0) {
+ appTimeout.put(ApplicationTimeoutType.LIFETIME, lifetime);
+ }
amLauncher.submissionContext.setApplicationTimeouts(appTimeout);
int maxAppAttempts = config.getInt(KEY_AM_RESTART_LIMIT, 0);
amLauncher.setMaxAppAttempts(maxAppAttempts);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/51] [abbrv] hadoop git commit: YARN-5812. Exception during GET
call - "Failed to retrieve application: null". Contributed by Gour Saha
Posted by ji...@apache.org.
YARN-5812. Exception during GET call - "Failed to retrieve application: null". Contributed by Gour Saha
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a55eb5d2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a55eb5d2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a55eb5d2
Branch: refs/heads/yarn-native-services
Commit: a55eb5d22375e85b81fa7b8554f23a8093074467
Parents: 1f83fef
Author: Jian He <ji...@apache.org>
Authored: Wed Nov 2 15:55:48 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../api/impl/ApplicationApiService.java | 36 ++++++++++++++------
1 file changed, 26 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55eb5d2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index 37bd134..6db69ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -50,7 +50,6 @@ import javax.ws.rs.core.Response.Status;
import org.apache.commons.lang.SerializationUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -905,6 +904,10 @@ public class ApplicationApiService implements ApplicationApi {
// state
String appState = jsonGetAsString(appStatus, "state");
+ if (appState == null) {
+ // consider that app is still in ACCEPTED state
+ appState = String.valueOf(StateValues.STATE_INCOMPLETE);
+ }
switch (Integer.parseInt(appState)) {
case StateValues.STATE_LIVE:
app.setState(ApplicationState.STARTED);
@@ -1069,6 +1072,9 @@ public class ApplicationApiService implements ApplicationApi {
String status = null;
try {
status = sliderClient.actionStatus(appName);
+ } catch (BadClusterStateException e) {
+ logger.warn("Application not running yet", e);
+ return EMPTY_JSON_OBJECT;
} catch (Exception e) {
logger.error("Exception calling slider.actionStatus", e);
return EMPTY_JSON_OBJECT;
@@ -1097,7 +1103,7 @@ public class ApplicationApiService implements ApplicationApi {
try {
registry = sliderClient.actionRegistryGetConfig(registryArgs)
.asJson();
- } catch (FileNotFoundException | PathNotFoundException e) {
+ } catch (FileNotFoundException | NotFoundException e) {
// ignore and return empty object
return EMPTY_JSON_OBJECT;
} catch (Exception e) {
@@ -1192,23 +1198,33 @@ public class ApplicationApiService implements ApplicationApi {
// little longer for it to stop from YARN point of view. Slider destroy
// fails if the application is not completely stopped. Hence the need to
// call destroy in a controlled loop few times (only if exit code is
- // EXIT_APPLICATION_IN_USE), before giving up.
+ // EXIT_APPLICATION_IN_USE or EXIT_INSTANCE_EXISTS), before giving up.
boolean keepTrying = true;
- int maxDeleteAttempt = 5;
- int deleteAttempt = 0;
- while (keepTrying && deleteAttempt < maxDeleteAttempt) {
+ int maxDeleteAttempts = 5;
+ int deleteAttempts = 0;
+ int sleepIntervalInMillis = 500;
+ while (keepTrying && deleteAttempts < maxDeleteAttempts) {
try {
destroySliderApplication(appName);
keepTrying = false;
} catch (SliderException e) {
- logger.error("Delete application threw exception", e);
- if (e.getExitCode() == SliderExitCodes.EXIT_APPLICATION_IN_USE) {
- deleteAttempt++;
+ if (e.getExitCode() == SliderExitCodes.EXIT_APPLICATION_IN_USE
+ || e.getExitCode() == SliderExitCodes.EXIT_INSTANCE_EXISTS) {
+ deleteAttempts++;
+ // If we used up all the allowed delete attempts, let's log it as
+ // error before giving up. Otherwise log as warn.
+ if (deleteAttempts < maxDeleteAttempts) {
+ logger.warn("Application not in stopped state, waiting for {}ms"
+ + " before trying delete again", sleepIntervalInMillis);
+ } else {
+ logger.error("Delete application failed", e);
+ }
try {
- Thread.sleep(500);
+ Thread.sleep(sleepIntervalInMillis);
} catch (InterruptedException e1) {
}
} else {
+ logger.error("Delete application threw exception", e);
return Response.status(Status.INTERNAL_SERVER_ERROR).build();
}
} catch (Exception e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/51] [abbrv] hadoop git commit: YARN-5701. Fix issues in yarn
native services apps-of-apps. Contributed by Billie Rinaldi
Posted by ji...@apache.org.
YARN-5701. Fix issues in yarn native services apps-of-apps. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/98e34978
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/98e34978
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/98e34978
Branch: refs/heads/yarn-native-services
Commit: 98e349784bc507e219c46149e593e4e4e7b069e3
Parents: de70e98
Author: Jian He <ji...@apache.org>
Authored: Sun Oct 16 17:01:09 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 72 ++++++++--------
.../slider/core/buildutils/InstanceBuilder.java | 4 +
.../apache/slider/providers/ProviderUtils.java | 62 ++++++++++++--
.../providers/docker/DockerClientProvider.java | 4 +-
.../providers/docker/DockerProviderService.java | 87 ++++++++++++++++----
5 files changed, 164 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98e34978/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 2840c4b..94e51e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -178,6 +178,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
+import java.io.Console;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
@@ -918,57 +919,56 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
return;
}
- BufferedReader br = null;
- try {
- for (Entry<String, List<String>> cred : tree.credentials.entrySet()) {
- String provider = cred.getKey()
- .replaceAll(Pattern.quote("${CLUSTER_NAME}"), clusterName)
- .replaceAll(Pattern.quote("${CLUSTER}"), clusterName);
- List<String> aliases = cred.getValue();
- if (aliases == null || aliases.isEmpty()) {
- continue;
- }
- Configuration c = new Configuration(conf);
- c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
- CredentialProvider credentialProvider = CredentialProviderFactory.getProviders(c).get(0);
- Set<String> existingAliases = new HashSet<>(credentialProvider.getAliases());
- for (String alias : aliases) {
- if (existingAliases.contains(alias.toLowerCase(Locale.ENGLISH))) {
- log.info("Credentials for " + alias + " found in " + provider);
- } else {
- if (br == null) {
- br = new BufferedReader(new InputStreamReader(System.in));
- }
- char[] pass = readPassword(alias, br);
- credentialProvider.createCredentialEntry(alias, pass);
- credentialProvider.flush();
- Arrays.fill(pass, ' ');
+ Console console = System.console();
+ for (Entry<String, List<String>> cred : tree.credentials.entrySet()) {
+ String provider = cred.getKey()
+ .replaceAll(Pattern.quote("${CLUSTER_NAME}"), clusterName)
+ .replaceAll(Pattern.quote("${CLUSTER}"), clusterName);
+ List<String> aliases = cred.getValue();
+ if (aliases == null || aliases.isEmpty()) {
+ continue;
+ }
+ Configuration c = new Configuration(conf);
+ c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
+ CredentialProvider credentialProvider = CredentialProviderFactory.getProviders(c).get(0);
+ Set<String> existingAliases = new HashSet<>(credentialProvider.getAliases());
+ for (String alias : aliases) {
+ if (existingAliases.contains(alias.toLowerCase(Locale.ENGLISH))) {
+ log.info("Credentials for " + alias + " found in " + provider);
+ } else {
+ if (console == null) {
+ throw new IOException("Unable to input password for " + alias +
+ " because System.console() is null; provider " + provider +
+ " must be populated manually");
}
+ char[] pass = readPassword(alias, console);
+ credentialProvider.createCredentialEntry(alias, pass);
+ credentialProvider.flush();
+ Arrays.fill(pass, ' ');
}
}
- } finally {
- org.apache.hadoop.io.IOUtils.closeStream(br);
}
}
private static char[] readOnePassword(String alias) throws IOException {
- try(BufferedReader br = new BufferedReader(new InputStreamReader(System.in))) {
- return readPassword(alias, br);
+ Console console = System.console();
+ if (console == null) {
+ throw new IOException("Unable to input password for " + alias +
+ " because System.console() is null");
}
+ return readPassword(alias, console);
}
- // using a normal reader instead of a secure one,
- // because stdin is not hooked up to the command line
- private static char[] readPassword(String alias, BufferedReader br)
+ private static char[] readPassword(String alias, Console console)
throws IOException {
char[] cred = null;
boolean noMatch;
do {
- log.info(String.format("%s %s: ", PASSWORD_PROMPT, alias));
- char[] newPassword1 = br.readLine().toCharArray();
- log.info(String.format("%s %s again: ", PASSWORD_PROMPT, alias));
- char[] newPassword2 = br.readLine().toCharArray();
+ console.printf("%s %s: \n", PASSWORD_PROMPT, alias);
+ char[] newPassword1 = console.readPassword();
+ console.printf("%s %s again: \n", PASSWORD_PROMPT, alias);
+ char[] newPassword2 = console.readPassword();
noMatch = !Arrays.equals(newPassword1, newPassword2);
if (noMatch) {
if (newPassword1 != null) Arrays.fill(newPassword1, ' ');
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98e34978/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java
index 25c65fc..f0686af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/InstanceBuilder.java
@@ -439,6 +439,10 @@ public class InstanceBuilder {
}
log.info("External appdefs after {}: {}", component, externalAppDefs);
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(
+ appConf.getConfTree().credentials,
+ componentAppConf.getConfTree().credentials);
+
mergeExternalComponent(appConf, componentAppConf, component, null);
mergeExternalComponent(resources, componentConf.getResourceOperations(),
component, getNextPriority());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98e34978/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index 47556f0..c5e6782 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -308,6 +308,7 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
public Map<String, String> filterSiteOptions(Map<String, String> options,
Map<String, String> tokenMap) {
String prefix = OptionKeys.SITE_XML_PREFIX;
+ String format = "${%s}";
Map<String, String> filteredOptions = new HashMap<>();
for (Map.Entry<String, String> entry : options.entrySet()) {
String key = entry.getKey();
@@ -319,7 +320,7 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
token.getValue());
}
}
- filteredOptions.put(key, value);
+ filteredOptions.put(String.format(format, key), value);
}
}
return filteredOptions;
@@ -545,9 +546,14 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
* @param clusterName app name
* @throws IOException file cannot be created
*/
- private void createConfigFile(SliderFileSystem fileSystem, File file,
- ConfigFormat configFormat, String configFileDN,
+ private synchronized void createConfigFile(SliderFileSystem fileSystem,
+ File file, ConfigFormat configFormat, String configFileDN,
Map<String, String> config, String clusterName) throws IOException {
+ if (file.exists()) {
+ log.info("Skipping writing {} file {} because it already exists",
+ configFormat, file);
+ return;
+ }
log.info("Writing {} file {}", configFormat, file);
ConfigUtils.prepConfigForTemplateOutputter(configFormat, config,
@@ -643,11 +649,10 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
String fileName = ConfigUtils.replaceProps(config, configFileName);
File localFile = new File(RESOURCE_DIR);
if (!localFile.exists()) {
- if (!localFile.mkdir()) {
+ if (!localFile.mkdir() && !localFile.exists()) {
throw new IOException(RESOURCE_DIR + " could not be created!");
}
}
- localFile = new File(localFile, new File(fileName).getName());
String folder = null;
if ("true".equals(config.get(PER_COMPONENT))) {
@@ -655,12 +660,25 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
} else if ("true".equals(config.get(PER_GROUP))) {
folder = roleGroup;
}
+ if (folder != null) {
+ localFile = new File(localFile, folder);
+ if (!localFile.exists()) {
+ if (!localFile.mkdir() && !localFile.exists()) {
+ throw new IOException(localFile + " could not be created!");
+ }
+ }
+ }
+ localFile = new File(localFile, new File(fileName).getName());
log.info("Localizing {} configs to config file {} (destination {}) " +
"based on {} configs", config.size(), localFile, fileName,
configFileDN);
- createConfigFile(fileSystem, localFile, configFormat, configFileDN, config,
- clusterName);
+ if (!localFile.exists()) {
+ createConfigFile(fileSystem, localFile, configFormat, configFileDN,
+ config, clusterName);
+ } else {
+ log.info("Local {} file {} already exists", configFormat, localFile);
+ }
Path destPath = uploadResource(localFile, fileSystem, folder, clusterName);
LocalResource configResource = fileSystem.createAmResource(destPath,
LocalResourceType.FILE);
@@ -807,12 +825,12 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
*/
public Map<String, Map<String, String>> buildConfigurations(
ConfTreeOperations appConf, ConfTreeOperations internalsConf,
- String containerId, String roleName, String roleGroup,
+ String containerId, String clusterName, String roleName, String roleGroup,
StateAccessForProviders amState) {
Map<String, Map<String, String>> configurations = new TreeMap<>();
Map<String, String> tokens = getStandardTokenMap(appConf,
- internalsConf, roleName, roleGroup, containerId);
+ internalsConf, roleName, roleGroup, containerId, clusterName);
Set<String> configs = new HashSet<>();
configs.addAll(getApplicationConfigurationTypes(roleGroup, appConf));
@@ -1164,6 +1182,32 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
}
/**
+ * Return a list of hostnames based on current ClusterNodes.
+ * @param values cluster nodes
+ * @return list of hosts
+ */
+ public Iterable<String> getHostNamesList(Collection<ClusterNode> values) {
+ List<String> hosts = new ArrayList<>();
+ for (ClusterNode cn : values) {
+ hosts.add(cn.hostname);
+ }
+ return hosts;
+ }
+
+ /**
+ * Return a list of IPs based on current ClusterNodes.
+ * @param values cluster nodes
+ * @return list of hosts
+ */
+ public Iterable<String> getIPsList(Collection<ClusterNode> values) {
+ List<String> hosts = new ArrayList<>();
+ for (ClusterNode cn : values) {
+ hosts.add(cn.ip);
+ }
+ return hosts;
+ }
+
+ /**
* Update ServiceRecord in Registry with IP and hostname.
* @param amState access to AM state
* @param yarnRegistry acces to YARN registry
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98e34978/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
index 13473e5..d554427 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
@@ -82,8 +82,8 @@ public class DockerClientProvider extends AbstractClientProvider
if (appConf.getComponentOptBool(roleGroup, AM_CONFIG_GENERATION, false)) {
// build and localize configuration files
Map<String, Map<String, String>> configurations =
- providerUtils.buildConfigurations(appConf, appConf, null, roleGroup,
- roleGroup, null);
+ providerUtils.buildConfigurations(appConf, appConf, null,
+ null, roleGroup, roleGroup, null);
try {
providerUtils.localizeConfigFiles(null, roleGroup, roleGroup, appConf,
configurations, null, fs, null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98e34978/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
index bebb5f0..af36620 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
@@ -41,6 +41,7 @@ import org.apache.slider.core.registry.docstore.ConfigFormat;
import org.apache.slider.core.registry.docstore.ConfigUtils;
import org.apache.slider.core.registry.docstore.ExportEntry;
import org.apache.slider.providers.AbstractProviderService;
+import org.apache.slider.providers.MonitorDetail;
import org.apache.slider.providers.ProviderCore;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderUtils;
@@ -62,6 +63,9 @@ import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Scanner;
+import java.util.regex.Pattern;
+
+import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
public class DockerProviderService extends AbstractProviderService implements
ProviderCore,
@@ -130,10 +134,13 @@ public class DockerProviderService extends AbstractProviderService implements
DOCKER_USE_PRIVILEGED, false));
// Set the environment
- launcher.putEnv(SliderUtils.buildEnvMap(appComponent,
- providerUtils.getStandardTokenMap(getAmState().getAppConfSnapshot(),
- getAmState().getInternalsSnapshot(), roleName, roleGroup,
- getClusterName())));
+ Map<String, String> standardTokens = providerUtils.getStandardTokenMap(
+ getAmState().getAppConfSnapshot(), getAmState().getInternalsSnapshot(),
+ roleName, roleGroup, container.getId().toString(), getClusterName());
+ Map<String, String> replaceTokens = providerUtils.filterSiteOptions(
+ appConf.getComponent(roleGroup).options, standardTokens);
+ replaceTokens.putAll(standardTokens);
+ launcher.putEnv(SliderUtils.buildEnvMap(appComponent, replaceTokens));
String workDir = ApplicationConstants.Environment.PWD.$();
launcher.setEnv("WORK_DIR", workDir);
@@ -169,8 +176,8 @@ public class DockerProviderService extends AbstractProviderService implements
providerUtils.buildConfigurations(
instanceDefinition.getAppConfOperations(),
instanceDefinition.getInternalOperations(),
- container.getId().toString(), roleName, roleGroup,
- getAmState());
+ container.getId().toString(), getClusterName(),
+ roleName, roleGroup, getAmState());
providerUtils.localizeConfigFiles(launcher, roleName, roleGroup,
appConf, configurations, launcher.getEnv(), fileSystem,
getClusterName());
@@ -251,8 +258,8 @@ public class DockerProviderService extends AbstractProviderService implements
// build and localize configuration files
Map<String, Map<String, String>> configurations =
providerUtils.buildConfigurations(appConf, getAmState()
- .getInternalsSnapshot(), null, clientName, clientName,
- getAmState());
+ .getInternalsSnapshot(), null, getClusterName(), clientName,
+ clientName, getAmState());
for (String configFileDN : configurations.keySet()) {
String configFileName = appConf.getComponentOpt(clientName,
@@ -316,19 +323,45 @@ public class DockerProviderService extends AbstractProviderService implements
getAmState().getAppConfSnapshot(), roleGroup);
String hostKeyFormat = "${%s_HOST}";
+ String hostNameKeyFormat = "${%s_HOSTNAME}";
+ String ipKeyFormat = "${%s_IP}";
// publish export groups if any
- Map<String, String> replaceTokens =
- providerUtils.filterSiteOptions(
- appConf.getComponent(roleGroup).options,
- providerUtils.getStandardTokenMap(appConf, internalsConf, roleName,
- roleGroup, containerId, getClusterName()));
+ Map<String, String> standardTokens = providerUtils.getStandardTokenMap(
+ appConf, internalsConf, roleName, roleGroup, containerId,
+ getClusterName());
+ Map<String, String> replaceTokens = providerUtils.filterSiteOptions(
+ appConf.getComponent(roleGroup).options, standardTokens);
+ replaceTokens.putAll(standardTokens);
+
+ String rolePrefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, "");
for (Map.Entry<String, Map<String, ClusterNode>> entry :
getAmState().getRoleClusterNodeMapping().entrySet()) {
- String hostName = providerUtils.getHostsList(
+ String otherRolePrefix = appConf.getComponentOpt(entry.getKey(),
+ ROLE_PREFIX, "");
+ if (!otherRolePrefix.equals(rolePrefix)) {
+ // hostname replacements are only made within role prefix groups
+ continue;
+ }
+ String key = entry.getKey();
+ if (!rolePrefix.isEmpty()) {
+ if (!key.startsWith(rolePrefix)) {
+ log.warn("Something went wrong, {} doesn't start with {}", key,
+ rolePrefix);
+ continue;
+ }
+ key = key.substring(rolePrefix.length());
+ }
+ key = key.toUpperCase(Locale.ENGLISH);
+ String host = providerUtils.getHostsList(
entry.getValue().values(), true).iterator().next();
- replaceTokens.put(String.format(hostKeyFormat,
- entry.getKey().toUpperCase(Locale.ENGLISH)), hostName);
+ replaceTokens.put(String.format(hostKeyFormat, key), host);
+ String hostName = providerUtils.getHostNamesList(
+ entry.getValue().values()).iterator().next();
+ replaceTokens.put(String.format(hostNameKeyFormat, key), hostName);
+ String ip = providerUtils.getIPsList(
+ entry.getValue().values()).iterator().next();
+ replaceTokens.put(String.format(ipKeyFormat, key), ip);
}
replaceTokens.put("${THIS_HOST}", thisHost);
@@ -338,7 +371,7 @@ public class DockerProviderService extends AbstractProviderService implements
// replace host names and site properties
for (String token : replaceTokens.keySet()) {
if (value.contains(token)) {
- value = value.replace(token, replaceTokens.get(token));
+ value = value.replaceAll(Pattern.quote(token), replaceTokens.get(token));
}
}
ExportEntry entry = new ExportEntry();
@@ -350,6 +383,24 @@ public class DockerProviderService extends AbstractProviderService implements
log.info("Preparing to publish. Key {} and Value {}",
export.getKey(), value);
}
- providerUtils.publishExportGroup(entries, getAmState(), EXPORT_GROUP);
+ if (!entries.isEmpty()) {
+ providerUtils.publishExportGroup(entries, getAmState(), EXPORT_GROUP);
+ }
+ }
+
+ @Override
+ public Map<String, MonitorDetail> buildMonitorDetails(ClusterDescription clusterDesc) {
+ Map<String, MonitorDetail> details = super.buildMonitorDetails(clusterDesc);
+ buildRoleHostDetails(details);
+ return details;
+ }
+
+ private void buildRoleHostDetails(Map<String, MonitorDetail> details) {
+ for (Map.Entry<String, Map<String, ClusterNode>> entry :
+ getAmState().getRoleClusterNodeMapping().entrySet()) {
+ details.put(entry.getKey() + " Host(s)/Container(s)",
+ new MonitorDetail(providerUtils.getHostsList(
+ entry.getValue().values(), false).toString(), false));
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/51] [abbrv] hadoop git commit: Modify pom file for slider
Posted by ji...@apache.org.
Modify pom file for slider
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d83f847
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d83f847
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d83f847
Branch: refs/heads/yarn-native-services
Commit: 0d83f84701e75ffc70ac380086bad09c5db30e1b
Parents: 21d2dd6
Author: Jian He <ji...@apache.org>
Authored: Fri Aug 12 09:28:41 2016 +0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:07:43 2016 -0800
----------------------------------------------------------------------
.../hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml | 2 +-
.../hadoop-yarn-applications/hadoop-yarn-slider/pom.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d83f847/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index 14130c5..591a5ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -18,7 +18,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-applications</artifactId>
+ <artifactId>hadoop-yarn-slider</artifactId>
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d83f847/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
index ad374f8d..780f68b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
@@ -17,7 +17,7 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
- <artifactId>hadoop-yarn</artifactId>
+ <artifactId>hadoop-yarn-applications</artifactId>
<groupId>org.apache.hadoop</groupId>
<version>3.0.0-alpha2-SNAPSHOT</version>
</parent>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/51] [abbrv] hadoop git commit: YARN-5943. Write native services
container stderr file to log directory. Contributed by Billie Rinaldi
Posted by ji...@apache.org.
YARN-5943. Write native services container stderr file to log directory. Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a31823a2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a31823a2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a31823a2
Branch: refs/heads/yarn-native-services
Commit: a31823a2dfe57bef0af866be0e26e1cbf2a3bb1b
Parents: b54bf1d
Author: Gour Saha <go...@apache.org>
Authored: Wed Nov 30 10:30:39 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../org/apache/slider/providers/docker/DockerProviderService.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a31823a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
index cc319ee..e4a7cdf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
@@ -193,7 +193,8 @@ public class DockerProviderService extends AbstractProviderService implements
"/bin/bash"));
operation.add("> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/"
- + OUT_FILE + " 2>" + ERR_FILE);
+ + OUT_FILE + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/"
+ + ERR_FILE);
launcher.addCommand(operation.build());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/51] [abbrv] hadoop git commit: YARN-5218. Initial core change for
DNS for YARN. Contributed by Jonathan Maron
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
new file mode 100644
index 0000000..37f0d23
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
@@ -0,0 +1,561 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.registry.server.dns;
+
+import org.apache.commons.net.util.Base64;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.xbill.DNS.AAAARecord;
+import org.xbill.DNS.ARecord;
+import org.xbill.DNS.CNAMERecord;
+import org.xbill.DNS.DClass;
+import org.xbill.DNS.DNSKEYRecord;
+import org.xbill.DNS.DNSSEC;
+import org.xbill.DNS.Flags;
+import org.xbill.DNS.Message;
+import org.xbill.DNS.Name;
+import org.xbill.DNS.OPTRecord;
+import org.xbill.DNS.PTRRecord;
+import org.xbill.DNS.RRSIGRecord;
+import org.xbill.DNS.RRset;
+import org.xbill.DNS.Rcode;
+import org.xbill.DNS.Record;
+import org.xbill.DNS.SRVRecord;
+import org.xbill.DNS.Section;
+import org.xbill.DNS.Type;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.net.Inet6Address;
+import java.net.InetAddress;
+import java.security.KeyFactory;
+import java.security.PrivateKey;
+import java.security.spec.RSAPrivateKeySpec;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_DNS_ZONE_MASK;
+import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_DNS_ZONE_SUBNET;
+
+/**
+ *
+ */
+public class TestRegistryDNS extends Assert {
+
+ private RegistryDNS registryDNS;
+ private RegistryUtils.ServiceRecordMarshal marshal;
+
+ private static final String APPLICATION_RECORD = "{\n"
+ + " \"type\" : \"JSONServiceRecord\",\n"
+ + " \"description\" : \"Slider Application Master\",\n"
+ + " \"external\" : [ {\n"
+ + " \"api\" : \"classpath:org.apache.slider.appmaster.ipc\",\n"
+ + " \"addressType\" : \"host/port\",\n"
+ + " \"protocolType\" : \"hadoop/IPC\",\n"
+ + " \"addresses\" : [ {\n"
+ + " \"host\" : \"192.168.1.5\",\n"
+ + " \"port\" : \"1026\"\n"
+ + " } ]\n"
+ + " }, {\n"
+ + " \"api\" : \"http://\",\n"
+ + " \"addressType\" : \"uri\",\n"
+ + " \"protocolType\" : \"webui\",\n"
+ + " \"addresses\" : [ {\n"
+ + " \"uri\" : \"http://192.168.1.5:1027\"\n"
+ + " } ]\n"
+ + " }, {\n"
+ + " \"api\" : \"classpath:org.apache.slider.management\",\n"
+ + " \"addressType\" : \"uri\",\n"
+ + " \"protocolType\" : \"REST\",\n"
+ + " \"addresses\" : [ {\n"
+ + " \"uri\" : \"http://192.168.1.5:1027/ws/v1/slider/mgmt\"\n"
+ + " } ]\n"
+ + " } ],\n"
+ + " \"internal\" : [ {\n"
+ + " \"api\" : \"classpath:org.apache.slider.agents.secure\",\n"
+ + " \"addressType\" : \"uri\",\n"
+ + " \"protocolType\" : \"REST\",\n"
+ + " \"addresses\" : [ {\n"
+ + " \"uri\" : \"https://192.168.1.5:47700/ws/v1/slider/agents\"\n"
+ + " } ]\n"
+ + " }, {\n"
+ + " \"api\" : \"classpath:org.apache.slider.agents.oneway\",\n"
+ + " \"addressType\" : \"uri\",\n"
+ + " \"protocolType\" : \"REST\",\n"
+ + " \"addresses\" : [ {\n"
+ + " \"uri\" : \"https://192.168.1.5:35531/ws/v1/slider/agents\"\n"
+ + " } ]\n"
+ + " } ],\n"
+ + " \"yarn:id\" : \"application_1451931954322_0016\",\n"
+ + " \"yarn:persistence\" : \"application\"\n"
+ + "}\n";
+ static final String CONTAINER_RECORD = "{\n"
+ + " \"type\" : \"JSONServiceRecord\",\n"
+ + " \"description\" : \"YCLOUD\",\n"
+ + " \"external\" : [ ],\n"
+ + " \"internal\" : [ ],\n"
+ + " \"yarn:id\" : \"container_e50_1451931954322_0016_01_000002\",\n"
+ + " \"yarn:persistence\" : \"container\",\n"
+ + " \"yarn:ip\" : \"172.17.0.19\",\n"
+ + " \"yarn:hostname\" : \"0a134d6329ba\"\n"
+ + "}\n";
+
+ private static final String CONTAINER_RECORD_NO_IP = "{\n"
+ + " \"type\" : \"JSONServiceRecord\",\n"
+ + " \"description\" : \"YCLOUD\",\n"
+ + " \"external\" : [ ],\n"
+ + " \"internal\" : [ ],\n"
+ + " \"yarn:id\" : \"container_e50_1451931954322_0016_01_000002\",\n"
+ + " \"yarn:persistence\" : \"container\"\n"
+ + "}\n";
+
+ @Before
+ public void initialize() throws Exception {
+ setRegistryDNS(new RegistryDNS("TestRegistry"));
+ Configuration conf = createConfiguration();
+
+ getRegistryDNS().setDomainName(conf);
+ getRegistryDNS().initializeZones(conf);
+
+ setMarshal(new RegistryUtils.ServiceRecordMarshal());
+ }
+
+ protected Configuration createConfiguration() {
+ Configuration conf = new Configuration();
+ conf.set(RegistryConstants.KEY_DNS_DOMAIN, "hwx.test");
+ conf.set(RegistryConstants.KEY_DNS_ZONE_SUBNET, "172.17.0");
+ conf.setTimeDuration(RegistryConstants.KEY_DNS_TTL, 30L, TimeUnit.SECONDS);
+ return conf;
+ }
+
+ protected boolean isSecure() {
+ return false;
+ }
+
+ @After
+ public void closeRegistry() throws Exception {
+ getRegistryDNS().stopExecutor();
+ }
+
+ @Test
+ public void testAppRegistration() throws Exception {
+ ServiceRecord record = getMarshal().fromBytes("somepath",
+ APPLICATION_RECORD.getBytes());
+ getRegistryDNS().register(
+ "/registry/users/root/services/org-apache-slider/test1/", record);
+
+ // start assessing whether correct records are available
+ Record[] recs = assertDNSQuery("test1.root.hwx.test.");
+ assertEquals("wrong result", "192.168.1.5",
+ ((ARecord) recs[0]).getAddress().getHostAddress());
+
+ recs = assertDNSQuery("management-api.test1.root.hwx.test.", 2);
+ assertEquals("wrong target name", "test1.root.hwx.test.",
+ ((CNAMERecord) recs[0]).getTarget().toString());
+ assertTrue("not an ARecord", recs[isSecure() ? 2 : 1] instanceof ARecord);
+
+ recs = assertDNSQuery("appmaster-ipc-api.test1.root.hwx.test.",
+ Type.SRV, 1);
+ assertTrue("not an SRV record", recs[0] instanceof SRVRecord);
+ assertEquals("wrong port", 1026, ((SRVRecord) recs[0]).getPort());
+
+ recs = assertDNSQuery("appmaster-ipc-api.test1.root.hwx.test.", 2);
+ assertEquals("wrong target name", "test1.root.hwx.test.",
+ ((CNAMERecord) recs[0]).getTarget().toString());
+ assertTrue("not an ARecord", recs[isSecure() ? 2 : 1] instanceof ARecord);
+
+ recs = assertDNSQuery("http-api.test1.root.hwx.test.", 2);
+ assertEquals("wrong target name", "test1.root.hwx.test.",
+ ((CNAMERecord) recs[0]).getTarget().toString());
+ assertTrue("not an ARecord", recs[isSecure() ? 2 : 1] instanceof ARecord);
+
+ recs = assertDNSQuery("http-api.test1.root.hwx.test.", Type.SRV,
+ 1);
+ assertTrue("not an SRV record", recs[0] instanceof SRVRecord);
+ assertEquals("wrong port", 1027, ((SRVRecord) recs[0]).getPort());
+
+ assertDNSQuery("test1.root.hwx.test.", Type.TXT, 3);
+ assertDNSQuery("appmaster-ipc-api.test1.root.hwx.test.", Type.TXT, 1);
+ assertDNSQuery("http-api.test1.root.hwx.test.", Type.TXT, 1);
+ assertDNSQuery("management-api.test1.root.hwx.test.", Type.TXT, 1);
+ }
+
+ @Test
+ public void testContainerRegistration() throws Exception {
+ ServiceRecord record = getMarshal().fromBytes("somepath",
+ CONTAINER_RECORD.getBytes());
+ getRegistryDNS().register(
+ "/registry/users/root/services/org-apache-slider/test1/components/"
+ + "container-e50-1451931954322-0016-01-000002",
+ record);
+
+ // start assessing whether correct records are available
+ Record[] recs =
+ assertDNSQuery("ctr-e50-1451931954322-0016-01-000002.hwx.test.");
+ assertEquals("wrong result", "172.17.0.19",
+ ((ARecord) recs[0]).getAddress().getHostAddress());
+
+ recs = assertDNSQuery("ycloud.test1.root.hwx.test.", 1);
+ assertTrue("not an ARecord", recs[0] instanceof ARecord);
+ }
+
+ @Test
+ public void testRecordTTL() throws Exception {
+ ServiceRecord record = getMarshal().fromBytes("somepath",
+ CONTAINER_RECORD.getBytes());
+ getRegistryDNS().register(
+ "/registry/users/root/services/org-apache-slider/test1/components/"
+ + "container-e50-1451931954322-0016-01-000002",
+ record);
+
+ // start assessing whether correct records are available
+ Record[] recs = assertDNSQuery(
+ "ctr-e50-1451931954322-0016-01-000002.hwx.test.");
+ assertEquals("wrong result", "172.17.0.19",
+ ((ARecord) recs[0]).getAddress().getHostAddress());
+ assertEquals("wrong ttl", 30L, recs[0].getTTL());
+
+ recs = assertDNSQuery("ycloud.test1.root.hwx.test.", 1);
+ assertTrue("not an ARecord", recs[0] instanceof ARecord);
+
+ assertEquals("wrong ttl", 30L, recs[0].getTTL());
+ }
+
+ @Test
+ public void testReverseLookup() throws Exception {
+ ServiceRecord record = getMarshal().fromBytes("somepath",
+ CONTAINER_RECORD.getBytes());
+ getRegistryDNS().register(
+ "/registry/users/root/services/org-apache-slider/test1/components/"
+ + "container-e50-1451931954322-0016-01-000002",
+ record);
+
+ // start assessing whether correct records are available
+ Record[] recs = assertDNSQuery("19.0.17.172.in-addr.arpa.", Type.PTR, 1);
+ assertEquals("wrong result",
+ "ctr-e50-1451931954322-0016-01-000002.hwx.test.",
+ ((PTRRecord) recs[0]).getTarget().toString());
+ }
+
+ @Test
+ public void testReverseLookupInLargeNetwork() throws Exception {
+ setRegistryDNS(new RegistryDNS("TestRegistry"));
+ Configuration conf = createConfiguration();
+ conf.set(RegistryConstants.KEY_DNS_DOMAIN, "hwx.test");
+ conf.set(KEY_DNS_ZONE_SUBNET, "172.17.0.0");
+ conf.set(KEY_DNS_ZONE_MASK, "255.255.224.0");
+ conf.setTimeDuration(RegistryConstants.KEY_DNS_TTL, 30L, TimeUnit.SECONDS);
+
+ getRegistryDNS().setDomainName(conf);
+ getRegistryDNS().initializeZones(conf);
+
+ ServiceRecord record = getMarshal().fromBytes("somepath",
+ CONTAINER_RECORD.getBytes());
+ getRegistryDNS().register(
+ "/registry/users/root/services/org-apache-slider/test1/components/"
+ + "container-e50-1451931954322-0016-01-000002",
+ record);
+
+ // start assessing whether correct records are available
+ Record[] recs = assertDNSQuery("19.0.17.172.in-addr.arpa.", Type.PTR, 1);
+ assertEquals("wrong result",
+ "ctr-e50-1451931954322-0016-01-000002.hwx.test.",
+ ((PTRRecord) recs[0]).getTarget().toString());
+ }
+
+ @Test
+ public void testMissingReverseLookup() throws Exception {
+ ServiceRecord record = getMarshal().fromBytes("somepath",
+ CONTAINER_RECORD.getBytes());
+ getRegistryDNS().register(
+ "/registry/users/root/services/org-apache-slider/test1/components/"
+ + "container-e50-1451931954322-0016-01-000002",
+ record);
+
+ // start assessing whether correct records are available
+ Name name = Name.fromString("19.1.17.172.in-addr.arpa.");
+ Record question = Record.newRecord(name, Type.PTR, DClass.IN);
+ Message query = Message.newQuery(question);
+ OPTRecord optRecord = new OPTRecord(4096, 0, 0, Flags.DO, null);
+ query.addRecord(optRecord, Section.ADDITIONAL);
+ byte[] responseBytes = getRegistryDNS().generateReply(query, null);
+ Message response = new Message(responseBytes);
+ assertEquals("No answer should be returned", Rcode.NOTAUTH,
+ response.getRcode());
+ }
+
+ @Test
+ public void testNoContainerIP() throws Exception {
+ ServiceRecord record = getMarshal().fromBytes("somepath",
+ CONTAINER_RECORD_NO_IP.getBytes());
+ getRegistryDNS().register(
+ "/registry/users/root/services/org-apache-slider/test1/components/"
+ + "container-e50-1451931954322-0016-01-000002",
+ record);
+
+ // start assessing whether correct records are available
+ Name name =
+ Name.fromString("ctr-e50-1451931954322-0016-01-000002.hwx.test.");
+ Record question = Record.newRecord(name, Type.A, DClass.IN);
+ Message query = Message.newQuery(question);
+
+ byte[] responseBytes = getRegistryDNS().generateReply(query, null);
+ Message response = new Message(responseBytes);
+ assertEquals("wrong status", Rcode.NXDOMAIN, response.getRcode());
+ }
+
+ private Record[] assertDNSQuery(String lookup) throws IOException {
+ return assertDNSQuery(lookup, Type.A, 1);
+ }
+
+ private Record[] assertDNSQuery(String lookup, int numRecs)
+ throws IOException {
+ return assertDNSQuery(lookup, Type.A, numRecs);
+ }
+
+ Record[] assertDNSQuery(String lookup, int type, int numRecs)
+ throws IOException {
+ Name name = Name.fromString(lookup);
+ Record question = Record.newRecord(name, type, DClass.IN);
+ Message query = Message.newQuery(question);
+ OPTRecord optRecord = new OPTRecord(4096, 0, 0, Flags.DO, null);
+ query.addRecord(optRecord, Section.ADDITIONAL);
+ byte[] responseBytes = getRegistryDNS().generateReply(query, null);
+ Message response = new Message(responseBytes);
+ assertEquals("not successful", Rcode.NOERROR, response.getRcode());
+ assertNotNull("Null response", response);
+ assertEquals("Questions do not match", query.getQuestion(),
+ response.getQuestion());
+ Record[] recs = response.getSectionArray(Section.ANSWER);
+ assertEquals("wrong number of answer records",
+ isSecure() ? numRecs * 2 : numRecs, recs.length);
+ if (isSecure()) {
+ boolean signed = false;
+ for (Record record : recs) {
+ signed = record.getType() == Type.RRSIG;
+ if (signed) {
+ break;
+ }
+ }
+ assertTrue("No signatures found", signed);
+ }
+ return recs;
+ }
+
+ @Test
+ public void testDNSKEYRecord() throws Exception {
+ String publicK =
+ "AwEAAe1Jev0Az1khlQCvf0nud1/CNHQwwPEu8BNchZthdDxKPVn29yrD "
+ + "CHoAWjwiGsOSw3SzIPrawSbHzyJsjn0oLBhGrH6QedFGnydoxjNsw3m/ "
+ + "SCmOjR/a7LGBAMDFKqFioi4gOyuN66svBeY+/5uw72+0ei9AQ20gqf6q "
+ + "l9Ozs5bV";
+ // byte[] publicBytes = Base64.decodeBase64(publicK);
+ // X509EncodedKeySpec keySpec = new X509EncodedKeySpec(publicBytes);
+ // KeyFactory keyFactory = KeyFactory.getInstance("RSA");
+ // PublicKey pubKey = keyFactory.generatePublic(keySpec);
+ DNSKEYRecord dnskeyRecord =
+ new DNSKEYRecord(Name.fromString("hwxstg.site."), DClass.IN, 0,
+ DNSKEYRecord.Flags.ZONE_KEY,
+ DNSKEYRecord.Protocol.DNSSEC,
+ DNSSEC.Algorithm.RSASHA256,
+ Base64.decodeBase64(publicK.getBytes()));
+ assertNotNull(dnskeyRecord);
+ RSAPrivateKeySpec privateSpec = new RSAPrivateKeySpec(new BigInteger(1,
+ Base64.decodeBase64(
+ "7Ul6/QDPWSGVAK9/Se53X8I0dDDA8S7wE1yFm2F0PEo9Wfb3KsMIegBaPCIaw5LDd"
+ + "LMg+trBJsfPImyOfSgsGEasfpB50UafJ2jGM2zDeb9IKY6NH9rssYEAwMUq"
+ + "oWKiLiA7K43rqy8F5j7/m7Dvb7R6L0BDbSCp/qqX07OzltU=")),
+ new BigInteger(1, Base64.decodeBase64(
+ "MgbQ6DBYhskeufNGGdct0cGG/4wb0X183ggenwCv2dopDyOTPq+5xMb4Pz9Ndzgk/"
+ + "yCY7mpaWIu9rttGOzrR+LBRR30VobPpMK1bMnzu2C0x08oYAguVwZB79DLC"
+ + "705qmZpiaaFB+LnhG7VtpPiOBm3UzZxdrBfeq/qaKrXid60=")));
+ KeyFactory factory = KeyFactory.getInstance("RSA");
+ PrivateKey priv = factory.generatePrivate(privateSpec);
+
+ ARecord aRecord = new ARecord(Name.fromString("some.test."), DClass.IN, 0,
+ InetAddress.getByName("192.168.0.1"));
+ Calendar cal = Calendar.getInstance();
+ Date inception = cal.getTime();
+ cal.add(Calendar.YEAR, 1);
+ Date expiration = cal.getTime();
+ RRset rrset = new RRset(aRecord);
+ RRSIGRecord rrsigRecord = DNSSEC.sign(rrset,
+ dnskeyRecord,
+ priv,
+ inception,
+ expiration);
+ DNSSEC.verify(rrset, rrsigRecord, dnskeyRecord);
+
+ }
+
+ @Test
+ public void testIpv4toIpv6() throws Exception {
+ InetAddress address =
+ BaseServiceRecordProcessor
+ .getIpv6Address(InetAddress.getByName("172.17.0.19"));
+ assertTrue("not an ipv6 address", address instanceof Inet6Address);
+ assertEquals("wrong IP", "172.17.0.19",
+ InetAddress.getByAddress(address.getAddress()).getHostAddress());
+ }
+
+ @Test
+ public void testAAAALookup() throws Exception {
+ ServiceRecord record = getMarshal().fromBytes("somepath",
+ CONTAINER_RECORD.getBytes());
+ getRegistryDNS().register(
+ "/registry/users/root/services/org-apache-slider/test1/components/"
+ + "container-e50-1451931954322-0016-01-000002",
+ record);
+
+ // start assessing whether correct records are available
+ Record[] recs = assertDNSQuery(
+ "ctr-e50-1451931954322-0016-01-000002.hwx.test.", Type.AAAA, 1);
+ assertEquals("wrong result", "172.17.0.19",
+ ((AAAARecord) recs[0]).getAddress().getHostAddress());
+
+ recs = assertDNSQuery("ycloud.test1.root.hwx.test.", Type.AAAA, 1);
+ assertTrue("not an ARecord", recs[0] instanceof AAAARecord);
+ }
+
+ @Test
+ public void testNegativeLookup() throws Exception {
+ ServiceRecord record = getMarshal().fromBytes("somepath",
+ CONTAINER_RECORD.getBytes());
+ getRegistryDNS().register(
+ "/registry/users/root/services/org-apache-slider/test1/components/"
+ + "container-e50-1451931954322-0016-01-000002",
+ record);
+
+ // start assessing whether correct records are available
+ Name name = Name.fromString("missing.hwx.test.");
+ Record question = Record.newRecord(name, Type.A, DClass.IN);
+ Message query = Message.newQuery(question);
+
+ byte[] responseBytes = getRegistryDNS().generateReply(query, null);
+ Message response = new Message(responseBytes);
+ assertEquals("not successful", Rcode.NXDOMAIN, response.getRcode());
+ assertNotNull("Null response", response);
+ assertEquals("Questions do not match", query.getQuestion(),
+ response.getQuestion());
+ Record[] sectionArray = response.getSectionArray(Section.AUTHORITY);
+ assertEquals("Wrong number of recs in AUTHORITY", isSecure() ? 2 : 1,
+ sectionArray.length);
+ boolean soaFound = false;
+ for (Record rec : sectionArray) {
+ soaFound = rec.getType() == Type.SOA;
+ if (soaFound) {
+ break;
+ }
+ }
+ assertTrue("wrong record type",
+ soaFound);
+
+ }
+
+ @Test
+ public void testReadMasterFile() throws Exception {
+ setRegistryDNS(new RegistryDNS("TestRegistry"));
+ Configuration conf = new Configuration();
+ conf.set(RegistryConstants.KEY_DNS_DOMAIN, "hwx.test");
+ conf.set(RegistryConstants.KEY_DNS_ZONE_SUBNET, "172.17.0");
+ conf.setTimeDuration(RegistryConstants.KEY_DNS_TTL, 30L, TimeUnit.SECONDS);
+ conf.set(RegistryConstants.KEY_DNS_ZONES_DIR,
+ getClass().getResource("/").getFile());
+ if (isSecure()) {
+ conf.setBoolean(RegistryConstants.KEY_DNSSEC_ENABLED, true);
+ conf.set(RegistryConstants.KEY_DNSSEC_PUBLIC_KEY,
+ "AwEAAe1Jev0Az1khlQCvf0nud1/CNHQwwPEu8BNchZthdDxKPVn29yrD "
+ + "CHoAWjwiGsOSw3SzIPrawSbHzyJsjn0oLBhGrH6QedFGnydoxjNsw3m/ "
+ + "SCmOjR/a7LGBAMDFKqFioi4gOyuN66svBeY+/5uw72+0ei9AQ20gqf6q "
+ + "l9Ozs5bV");
+ conf.set(RegistryConstants.KEY_DNSSEC_PRIVATE_KEY_FILE,
+ getClass().getResource("/test.private").getFile());
+ }
+
+ getRegistryDNS().setDomainName(conf);
+ getRegistryDNS().initializeZones(conf);
+
+ ServiceRecord record = getMarshal().fromBytes("somepath",
+ CONTAINER_RECORD.getBytes());
+ getRegistryDNS().register(
+ "/registry/users/root/services/org-apache-slider/test1/components/"
+ + "container-e50-1451931954322-0016-01-000002",
+ record);
+
+ // start assessing whether correct records are available
+ Record[] recs =
+ assertDNSQuery("ctr-e50-1451931954322-0016-01-000002.hwx.test.");
+ assertEquals("wrong result", "172.17.0.19",
+ ((ARecord) recs[0]).getAddress().getHostAddress());
+
+ recs = assertDNSQuery("ycloud.test1.root.hwx.test.", 1);
+ assertTrue("not an ARecord", recs[0] instanceof ARecord);
+
+ // lookup dyanmic reverse records
+ recs = assertDNSQuery("19.0.17.172.in-addr.arpa.", Type.PTR, 1);
+ assertEquals("wrong result",
+ "ctr-e50-1451931954322-0016-01-000002.hwx.test.",
+ ((PTRRecord) recs[0]).getTarget().toString());
+
+ // now lookup static reverse records
+ Name name = Name.fromString("5.0.17.172.in-addr.arpa.");
+ Record question = Record.newRecord(name, Type.PTR, DClass.IN);
+ Message query = Message.newQuery(question);
+ OPTRecord optRecord = new OPTRecord(4096, 0, 0, Flags.DO, null);
+ query.addRecord(optRecord, Section.ADDITIONAL);
+ byte[] responseBytes = getRegistryDNS().generateReply(query, null);
+ Message response = new Message(responseBytes);
+ recs = response.getSectionArray(Section.ANSWER);
+ assertEquals("wrong result", "cn005.hwx.test.",
+ ((PTRRecord) recs[0]).getTarget().toString());
+ }
+
+ @Test
+ public void testReverseZoneNames() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(KEY_DNS_ZONE_SUBNET, "172.26.32.0");
+ conf.set(KEY_DNS_ZONE_MASK, "255.255.224.0");
+
+ Name name = getRegistryDNS().getReverseZoneName(conf);
+ assertEquals("wrong name", "26.172.in-addr.arpa.", name.toString());
+ }
+
+ public RegistryDNS getRegistryDNS() {
+ return registryDNS;
+ }
+
+ public void setRegistryDNS(
+ RegistryDNS registryDNS) {
+ this.registryDNS = registryDNS;
+ }
+
+ public RegistryUtils.ServiceRecordMarshal getMarshal() {
+ return marshal;
+ }
+
+ public void setMarshal(
+ RegistryUtils.ServiceRecordMarshal marshal) {
+ this.marshal = marshal;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestSecureRegistryDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestSecureRegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestSecureRegistryDNS.java
new file mode 100644
index 0000000..ded63bd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestSecureRegistryDNS.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.registry.server.dns;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+
+/**
+ *
+ */
+public class TestSecureRegistryDNS extends TestRegistryDNS {
+ @Override protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ conf.setBoolean(RegistryConstants.KEY_DNSSEC_ENABLED, true);
+ conf.set(RegistryConstants.KEY_DNSSEC_PUBLIC_KEY,
+ "AwEAAe1Jev0Az1khlQCvf0nud1/CNHQwwPEu8BNchZthdDxKPVn29yrD "
+ + "CHoAWjwiGsOSw3SzIPrawSbHzyJsjn0oLBhGrH6QedFGnydoxjNsw3m/ "
+ + "SCmOjR/a7LGBAMDFKqFioi4gOyuN66svBeY+/5uw72+0ei9AQ20gqf6q "
+ + "l9Ozs5bV");
+ conf.set(RegistryConstants.KEY_DNSSEC_PRIVATE_KEY_FILE,
+ getClass().getResource("/test.private").getFile());
+
+ return conf;
+ }
+
+ @Override protected boolean isSecure() {
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/resources/0.17.172.in-addr.arpa.zone
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/resources/0.17.172.in-addr.arpa.zone b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/resources/0.17.172.in-addr.arpa.zone
new file mode 100644
index 0000000..0165f0d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/resources/0.17.172.in-addr.arpa.zone
@@ -0,0 +1,36 @@
+;
+; Licensed to the Apache Software Foundation (ASF) under one
+; or more contributor license agreements. See the NOTICE file
+; distributed with this work for additional information
+; regarding copyright ownership. The ASF licenses this file
+; to you under the Apache License, Version 2.0 (the
+; "License"); you may not use this file except in compliance
+; with the License. You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+;
+;
+$ORIGIN .
+$TTL 1800 ; 30 minutes
+0.17.172.in-addr.arpa IN SOA ns.hwhq.hortonworks.com. it.hortonworks.com. (
+ 2015081000 ; serial
+ 10800 ; refresh (3 hours)
+ 900 ; retry (15 minutes)
+ 1814400 ; expire (3 weeks)
+ 10800 ; minimum (3 hours)
+)
+ NS ns.hwhq.hortonworks.com.
+ NS ns2.hwhq.hortonworks.com.
+
+$ORIGIN 0.17.172.in-addr.arpa.
+5 PTR cn005.hwx.test.
+6 PTR cn006.hwx.test.
+7 PTR cn007.hwx.test.
+8 PTR cn008.hwx.test.
+9 PTR cn009.hwx.test.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3031e921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/resources/test.private
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/resources/test.private b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/resources/test.private
new file mode 100644
index 0000000..5f0da9d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/resources/test.private
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+Private-key-format: v1.3
+Algorithm: 8 (RSASHA256)
+Modulus: 7Ul6/QDPWSGVAK9/Se53X8I0dDDA8S7wE1yFm2F0PEo9Wfb3KsMIegBaPCIaw5LDdLMg+trBJsfPImyOfSgsGEasfpB50UafJ2jGM2zDeb9IKY6NH9rssYEAwMUqoWKiLiA7K43rqy8F5j7/m7Dvb7R6L0BDbSCp/qqX07OzltU=
+PublicExponent: AQAB
+PrivateExponent: MgbQ6DBYhskeufNGGdct0cGG/4wb0X183ggenwCv2dopDyOTPq+5xMb4Pz9Ndzgk/yCY7mpaWIu9rttGOzrR+LBRR30VobPpMK1bMnzu2C0x08oYAguVwZB79DLC705qmZpiaaFB+LnhG7VtpPiOBm3UzZxdrBfeq/qaKrXid60=
+Prime1: /HFdjI4cRuJBjK9IGWWmmVZWwaFsQYO9GHLCDwjm691GxaDpXuMdPd0uH9EqQvskyF8JPmzQXI43swyUFjizow==
+Prime2: 8KFxkWEHlhgB2GLi8tk39TKY5vmFUvh4FO28COl1N/rWjKVpfM1p6HQ6YavoGNZQmDBazv4WOZRqSQukHApzJw==
+Exponent1: alX+h/RcqOcpoW88OaZ99N1PkiTDCx3JC4FbiSXAz93Xr+vGIfgdGzAN+80JtklABz8xD6CabEJj6AIGZw3fbQ==
+Exponent2: vvPusqZkJcjBVh0K6hpUXKEdU1W5ZmFEsZ8Cs7PH0Hee4Je3QVGk9NGfLrkDgwo3hL4CofZiXqkXOwYg4husyw==
+Coefficient: omxpbNU6u/swbnkTC6MicaDqbJP7ETnCCJ1iN2+HZO/AlQCFlqVzLwGZmvGMAGA9ZWF+YpqpPhvzi4bWmi5XrQ==
+Created: 20160119155251
+Publish: 20160119155251
+Activate: 20160119155251
+
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/51] [abbrv] hadoop git commit: YARN-5883 Avoid or eliminate
expensive YARN get all applications call. Contributed by Gour Saha
Posted by ji...@apache.org.
YARN-5883 Avoid or eliminate expensive YARN get all applications call. Contributed by Gour Saha
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b54bf1d1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b54bf1d1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b54bf1d1
Branch: refs/heads/yarn-native-services
Commit: b54bf1d16032c54e2f2a53f6fe707ae4c2ccf9d1
Parents: 340967d
Author: Billie Rinaldi <bi...@apache.org>
Authored: Tue Nov 29 13:14:45 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 6 +-
.../slider/client/SliderYarnClientImpl.java | 94 +++++++++++++-------
.../apache/slider/common/tools/SliderUtils.java | 37 +++++++-
.../slider/core/registry/YarnAppListClient.java | 21 ++++-
.../providers/AbstractClientProvider.java | 28 +++++-
.../providers/docker/DockerClientProvider.java | 8 ++
.../servicemonitor/YarnApplicationProbe.java | 12 +--
7 files changed, 160 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b54bf1d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index ea10ed0..3f2df0a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -198,6 +198,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -2083,7 +2084,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// add the tags if available
Set<String> applicationTags = provider.getApplicationTags(sliderFileSystem,
- appOperations);
+ appOperations, clustername);
Credentials credentials = null;
if (clusterSecure) {
@@ -3031,9 +3032,10 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
appstate.ordinal() < YarnApplicationState.FINISHED.ordinal();
} else {
// scan for instance in single --state state
- List<ApplicationReport> userInstances = yarnClient.listDeployedInstances("");
state = state.toUpperCase(Locale.ENGLISH);
YarnApplicationState desiredState = extractYarnApplicationState(state);
+ List<ApplicationReport> userInstances = yarnClient
+ .listDeployedInstances("", EnumSet.of(desiredState), name);
ApplicationReport foundInstance =
yarnClient.findAppInInstanceList(userInstances, name, desiredState);
if (foundInstance != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b54bf1d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index d471cdb..258ef31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -53,6 +53,8 @@ import java.net.BindException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -115,10 +117,59 @@ public class SliderYarnClientImpl extends YarnClientImpl {
*/
public List<ApplicationReport> listDeployedInstances(String user)
throws YarnException, IOException {
+ return listDeployedInstances(user, null);
+ }
+
+ /**
+ * List Slider <i>deployed</i>instances belonging to a specific user in a
+ * given set of states.
+ * <p>
+ * Deployed means: known about in the YARN cluster; it will include all apps
+ * in the specified set of states.
+ *
+ * @param user
+ * user: "" means all users
+ * @param appStates
+ * filter by a set of YarnApplicationState
+ * @return a possibly empty list of Slider AMs
+ * @throws YarnException
+ * @throws IOException
+ */
+ public List<ApplicationReport> listDeployedInstances(String user,
+ EnumSet<YarnApplicationState> appStates)
+ throws YarnException, IOException {
+ return listDeployedInstances(user, appStates, null);
+ }
+
+ /**
+ * List Slider <i>deployed</i>instances belonging to a specific user in a
+ * given set of states and filtered by an application name tag.
+ * <p>
+ * Deployed means: known about in the YARN cluster; it will include all apps
+ * in the specified set of states and tagged with the specified app name.
+ *
+ * @param user
+ * user: "" means all users
+ * @param appStates
+ * filter by a set of YarnApplicationState
+ * @param appname
+ * an application name tag in the format defined by
+ * {@link SliderUtils#createNameTag(String)}
+ * @return a possibly empty list of Slider AMs
+ * @throws YarnException
+ * @throws IOException
+ */
+ public List<ApplicationReport> listDeployedInstances(String user,
+ EnumSet<YarnApplicationState> appStates, String appname)
+ throws YarnException, IOException {
Preconditions.checkArgument(user != null, "Null User");
Set<String> types = new HashSet<>(1);
types.add(SliderKeys.APP_TYPE);
- List<ApplicationReport> allApps = getApplications(types);
+ Set<String> tags = null;
+ if (appname != null) {
+ tags = Collections.singleton(SliderUtils.createNameTag(appname));
+ }
+ List<ApplicationReport> allApps = getApplications(types, appStates, tags);
List<ApplicationReport> results = new ArrayList<>();
for (ApplicationReport report : allApps) {
if (StringUtils.isEmpty(user) || user.equals(report.getUser())) {
@@ -136,20 +187,11 @@ public class SliderYarnClientImpl extends YarnClientImpl {
* @param appname application name
* @return the list of all matching application instances
*/
- public List<ApplicationReport> findAllInstances(String user,
- String appname)
+ public List<ApplicationReport> findAllInstances(String user, String appname)
throws IOException, YarnException {
Preconditions.checkArgument(appname != null, "Null application name");
- List<ApplicationReport> instances = listDeployedInstances(user);
- List<ApplicationReport> results =
- new ArrayList<>(instances.size());
- for (ApplicationReport report : instances) {
- if (report.getName().equals(appname)) {
- results.add(report);
- }
- }
- return results;
+ return listDeployedInstances(user, null, appname);
}
/**
@@ -204,14 +246,12 @@ public class SliderYarnClientImpl extends YarnClientImpl {
// user wants all instances killed
String user = getUsername();
log.info("Killing all applications belonging to {}", user);
- Collection<ApplicationReport> instances = listDeployedInstances(user);
+ Collection<ApplicationReport> instances = listDeployedInstances(user,
+ SliderUtils.getAllLiveAppStates());
for (ApplicationReport instance : instances) {
- if (isApplicationLive(instance)) {
- ApplicationId appId = instance.getApplicationId();
- log.info("Killing Application {}", appId);
-
- killRunningApplication(appId, "forced kill");
- }
+ ApplicationId appId = instance.getApplicationId();
+ log.info("Killing Application {}", appId);
+ killRunningApplication(appId, "forced kill");
}
} else {
ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
@@ -290,21 +330,11 @@ public class SliderYarnClientImpl extends YarnClientImpl {
* @return the list of all matching application instances
*/
public List<ApplicationReport> findAllLiveInstances(String user,
- String appname) throws
- YarnException,
- IOException {
+ String appname) throws YarnException, IOException {
Preconditions.checkArgument(StringUtils.isNotEmpty(appname),
"Null/empty application name");
- List<ApplicationReport> instances = listDeployedInstances(user);
- List<ApplicationReport> results =
- new ArrayList<ApplicationReport>(instances.size());
- for (ApplicationReport app : instances) {
- if (app.getName().equals(appname)
- && isApplicationLive(app)) {
- results.add(app);
- }
- }
- return results;
+ return listDeployedInstances(user, SliderUtils.getAllLiveAppStates(),
+ appname);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b54bf1d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index b101d34..4457957 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -104,6 +104,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
+import java.util.EnumSet;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.List;
@@ -2656,4 +2657,38 @@ public final class SliderUtils {
}
return prefix;
}
-}
+
+ public static String createNameTag(String name) {
+ return "Name: " + name;
+ }
+
+ public static String createVersionTag(String version) {
+ return "Version: " + version;
+ }
+
+ public static String createDescriptionTag(String description) {
+ return "Description: " + description;
+ }
+
+ /**
+ * Get all YarnApplicationState values which signify that an application is
+ * in RUNNING or pre-RUNNING state.
+ *
+ * @return all live app states
+ */
+ public static EnumSet<YarnApplicationState> getAllLiveAppStates() {
+ return EnumSet.range(YarnApplicationState.NEW,
+ YarnApplicationState.RUNNING);
+ }
+
+ /**
+ * Get all YarnApplicationState values which signify that an application is
+ * not live, which means it is in one of the post RUNNING states.
+ *
+ * @return all non live app states
+ */
+ public static EnumSet<YarnApplicationState> getAllNonLiveAppStates() {
+ return EnumSet.range(YarnApplicationState.FINISHED,
+ YarnApplicationState.KILLED);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b54bf1d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
index 1bdfb9c..71cc193 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
@@ -86,7 +86,7 @@ public class YarnAppListClient {
public ApplicationReport findInstance(String appname) throws
YarnException,
IOException {
- List<ApplicationReport> instances = listInstances(null);
+ List<ApplicationReport> instances = listInstances(null, appname);
return yarnClient.findClusterInInstanceList(instances, appname);
}
@@ -107,8 +107,25 @@ public class YarnAppListClient {
*/
public List<ApplicationReport> listInstances(String user)
throws YarnException, IOException {
+ return listInstances(user, null);
+ }
+
+ /**
+ * List all instances belonging to a specific user and a specific appname.
+ *
+ * @param user
+ * user if not the default. null means default, "" means all users,
+ * otherwise it is the name of a user
+ * @param appname
+ * application name set as an application tag
+ * @return a possibly empty list of AMs
+ * @throws YarnException
+ * @throws IOException
+ */
+ public List<ApplicationReport> listInstances(String user, String appname)
+ throws YarnException, IOException {
String listUser = user == null ? username : user;
- return yarnClient.listDeployedInstances(listUser);
+ return yarnClient.listDeployedInstances(listUser, null, appname);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b54bf1d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
index f59c347..01444fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.conf.MapOperations;
@@ -37,6 +38,7 @@ import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
+import java.util.HashSet;
import java.util.Set;
import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES;
@@ -217,11 +219,35 @@ public abstract class AbstractClientProvider extends Configured {
* @return the set of tags.
*/
public Set<String> getApplicationTags(SliderFileSystem fileSystem,
- ConfTreeOperations appConf) throws SliderException {
+ ConfTreeOperations appConf, String appName) throws SliderException {
return Collections.emptySet();
}
/**
+ * Generates a fixed format of application tags given one or more of
+ * application name, version and description. This allows subsequent query for
+ * an application with a name only, version only or description only or any
+ * combination of those as filters.
+ *
+ * @param appName name of the application
+ * @param appVersion version of the application
+ * @param appDescription brief description of the application
+ * @return
+ */
+ public final Set<String> createApplicationTags(String appName,
+ String appVersion, String appDescription) {
+ Set<String> tags = new HashSet<>();
+ tags.add(SliderUtils.createNameTag(appName));
+ if (appVersion != null) {
+ tags.add(SliderUtils.createVersionTag(appVersion));
+ }
+ if (appDescription != null) {
+ tags.add(SliderUtils.createDescriptionTag(appDescription));
+ }
+ return tags;
+ }
+
+ /**
* Process client operations for applications such as install, configure
* @param fileSystem
* @param registryOperations
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b54bf1d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
index d554427..86d87ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
@@ -34,6 +34,7 @@ import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import static org.apache.slider.providers.docker.DockerKeys.DOCKER_IMAGE;
@@ -93,4 +94,11 @@ public class DockerClientProvider extends AbstractClientProvider
}
}
}
+
+ @Override
+ public Set<String> getApplicationTags(SliderFileSystem fileSystem,
+ ConfTreeOperations appConf, String appName) throws SliderException {
+ return createApplicationTags(appName, null, null);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b54bf1d1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java
index adf613c..92df048 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/YarnApplicationProbe.java
@@ -68,23 +68,19 @@ public class YarnApplicationProbe extends Probe {
*/
@Override
public ProbeStatus ping(boolean livePing) {
-
ProbeStatus status = new ProbeStatus();
try {
-
- List<ApplicationReport> instances =
- yarnClient.listDeployedInstances(username);
- ApplicationReport instance =
- yarnClient.findClusterInInstanceList(instances, clustername);
+ List<ApplicationReport> instances = yarnClient
+ .listDeployedInstances(username, null, clustername);
+ ApplicationReport instance = yarnClient
+ .findClusterInInstanceList(instances, clustername);
if (null == instance) {
throw UnknownApplicationInstanceException.unknownInstance(clustername);
}
-
status.succeed(this);
} catch (Exception e) {
status.fail(this, e);
}
return status;
-
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/51] [abbrv] hadoop git commit: YARN-4757. Add the ability to
split reverse zone subnets. Contributed by Shane Kumpf.
Posted by ji...@apache.org.
YARN-4757. Add the ability to split reverse zone subnets. Contributed by Shane Kumpf.
(cherry picked from commit 9bff70f1316d889c3109c68dd308c8f456229bf8)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e9716f59
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e9716f59
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e9716f59
Branch: refs/heads/yarn-native-services
Commit: e9716f597d936ec580898afde9e66e2f4786d85a
Parents: 3031e92
Author: Varun Vasudev <vv...@apache.org>
Authored: Tue Aug 2 17:13:20 2016 +0530
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../registry/client/api/RegistryConstants.java | 20 +++
.../hadoop/registry/server/dns/RegistryDNS.java | 58 ++++++-
.../registry/server/dns/ReverseZoneUtils.java | 171 +++++++++++++++++++
.../registry/server/dns/TestRegistryDNS.java | 31 +++-
.../server/dns/TestReverseZoneUtils.java | 89 ++++++++++
5 files changed, 361 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9716f59/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
index 7115a4c..f4fecfd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
@@ -144,6 +144,26 @@ public interface RegistryConstants {
String KEY_DNS_ZONES_DIR = DNS_PREFIX + "zones-dir";
/**
+ * Split Reverse Zone.
+ * It may be necessary to spit large reverse zone subnets
+ * into multiple zones to handle existing hosts collocated
+ * with containers.
+ */
+ String KEY_DNS_SPLIT_REVERSE_ZONE = DNS_PREFIX + "split-reverse-zone";
+
+ /**
+ * Default value for splitting the reverse zone.
+ */
+ boolean DEFAULT_DNS_SPLIT_REVERSE_ZONE = false;
+
+ /**
+ * Split Reverse Zone IP Range.
+ * How many IPs should be part of each reverse zone split
+ */
+ String KEY_DNS_SPLIT_REVERSE_ZONE_RANGE = DNS_PREFIX +
+ "split-reverse-zone-range";
+
+ /**
* Key to set if the registry is secure: {@value}.
* Turning it on changes the permissions policy from "open access"
* to restrictions on kerberos with the option of
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9716f59/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
index 52b3c37..126795a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
@@ -16,6 +16,7 @@
*/
package org.apache.hadoop.registry.server.dns;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.IOFileFilter;
import org.apache.commons.net.util.Base64;
@@ -269,17 +270,61 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
}
/**
+ * Return the number of zones in the map.
+ *
+ * @return number of zones in the map
+ */
+ @VisibleForTesting
+ protected int getZoneCount() {
+ return zones.size();
+ }
+
+ /**
* Initializes the reverse lookup zone (mapping IP to name).
*
* @param conf the Hadoop configuration.
- * @throws IOException
+ * @throws IOException if the DNSSEC key can not be read.
*/
private void initializeReverseLookupZone(Configuration conf)
throws IOException {
- Name reverseLookupZoneName = getReverseZoneName(conf);
- Zone reverseLookupZone =
- configureZone(reverseLookupZoneName, conf);
- zones.put(reverseLookupZone.getOrigin(), reverseLookupZone);
+ // Determine if the subnet should be split into
+ // multiple reverse zones, this can be necessary in
+ // network configurations where the hosts and containers
+ // are part of the same subnet (i.e. the containers only use
+ // part of the subnet).
+ Boolean shouldSplitReverseZone = conf.getBoolean(KEY_DNS_SPLIT_REVERSE_ZONE,
+ DEFAULT_DNS_SPLIT_REVERSE_ZONE);
+ if (shouldSplitReverseZone) {
+ int subnetCount = ReverseZoneUtils.getSubnetCountForReverseZones(conf);
+ addSplitReverseZones(conf, subnetCount);
+ // Single reverse zone
+ } else {
+ Name reverseLookupZoneName = getReverseZoneName(conf);
+ Zone reverseLookupZone = configureZone(reverseLookupZoneName, conf);
+ zones.put(reverseLookupZone.getOrigin(), reverseLookupZone);
+ }
+ }
+
+ /**
+ * Create the zones based on the zone count.
+ *
+ * @param conf the Hadoop configuration.
+ * @param subnetCount number of subnets to create reverse zones for.
+ * @throws IOException if the DNSSEC key can not be read.
+ */
+ @VisibleForTesting
+ protected void addSplitReverseZones(Configuration conf, int subnetCount)
+ throws IOException {
+ String subnet = conf.get(KEY_DNS_ZONE_SUBNET);
+ String range = conf.get(KEY_DNS_SPLIT_REVERSE_ZONE_RANGE);
+
+ // Add the split reverse zones
+ for (int idx = 0; idx < subnetCount; idx++) {
+ Name reverseLookupZoneName = getReverseZoneName(ReverseZoneUtils
+ .getReverseZoneNetworkAddress(subnet, Integer.parseInt(range), idx));
+ Zone reverseLookupZone = configureZone(reverseLookupZoneName, conf);
+ zones.put(reverseLookupZone.getOrigin(), reverseLookupZone);
+ }
}
/**
@@ -427,7 +472,8 @@ public class RegistryDNS extends AbstractService implements DNSOperations,
*
* @param conf the Hadoop configuration.
*/
- private void setDNSSECEnabled(Configuration conf) {
+ @VisibleForTesting
+ protected void setDNSSECEnabled(Configuration conf) {
dnssecEnabled = conf.getBoolean(KEY_DNSSEC_ENABLED, false);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9716f59/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ReverseZoneUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ReverseZoneUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ReverseZoneUtils.java
new file mode 100644
index 0000000..cb04f9e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/ReverseZoneUtils.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.registry.server.dns;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.net.Inet6Address;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.net.util.SubnetUtils;
+import org.apache.hadoop.conf.Configuration;
+import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_DNS_SPLIT_REVERSE_ZONE_RANGE;
+import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_DNS_ZONE_MASK;
+import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_DNS_ZONE_SUBNET;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utilities for configuring reverse zones.
+ */
+public final class ReverseZoneUtils {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ReverseZoneUtils.class);
+
+ private static final long POW3 = (long) Math.pow(256, 3);
+ private static final long POW2 = (long) Math.pow(256, 2);
+ private static final long POW1 = (long) Math.pow(256, 1);
+
+ private ReverseZoneUtils() {
+ }
+
+ /**
+ * Given a baseIp, range and index, return the network address for the
+ * reverse zone.
+ *
+ * @param baseIp base ip address to perform calculations against.
+ * @param range number of ip addresses per subnet.
+ * @param index the index of the subnet to calculate.
+ * @return the calculated ip address.
+ * @throws UnknownHostException if an invalid ip is provided.
+ */
+ protected static String getReverseZoneNetworkAddress(String baseIp, int range,
+ int index) throws UnknownHostException {
+ if (index < 0) {
+ throw new IllegalArgumentException(
+ String.format("Invalid index provided, must be positive: %d", index));
+ }
+ if (range < 0) {
+ throw new IllegalArgumentException(
+ String.format("Invalid range provided, cannot be negative: %d",
+ range));
+ }
+ return calculateIp(baseIp, range, index);
+ }
+
+ /**
+ * When splitting the reverse zone, return the number of subnets needed,
+ * given the range and netmask.
+ *
+ * @param conf the Hadoop configuration.
+ * @return The number of subnets given the range and netmask.
+ */
+ protected static int getSubnetCountForReverseZones(Configuration conf) {
+ String subnet = conf.get(KEY_DNS_ZONE_SUBNET);
+ String mask = conf.get(KEY_DNS_ZONE_MASK);
+ String range = conf.get(KEY_DNS_SPLIT_REVERSE_ZONE_RANGE);
+
+ int parsedRange;
+ try {
+ parsedRange = Integer.parseInt(range);
+ } catch (NumberFormatException e) {
+ LOG.error("The supplied range is not a valid integer: Supplied range: ",
+ range);
+ throw e;
+ }
+ if (parsedRange < 0) {
+ String msg = String
+ .format("Range cannot be negative: Supplied range: %d", parsedRange);
+ LOG.error(msg);
+ throw new IllegalArgumentException(msg);
+ }
+
+ int ipCount;
+ try {
+ SubnetUtils subnetUtils = new SubnetUtils(subnet, mask);
+ subnetUtils.setInclusiveHostCount(true);
+ ipCount = subnetUtils.getInfo().getAddressCount();
+
+ } catch (IllegalArgumentException e) {
+ LOG.error("The subnet or mask is invalid: Subnet: {} Mask: {}", subnet,
+ mask);
+ throw e;
+ }
+
+ if (parsedRange == 0) {
+ return ipCount;
+ }
+ return ipCount / parsedRange;
+ }
+
+ private static String calculateIp(String baseIp, int range, int index)
+ throws UnknownHostException {
+ long[] ipParts = splitIp(baseIp);
+
+ long ipNum1 = POW3 * ipParts[0];
+ long ipNum2 = POW2 * ipParts[1];
+ long ipNum3 = POW1 * ipParts[2];
+ long ipNum4 = ipParts[3];
+ long ipNum = ipNum1 + ipNum2 + ipNum3 + ipNum4;
+
+ ArrayList<Long> ipPartsOut = new ArrayList<>();
+ // First octet
+ long temp = ipNum + range * (long) index;
+ ipPartsOut.add(0, temp / POW3);
+
+ // Second octet
+ temp = temp - ipPartsOut.get(0) * POW3;
+ ipPartsOut.add(1, temp / POW2);
+
+ // Third octet
+ temp = temp - ipPartsOut.get(1) * POW2;
+ ipPartsOut.add(2, temp / POW1);
+
+ // Fourth octet
+ temp = temp - ipPartsOut.get(2) * POW1;
+ ipPartsOut.add(3, temp);
+
+ return StringUtils.join(ipPartsOut, '.');
+ }
+
+ @VisibleForTesting
+ protected static long[] splitIp(String baseIp) throws UnknownHostException {
+ InetAddress inetAddress;
+ try {
+ inetAddress = InetAddress.getByName(baseIp);
+ } catch (UnknownHostException e) {
+ LOG.error("Base IP address is invalid");
+ throw e;
+ }
+ if (inetAddress instanceof Inet6Address) {
+ throw new IllegalArgumentException(
+ "IPv6 is not yet supported for " + "reverse zones");
+ }
+ byte[] octets = inetAddress.getAddress();
+ if (octets.length != 4) {
+ throw new IllegalArgumentException("Base IP address is invalid");
+ }
+ long[] results = new long[4];
+ for (int i = 0; i < octets.length; i++) {
+ results[i] = octets[i] & 0xff;
+ }
+ return results;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9716f59/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
index 37f0d23..d58b1c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
@@ -55,8 +55,7 @@ import java.util.Calendar;
import java.util.Date;
import java.util.concurrent.TimeUnit;
-import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_DNS_ZONE_MASK;
-import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_DNS_ZONE_SUBNET;
+import static org.apache.hadoop.registry.client.api.RegistryConstants.*;
/**
*
@@ -541,6 +540,34 @@ public class TestRegistryDNS extends Assert {
assertEquals("wrong name", "26.172.in-addr.arpa.", name.toString());
}
+ @Test
+ public void testSplitReverseZoneNames() throws Exception {
+ Configuration conf = new Configuration();
+ registryDNS = new RegistryDNS("TestRegistry");
+ conf.set(RegistryConstants.KEY_DNS_DOMAIN, "example.com");
+ conf.set(KEY_DNS_SPLIT_REVERSE_ZONE, "true");
+ conf.set(KEY_DNS_SPLIT_REVERSE_ZONE_RANGE, "256");
+ conf.set(KEY_DNS_ZONE_SUBNET, "172.26.32.0");
+ conf.set(KEY_DNS_ZONE_MASK, "255.255.224.0");
+ conf.setTimeDuration(RegistryConstants.KEY_DNS_TTL, 30L, TimeUnit.SECONDS);
+ conf.set(RegistryConstants.KEY_DNS_ZONES_DIR,
+ getClass().getResource("/").getFile());
+ if (isSecure()) {
+ conf.setBoolean(RegistryConstants.KEY_DNSSEC_ENABLED, true);
+ conf.set(RegistryConstants.KEY_DNSSEC_PUBLIC_KEY,
+ "AwEAAe1Jev0Az1khlQCvf0nud1/CNHQwwPEu8BNchZthdDxKPVn29yrD "
+ + "CHoAWjwiGsOSw3SzIPrawSbHzyJsjn0oLBhGrH6QedFGnydoxjNsw3m/ "
+ + "SCmOjR/a7LGBAMDFKqFioi4gOyuN66svBeY+/5uw72+0ei9AQ20gqf6q "
+ + "l9Ozs5bV");
+ conf.set(RegistryConstants.KEY_DNSSEC_PRIVATE_KEY_FILE,
+ getClass().getResource("/test.private").getFile());
+ }
+ registryDNS.setDomainName(conf);
+ registryDNS.setDNSSECEnabled(conf);
+ registryDNS.addSplitReverseZones(conf, 4);
+ assertEquals(4, registryDNS.getZoneCount());
+ }
+
public RegistryDNS getRegistryDNS() {
return registryDNS;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9716f59/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestReverseZoneUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestReverseZoneUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestReverseZoneUtils.java
new file mode 100644
index 0000000..1331f75
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestReverseZoneUtils.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.registry.server.dns;
+
+import java.net.UnknownHostException;
+import static org.junit.Assert.assertEquals;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+/**
+ * Tests for the reverse zone utilities.
+ */
+public class TestReverseZoneUtils {
+ private static final String NET = "172.17.4.0";
+ private static final int RANGE = 256;
+ private static final int INDEX = 0;
+
+ @Rule public ExpectedException exception = ExpectedException.none();
+
+ @Test
+ public void testGetReverseZoneNetworkAddress() throws Exception {
+ assertEquals("172.17.4.0",
+ ReverseZoneUtils.getReverseZoneNetworkAddress(NET, RANGE, INDEX));
+ }
+
+ @Test
+ public void testSplitIp() throws Exception {
+ long[] splitIp = ReverseZoneUtils.splitIp(NET);
+ assertEquals(172, splitIp[0]);
+ assertEquals(17, splitIp[1]);
+ assertEquals(4, splitIp[2]);
+ assertEquals(0, splitIp[3]);
+ }
+
+ @Test
+ public void testThrowIllegalArgumentExceptionIfIndexIsNegative()
+ throws Exception {
+ exception.expect(IllegalArgumentException.class);
+ ReverseZoneUtils.getReverseZoneNetworkAddress(NET, RANGE, -1);
+ }
+
+ @Test
+ public void testThrowUnknownHostExceptionIfIpIsInvalid() throws Exception {
+ exception.expect(UnknownHostException.class);
+ ReverseZoneUtils
+ .getReverseZoneNetworkAddress("213124.21231.14123.13", RANGE, INDEX);
+ }
+
+ @Test
+ public void testThrowIllegalArgumentExceptionIfRangeIsNegative()
+ throws Exception {
+ exception.expect(IllegalArgumentException.class);
+ ReverseZoneUtils.getReverseZoneNetworkAddress(NET, -1, INDEX);
+ }
+
+ @Test
+ public void testVariousRangeAndIndexValues() throws Exception {
+ // Given the base address of 172.17.4.0, step 256 IP addresses, 5 times.
+ assertEquals("172.17.9.0",
+ ReverseZoneUtils.getReverseZoneNetworkAddress(NET, 256, 5));
+ assertEquals("172.17.4.128",
+ ReverseZoneUtils.getReverseZoneNetworkAddress(NET, 128, 1));
+ assertEquals("172.18.0.0",
+ ReverseZoneUtils.getReverseZoneNetworkAddress(NET, 256, 252));
+ assertEquals("172.17.12.0",
+ ReverseZoneUtils.getReverseZoneNetworkAddress(NET, 1024, 2));
+ assertEquals("172.17.4.0",
+ ReverseZoneUtils.getReverseZoneNetworkAddress(NET, 0, 1));
+ assertEquals("172.17.4.0",
+ ReverseZoneUtils.getReverseZoneNetworkAddress(NET, 1, 0));
+ assertEquals("172.17.4.1",
+ ReverseZoneUtils.getReverseZoneNetworkAddress(NET, 1, 1));
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/51] [abbrv] hadoop git commit: YARN-5729. Bug fixes for the
service Rest API. Contributed by Gour Saha
Posted by ji...@apache.org.
YARN-5729. Bug fixes for the service Rest API. Contributed by Gour Saha
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/39e18c16
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/39e18c16
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/39e18c16
Branch: refs/heads/yarn-native-services
Commit: 39e18c1629119ddf8b4545f9736edca7cd43cd60
Parents: 3862e38
Author: Jian He <ji...@apache.org>
Authored: Fri Oct 14 13:47:38 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../services/api/impl/ApplicationApiService.java | 16 ++++++++++++++--
.../hadoop/yarn/services/resource/Application.java | 6 +++---
.../hadoop/yarn/services/resource/Artifact.java | 4 +++-
.../hadoop/yarn/services/resource/Component.java | 4 +++-
.../hadoop/yarn/services/resource/ConfigFile.java | 4 +++-
.../yarn/services/resource/Configuration.java | 4 +++-
.../hadoop/yarn/services/resource/Container.java | 6 +++---
.../yarn/services/resource/PlacementPolicy.java | 4 +++-
.../yarn/services/resource/ReadinessCheck.java | 4 +++-
.../hadoop/yarn/services/resource/Resource.java | 2 +-
10 files changed, 39 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39e18c16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index 0a62629..21cf113 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -276,7 +276,7 @@ public class ApplicationApiService implements ApplicationApi {
// If it is a simple app with no components, then create a default component
if (application.getComponents() == null) {
- application.setComponents(getDefaultComponentAsList());
+ application.setComponents(getDefaultComponentAsList(application));
}
// Application lifetime if not specified, is set to unlimited lifetime
@@ -1029,7 +1029,8 @@ public class ApplicationApiService implements ApplicationApi {
// end-users point of view, is out of scope of the REST API. Also, this
// readiness has nothing to do with readiness-check defined at the component
// level (which is used for dependency resolution of component DAG).
- if (totalNumberOfIpAssignedContainers == totalExpectedNumberOfRunningContainers) {
+ if (totalNumberOfIpAssignedContainers
+ .longValue() == totalExpectedNumberOfRunningContainers.longValue()) {
app.setState(ApplicationState.READY);
}
logger.info("Application = {}", app);
@@ -1389,6 +1390,17 @@ public class ApplicationApiService implements ApplicationApi {
return Response.status(Status.NO_CONTENT).build();
}
+ // create default component and initialize with app level global values
+ private List<Component> getDefaultComponentAsList(Application app) {
+ List<Component> comps = getDefaultComponentAsList();
+ Component comp = comps.get(0);
+ comp.setArtifact(app.getArtifact());
+ comp.setResource(app.getResource());
+ comp.setNumberOfContainers(app.getNumberOfContainers());
+ comp.setLaunchCommand(app.getLaunchCommand());
+ return comps;
+ }
+
private List<Component> getDefaultComponentAsList() {
Component comp = new Component();
comp.setName(DEFAULT_COMPONENT_NAME);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39e18c16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
index 719bf95..ed65ad2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
@@ -168,19 +168,19 @@ public class Application extends BaseResource {
* The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.
**/
public Application launchTime(Date launchTime) {
- this.launchTime = launchTime;
+ this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
return this;
}
@ApiModelProperty(example = "null", value = "The time when the application was created, e.g. 2016-03-16T01:01:49.000Z.")
@JsonProperty("launch_time")
public String getLaunchTime() {
- return launchTime.toString();
+ return launchTime == null ? null : launchTime.toString();
}
@XmlElement(name = "launch_time")
public void setLaunchTime(Date launchTime) {
- this.launchTime = launchTime;
+ this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39e18c16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
index af0ad12..9ac2bc7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.services.resource;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
+import java.io.Serializable;
import java.util.Objects;
import com.fasterxml.jackson.annotation.JsonInclude;
@@ -33,7 +34,8 @@ import com.fasterxml.jackson.annotation.JsonValue;
@ApiModel(description = "Artifact of an application component")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@JsonInclude(JsonInclude.Include.NON_NULL)
-public class Artifact {
+public class Artifact implements Serializable {
+ private static final long serialVersionUID = 3608929500111099035L;
private String id = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39e18c16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Component.java
index 1246aa8..75f579a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Component.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.services.resource;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@@ -42,7 +43,8 @@ import com.fasterxml.jackson.annotation.JsonProperty;
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@XmlRootElement
@JsonInclude(JsonInclude.Include.NON_NULL)
-public class Component {
+public class Component implements Serializable {
+ private static final long serialVersionUID = -8430058381509087805L;
private String name = null;
private List<String> dependencies = new ArrayList<String>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39e18c16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
index d06c1b8..3ced153 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.services.resource;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
+import java.io.Serializable;
import java.util.Objects;
import javax.xml.bind.annotation.XmlElement;
@@ -38,7 +39,8 @@ import com.fasterxml.jackson.annotation.JsonValue;
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@XmlRootElement
@JsonInclude(JsonInclude.Include.NON_NULL)
-public class ConfigFile {
+public class ConfigFile implements Serializable {
+ private static final long serialVersionUID = -7009402089417704612L;
public enum TypeEnum {
XML("xml"), PROPERTIES("properties"), JSON("json"), YAML("yaml"), TEMPLATE(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39e18c16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Configuration.java
index 05983db..908220a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Configuration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Configuration.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.services.resource;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -39,7 +40,8 @@ import com.fasterxml.jackson.annotation.JsonProperty;
@ApiModel(description = "Set of configuration properties that can be injected into the application components via envs, files and custom pluggable helper docker containers. Files of several standard formats like xml, properties, json, yaml and templates will be supported.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@JsonInclude(JsonInclude.Include.NON_NULL)
-public class Configuration {
+public class Configuration implements Serializable {
+ private static final long serialVersionUID = -4330788704981074466L;
private Map<String, String> properties = new HashMap<String, String>();
private Map<String, String> env = new HashMap<String, String>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39e18c16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java
index 4e40102..a4efdf3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java
@@ -73,19 +73,19 @@ public class Container extends BaseResource {
* This will most likely be different from cluster launch time.
**/
public Container launchTime(Date launchTime) {
- this.launchTime = launchTime;
+ this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
return this;
}
@ApiModelProperty(example = "null", value = "The time when the container was created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from cluster launch time.")
@JsonProperty("launch_time")
public String getLaunchTime() {
- return launchTime.toString();
+ return launchTime == null ? null : launchTime.toString();
}
@XmlElement(name = "launch_time")
public void setLaunchTime(Date launchTime) {
- this.launchTime = launchTime;
+ this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39e18c16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/PlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/PlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/PlacementPolicy.java
index 7541e2f..5df00a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/PlacementPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/PlacementPolicy.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.services.resource;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
+import java.io.Serializable;
import java.util.Objects;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -31,7 +32,8 @@ import com.fasterxml.jackson.annotation.JsonProperty;
@ApiModel(description = "Placement policy of an instance of an application. This feature is in the works in YARN-4902.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-public class PlacementPolicy {
+public class PlacementPolicy implements Serializable {
+ private static final long serialVersionUID = 4341110649551172231L;
private String label = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39e18c16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ReadinessCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ReadinessCheck.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ReadinessCheck.java
index 10c951a..f549746 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ReadinessCheck.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ReadinessCheck.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.services.resource;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
+import java.io.Serializable;
import java.util.Objects;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -34,7 +35,8 @@ import com.fasterxml.jackson.annotation.JsonValue;
@ApiModel(description = "A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every application is different. Hence the need for a simple interface, with scope to support advanced usecases.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-public class ReadinessCheck {
+public class ReadinessCheck implements Serializable {
+ private static final long serialVersionUID = -3836839816887186801L;
public enum TypeEnum {
HTTP("http");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39e18c16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java
index a3780cc..234ccb3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java
@@ -35,7 +35,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
@ApiModel(description = "Resource determines the amount of resources (vcores, memory, network, etc.) usable by a container. This field determines the resource to be applied for all the containers of a component or application. The resource specified at the app (or global) level can be overriden at the component level. Only one of profile OR cpu & memory are exepected. It raises a validation exception otherwise.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
-public class Resource extends BaseResource {
+public class Resource extends BaseResource implements Cloneable {
private static final long serialVersionUID = -6431667797380250037L;
private String profile = null;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/51] [abbrv] hadoop git commit: YARN-5513. Move Java only tests
from slider develop to yarn-native-services. Contributed by Gour Saha
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
new file mode 100644
index 0000000..7fceac7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.publisher;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.providers.agent.AgentProviderService;
+import org.apache.slider.server.appmaster.actions.QueueAccess;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ *
+ */
+public class TestAgentProviderService extends AgentProviderService {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestAgentProviderService.class);
+
+ public TestAgentProviderService() {
+ super();
+ log.info("TestAgentProviderService created");
+ }
+
+ @Override
+ public void bind(StateAccessForProviders stateAccessor,
+ QueueAccess queueAccess,
+ List<Container> liveContainers) {
+ super.bind(stateAccessor, queueAccess, liveContainers);
+ Map<String,String> dummyProps = new HashMap<String, String>();
+ dummyProps.put("prop1", "val1");
+ dummyProps.put("prop2", "val2");
+ log.info("publishing dummy-site.xml with values {}", dummyProps);
+ publishApplicationInstanceData("dummy-site", "dummy configuration",
+ dummyProps.entrySet());
+ // publishing global config for testing purposes
+ publishApplicationInstanceData("global", "global configuration",
+ stateAccessor.getAppConfSnapshot()
+ .getGlobalOptions().entrySet());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestSliderProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestSliderProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestSliderProviderFactory.java
new file mode 100644
index 0000000..f49e15a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestSliderProviderFactory.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.publisher;
+
+import org.apache.slider.providers.ProviderService;
+import org.apache.slider.providers.agent.AgentProviderFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class TestSliderProviderFactory extends AgentProviderFactory{
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestSliderProviderFactory.class);
+
+ public TestSliderProviderFactory() {
+ log.info("Created TestSliderProviderFactory");
+ }
+
+ @Override
+ public ProviderService createServerProvider() {
+ log.info("Creating TestAgentProviderService");
+ return new TestAgentProviderService();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/servicemonitor/TestPortProbe.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/servicemonitor/TestPortProbe.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/servicemonitor/TestPortProbe.java
new file mode 100644
index 0000000..a93ec57
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/servicemonitor/TestPortProbe.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.servicemonitor;
+
+import org.junit.Assert;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+public class TestPortProbe extends Assert {
+ /**
+ * Assert that a port probe failed if the port is closed
+ * @throws Throwable
+ */
+ @Test
+ public void testPortProbeFailsClosedPort() throws Throwable {
+ PortProbe probe = new PortProbe("127.0.0.1", 65500, 100, "", new Configuration());
+ probe.init();
+ ProbeStatus status = probe.ping(true);
+ assertFalse("Expected a failure but got successful result: " + status,
+ status.isSuccess());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java
new file mode 100644
index 0000000..7a4a586
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java
@@ -0,0 +1,540 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.services.security;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
+import org.apache.slider.Slider;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.SliderException;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.X509TrustManager;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.Principal;
+import java.security.cert.Certificate;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ *
+ */
+public class TestCertificateManager {
+ @Rule
+ public TemporaryFolder workDir = new TemporaryFolder();
+ private File secDir;
+ private CertificateManager certMan;
+
+ @Before
+ public void setup() throws Exception {
+ certMan = new CertificateManager();
+ MapOperations compOperations = new MapOperations();
+ secDir = new File(workDir.getRoot(), SliderKeys.SECURITY_DIR);
+ File keystoreFile = new File(secDir, SliderKeys.KEYSTORE_FILE_NAME);
+ compOperations.put(SliderXmlConfKeys.KEY_KEYSTORE_LOCATION,
+ keystoreFile.getAbsolutePath());
+ certMan.initialize(compOperations, "cahost", null, null);
+ }
+
+ @Test
+ public void testServerCertificateGenerated() throws Exception {
+ File serverCrt = new File(secDir, SliderKeys.CRT_FILE_NAME);
+ Assert.assertTrue("Server CRD does not exist:" + serverCrt,
+ serverCrt.exists());
+ }
+
+ @Test
+ public void testAMKeystoreGenerated() throws Exception {
+ File keystoreFile = new File(secDir, SliderKeys.KEYSTORE_FILE_NAME);
+ Assert.assertTrue("Keystore does not exist: " + keystoreFile,
+ keystoreFile.exists());
+ InputStream is = null;
+ try {
+
+ is = new FileInputStream(keystoreFile);
+ KeyStore keystore = KeyStore.getInstance("pkcs12");
+ String password = SecurityUtils.getKeystorePass();
+ keystore.load(is, password.toCharArray());
+
+ Certificate certificate = keystore.getCertificate(
+ keystore.aliases().nextElement());
+ Assert.assertNotNull(certificate);
+
+ if (certificate instanceof X509Certificate) {
+ X509Certificate x509cert = (X509Certificate) certificate;
+
+ // Get subject
+ Principal principal = x509cert.getSubjectDN();
+ String subjectDn = principal.getName();
+ Assert.assertEquals("wrong DN",
+ "CN=cahost",
+ subjectDn);
+
+ // Get issuer
+ principal = x509cert.getIssuerDN();
+ String issuerDn = principal.getName();
+ Assert.assertEquals("wrong Issuer DN",
+ "CN=cahost",
+ issuerDn);
+ }
+ } finally {
+ if(null != is) {
+ is.close();
+ }
+ }
+ }
+
+ @Test
+ public void testContainerCertificateGeneration() throws Exception {
+ certMan.generateContainerCertificate("testhost", "container1");
+ Assert.assertTrue("container certificate not generated",
+ new File(secDir, "container1.crt").exists());
+ }
+
+ @Test
+ public void testContainerKeystoreGeneration() throws Exception {
+ SecurityStore keystoreFile = certMan.generateContainerKeystore("testhost",
+ "container1",
+ "component1",
+ "password");
+ validateKeystore(keystoreFile.getFile(), "testhost", "cahost");
+ }
+
+ private void validateKeystore(File keystoreFile, String certHostname,
+ String issuerHostname)
+ throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException {
+ Assert.assertTrue("container keystore not generated",
+ keystoreFile.exists());
+
+ InputStream is = null;
+ try {
+
+ is = new FileInputStream(keystoreFile);
+ KeyStore keystore = KeyStore.getInstance("pkcs12");
+ String password = "password";
+ keystore.load(is, password.toCharArray());
+
+ Certificate certificate = keystore.getCertificate(
+ keystore.aliases().nextElement());
+ Assert.assertNotNull(certificate);
+
+ if (certificate instanceof X509Certificate) {
+ X509Certificate x509cert = (X509Certificate) certificate;
+
+ // Get subject
+ Principal principal = x509cert.getSubjectDN();
+ String subjectDn = principal.getName();
+ Assert.assertEquals("wrong DN", "CN=" + certHostname + ", OU=container1",
+ subjectDn);
+
+ // Get issuer
+ principal = x509cert.getIssuerDN();
+ String issuerDn = principal.getName();
+ Assert.assertEquals("wrong Issuer DN",
+ "CN=" + issuerHostname,
+ issuerDn);
+ }
+ } finally {
+ if(null != is) {
+ is.close();
+ }
+ }
+ }
+
+ @Test
+ public void testContainerKeystoreGenerationViaStoresGenerator() throws Exception {
+ AggregateConf instanceDefinition = new AggregateConf();
+ MapOperations compOps = new MapOperations();
+ instanceDefinition.getAppConf().components.put("component1", compOps);
+ compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_PROPERTY_KEY,
+ "app1.component1.password.property");
+ compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true");
+ instanceDefinition.getAppConf().global.put(
+ "app1.component1.password.property", "password");
+ instanceDefinition.resolve();
+ SecurityStore[]
+ files = StoresGenerator.generateSecurityStores("testhost",
+ "container1",
+ "component1",
+ instanceDefinition,
+ compOps);
+ assertEquals("wrong number of stores", 1, files.length);
+ validateKeystore(files[0].getFile(), "testhost", "cahost");
+ }
+
+ @Test
+ public void testContainerKeystoreGenerationViaStoresGeneratorUsingGlobalProps() throws Exception {
+ AggregateConf instanceDefinition = new AggregateConf();
+ MapOperations compOps = new MapOperations();
+ instanceDefinition.getAppConf().components.put("component1", compOps);
+ compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_PROPERTY_KEY,
+ "app1.component1.password.property");
+ instanceDefinition.getAppConf().global.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true");
+ compOps.put(
+ "app1.component1.password.property", "password");
+ instanceDefinition.resolve();
+ SecurityStore[]
+ files = StoresGenerator.generateSecurityStores("testhost",
+ "container1",
+ "component1",
+ instanceDefinition,
+ compOps);
+ assertEquals("wrong number of stores", 1, files.length);
+ validateKeystore(files[0].getFile(), "testhost", "cahost");
+ }
+
+ @Test
+ public void testContainerKeystoreGenerationViaStoresGeneratorOverrideGlobalSetting() throws Exception {
+ AggregateConf instanceDefinition = new AggregateConf();
+ MapOperations compOps = setupComponentOptions(true, null,
+ "app1.component1.password.property",
+ null, null);
+ instanceDefinition.getAppConf().components.put("component1", compOps);
+ instanceDefinition.getAppConf().global.put(
+ "app1.component1.password.property", "password");
+ instanceDefinition.getAppConf().global.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "false");
+ instanceDefinition.resolve();
+ SecurityStore[]
+ files = StoresGenerator.generateSecurityStores("testhost",
+ "container1",
+ "component1",
+ instanceDefinition,
+ compOps);
+ assertEquals("wrong number of stores", 1, files.length);
+ validateKeystore(files[0].getFile(), "testhost", "cahost");
+ }
+
+ @Test
+ public void testContainerTrusttoreGeneration() throws Exception {
+ SecurityStore keystoreFile =
+ certMan.generateContainerKeystore("testhost",
+ "container1",
+ "component1",
+ "keypass");
+ Assert.assertTrue("container keystore not generated",
+ keystoreFile.getFile().exists());
+ SecurityStore truststoreFile =
+ certMan.generateContainerTruststore("container1",
+ "component1", "trustpass"
+ );
+ Assert.assertTrue("container truststore not generated",
+ truststoreFile.getFile().exists());
+
+ validateTruststore(keystoreFile.getFile(), truststoreFile.getFile());
+ }
+
+ @Test
+ public void testContainerGenerationUsingStoresGeneratorNoTruststore() throws Exception {
+ AggregateConf instanceDefinition = new AggregateConf();
+ MapOperations compOps = new MapOperations();
+ compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true");
+ compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_KEY,
+ "test.keystore.password");
+
+ setupCredentials(instanceDefinition, "test.keystore.password", null);
+
+ SecurityStore[]
+ files = StoresGenerator.generateSecurityStores("testhost",
+ "container1",
+ "component1",
+ instanceDefinition,
+ compOps);
+ assertEquals("wrong number of stores", 1, files.length);
+ File keystoreFile = CertificateManager.getContainerKeystoreFilePath(
+ "container1", "component1");
+ Assert.assertTrue("container keystore not generated",
+ keystoreFile.exists());
+
+ Assert.assertTrue("keystore not in returned list",
+ Arrays.asList(files).contains(new SecurityStore(keystoreFile,
+ SecurityStore.StoreType.keystore)));
+ File truststoreFile =
+ CertificateManager.getContainerTruststoreFilePath("component1",
+ "container1");
+ Assert.assertFalse("container truststore generated",
+ truststoreFile.exists());
+ Assert.assertFalse("truststore in returned list",
+ Arrays.asList(files).contains(new SecurityStore(truststoreFile,
+ SecurityStore.StoreType.truststore)));
+
+ }
+
+ @Test
+ public void testContainerGenerationUsingStoresGeneratorJustTruststoreWithDefaultAlias() throws Exception {
+ AggregateConf instanceDefinition = new AggregateConf();
+ MapOperations compOps = setupComponentOptions(true);
+
+ setupCredentials(instanceDefinition, null,
+ SliderKeys.COMP_TRUSTSTORE_PASSWORD_ALIAS_DEFAULT);
+
+ SecurityStore[]
+ files = StoresGenerator.generateSecurityStores("testhost",
+ "container1",
+ "component1",
+ instanceDefinition,
+ compOps);
+ assertEquals("wrong number of stores", 1, files.length);
+ File keystoreFile = CertificateManager.getContainerKeystoreFilePath(
+ "container1", "component1");
+ Assert.assertFalse("container keystore generated",
+ keystoreFile.exists());
+ Assert.assertFalse("keystore in returned list",
+ Arrays.asList(files).contains(keystoreFile));
+ File truststoreFile =
+ CertificateManager.getContainerTruststoreFilePath("component1",
+ "container1");
+ Assert.assertTrue("container truststore not generated",
+ truststoreFile.exists());
+ Assert.assertTrue("truststore not in returned list",
+ Arrays.asList(files).contains(new SecurityStore(truststoreFile,
+ SecurityStore.StoreType.truststore)));
+
+ }
+
+ @Test
+ public void testContainerTrusttoreGenerationUsingStoresGenerator() throws Exception {
+ AggregateConf instanceDefinition = new AggregateConf();
+ MapOperations compOps = setupComponentOptions(true,
+ "test.keystore.password",
+ null,
+ "test.truststore.password",
+ null);
+
+ setupCredentials(instanceDefinition, "test.keystore.password",
+ "test.truststore.password");
+
+ SecurityStore[]
+ files = StoresGenerator.generateSecurityStores("testhost",
+ "container1",
+ "component1",
+ instanceDefinition,
+ compOps);
+ assertEquals("wrong number of stores", 2, files.length);
+ File keystoreFile = CertificateManager.getContainerKeystoreFilePath(
+ "container1", "component1");
+ Assert.assertTrue("container keystore not generated",
+ keystoreFile.exists());
+ Assert.assertTrue("keystore not in returned list",
+ Arrays.asList(files).contains(new SecurityStore(keystoreFile,
+ SecurityStore.StoreType.keystore)));
+ File truststoreFile =
+ CertificateManager.getContainerTruststoreFilePath("component1",
+ "container1");
+ Assert.assertTrue("container truststore not generated",
+ truststoreFile.exists());
+ Assert.assertTrue("truststore not in returned list",
+ Arrays.asList(files).contains(new SecurityStore(truststoreFile,
+ SecurityStore.StoreType.truststore)));
+
+ validateTruststore(keystoreFile, truststoreFile);
+ }
+
+ private void setupCredentials(AggregateConf instanceDefinition,
+ String keyAlias, String trustAlias)
+ throws Exception {
+ Configuration conf = new Configuration();
+ final Path jksPath = new Path(SecurityUtils.getSecurityDir(), "test.jks");
+ final String ourUrl =
+ JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
+
+ File file = new File(SecurityUtils.getSecurityDir(), "test.jks");
+ file.delete();
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl);
+
+ instanceDefinition.getAppConf().credentials.put(ourUrl, new ArrayList<String>());
+
+ CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+
+ // create new aliases
+ try {
+
+ if (keyAlias != null) {
+ char[] storepass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
+ provider.createCredentialEntry(
+ keyAlias, storepass);
+ }
+
+ if (trustAlias != null) {
+ char[] trustpass = {'t', 'r', 'u', 's', 't', 'p', 'a', 's', 's'};
+ provider.createCredentialEntry(
+ trustAlias, trustpass);
+ }
+
+ // write out so that it can be found in checks
+ provider.flush();
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ private MapOperations setupComponentOptions(boolean storesRequired) {
+ return this.setupComponentOptions(storesRequired, null, null, null, null);
+ }
+
+ private MapOperations setupComponentOptions(boolean storesRequired,
+ String keyAlias,
+ String keyPwd,
+ String trustAlias,
+ String trustPwd) {
+ MapOperations compOps = new MapOperations();
+ compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY,
+ Boolean.toString(storesRequired));
+ if (keyAlias != null) {
+ compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_KEY,
+ "test.keystore.password");
+ }
+ if (trustAlias != null) {
+ compOps.put(SliderKeys.COMP_TRUSTSTORE_PASSWORD_ALIAS_KEY,
+ "test.truststore.password");
+ }
+ if (keyPwd != null) {
+ compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_PROPERTY_KEY,
+ keyPwd);
+ }
+ if (trustPwd != null) {
+ compOps.put(SliderKeys.COMP_TRUSTSTORE_PASSWORD_PROPERTY_KEY,
+ trustPwd);
+ }
+ return compOps;
+ }
+
+ @Test
+ public void testContainerStoresGenerationKeystoreOnly() throws Exception {
+ AggregateConf instanceDefinition = new AggregateConf();
+ MapOperations compOps = new MapOperations();
+ compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true");
+
+ setupCredentials(instanceDefinition,
+ SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_DEFAULT, null);
+
+ SecurityStore[]
+ files = StoresGenerator.generateSecurityStores("testhost",
+ "container1",
+ "component1",
+ instanceDefinition,
+ compOps);
+ assertEquals("wrong number of stores", 1, files.length);
+ File keystoreFile = CertificateManager.getContainerKeystoreFilePath(
+ "container1", "component1");
+ Assert.assertTrue("container keystore not generated",
+ keystoreFile.exists());
+ Assert.assertTrue("keystore not in returned list",
+ Arrays.asList(files).contains(new SecurityStore(keystoreFile,
+ SecurityStore.StoreType.keystore)));
+ File truststoreFile =
+ CertificateManager.getContainerTruststoreFilePath("component1",
+ "container1");
+ Assert.assertFalse("container truststore generated",
+ truststoreFile.exists());
+ Assert.assertFalse("truststore in returned list",
+ Arrays.asList(files).contains(new SecurityStore(truststoreFile,
+ SecurityStore.StoreType.truststore)));
+
+ }
+
+ @Test
+ public void testContainerStoresGenerationMisconfiguration() throws Exception {
+ AggregateConf instanceDefinition = new AggregateConf();
+ MapOperations compOps = new MapOperations();
+ compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true");
+
+ setupCredentials(instanceDefinition, "cant.be.found", null);
+
+ try {
+ StoresGenerator.generateSecurityStores("testhost", "container1",
+ "component1", instanceDefinition,
+ compOps);
+ Assert.fail("SliderException should have been generated");
+ } catch (SliderException e) {
+ // ignore - should be thrown
+ }
+ }
+
+ private void validateTruststore(File keystoreFile, File truststoreFile)
+ throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException {
+ InputStream keyis = null;
+ InputStream trustis = null;
+ try {
+
+ // create keystore
+ keyis = new FileInputStream(keystoreFile);
+ KeyStore keystore = KeyStore.getInstance("pkcs12");
+ String password = "keypass";
+ keystore.load(keyis, password.toCharArray());
+
+ // obtain server cert
+ Certificate certificate = keystore.getCertificate(
+ keystore.aliases().nextElement());
+ Assert.assertNotNull(certificate);
+
+ // create trust store from generated trust store file
+ trustis = new FileInputStream(truststoreFile);
+ KeyStore truststore = KeyStore.getInstance("pkcs12");
+ password = "trustpass";
+ truststore.load(trustis, password.toCharArray());
+
+ // validate keystore cert using trust store
+ TrustManagerFactory
+ trustManagerFactory =
+ TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+ trustManagerFactory.init(truststore);
+
+ for (TrustManager trustManager: trustManagerFactory.getTrustManagers()) {
+ if (trustManager instanceof X509TrustManager) {
+ X509TrustManager x509TrustManager = (X509TrustManager)trustManager;
+ x509TrustManager.checkServerTrusted(
+ new X509Certificate[] {(X509Certificate) certificate},
+ "RSA_EXPORT");
+ }
+ }
+
+ } finally {
+ if(null != keyis) {
+ keyis.close();
+ }
+ if(null != trustis) {
+ trustis.close();
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/security/TestMultiThreadedStoreGeneration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/security/TestMultiThreadedStoreGeneration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/security/TestMultiThreadedStoreGeneration.java
new file mode 100644
index 0000000..2e2ffce
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/security/TestMultiThreadedStoreGeneration.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.services.security;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.SliderException;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ *
+ */
+public class TestMultiThreadedStoreGeneration {
+
+ public static final int NUM_THREADS = 30;
+ @Rule
+ public TemporaryFolder workDir = new TemporaryFolder();;
+
+ private void setupCredentials(AggregateConf instanceDefinition,
+ String keyAlias, String trustAlias)
+ throws Exception {
+ Configuration conf = new Configuration();
+ final Path jksPath = new Path(SecurityUtils.getSecurityDir(), "test.jks");
+ final String ourUrl =
+ JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
+
+ File file = new File(SecurityUtils.getSecurityDir(), "test.jks");
+ file.delete();
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl);
+
+ instanceDefinition.getAppConf().credentials.put(ourUrl, new ArrayList<String>());
+
+ CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+
+ // create new aliases
+ try {
+
+ if (keyAlias != null) {
+ char[] storepass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
+ provider.createCredentialEntry(
+ keyAlias, storepass);
+ }
+
+ if (trustAlias != null) {
+ char[] trustpass = {'t', 'r', 'u', 's', 't', 'p', 'a', 's', 's'};
+ provider.createCredentialEntry(
+ trustAlias, trustpass);
+ }
+
+ // write out so that it can be found in checks
+ provider.flush();
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+
+ @Test
+ public void testMultiThreadedStoreGeneration() throws Exception {
+
+ CertificateManager certMan = new CertificateManager();
+ MapOperations compOperations = new MapOperations();
+ File secDir = new File(workDir.getRoot(), SliderKeys.SECURITY_DIR);
+ File keystoreFile = new File(secDir, SliderKeys.KEYSTORE_FILE_NAME);
+ compOperations.put(SliderXmlConfKeys.KEY_KEYSTORE_LOCATION,
+ keystoreFile.getAbsolutePath());
+ certMan.initialize(compOperations, "cahost", null, null);
+
+ final CountDownLatch latch = new CountDownLatch(1);
+ final List<SecurityStore> stores = new ArrayList<>();
+ List<Thread> threads = new ArrayList<>();
+ final AggregateConf instanceDefinition = new AggregateConf();
+
+ setupCredentials(instanceDefinition,
+ SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_DEFAULT, null);
+ final MapOperations compOps = new MapOperations();
+ compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true");
+
+ for (int i=0; i<NUM_THREADS; ++i) {
+ final int finalI = i;
+ Runnable runner = new Runnable() {
+ public void run() {
+ System.out.println ("----> In run");
+ try {
+ latch.await();
+ SecurityStore[] stores1 = StoresGenerator.generateSecurityStores(
+ "testhost",
+ "container" + finalI,
+ "component" + finalI,
+ instanceDefinition,
+ compOps);
+ System.out.println ("----> stores1" + stores1);
+ List<SecurityStore>
+ securityStores =
+ Arrays.asList(stores1);
+ stores.addAll(securityStores);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ } catch (SliderException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ };
+ Thread thread = new Thread(runner, "TestThread" + i);
+ threads.add(thread);
+ thread.start();
+ }
+ latch.countDown();
+ for (Thread t : threads) {
+ t.join();
+ }
+
+ for (int i=0; i < NUM_THREADS; i++) {
+ assertTrue("keystore " + i + " not generated", stores.get(i).getFile().exists());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/MockService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/MockService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/MockService.java
new file mode 100644
index 0000000..588f621
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/MockService.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.workflow;
+
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.service.ServiceStateException;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+public class MockService extends AbstractService {
+ private final boolean fail;
+ private final int lifespan;
+ private final ExecutorService executorService =
+ Executors.newSingleThreadExecutor();
+
+ MockService() {
+ this("mock", false, -1);
+ }
+
+ MockService(String name, boolean fail, int lifespan) {
+ super(name);
+ this.fail = fail;
+ this.lifespan = lifespan;
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ //act on the lifespan here
+ if (lifespan > 0) {
+ executorService.submit(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ Thread.sleep(lifespan);
+ } catch (InterruptedException ignored) {
+
+ }
+ finish();
+ }
+ });
+ } else {
+ if (lifespan == 0) {
+ finish();
+ } else {
+ //continue until told not to
+ }
+ }
+ }
+
+ void finish() {
+ if (fail) {
+ ServiceStateException e =
+ new ServiceStateException(getName() + " failed");
+
+ noteFailure(e);
+ stop();
+ throw e;
+ } else {
+ stop();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/ParentWorkflowTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/ParentWorkflowTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/ParentWorkflowTestBase.java
new file mode 100644
index 0000000..a11a1cf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/ParentWorkflowTestBase.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.workflow;
+
+import org.apache.hadoop.service.Service;
+
+/**
+ * Extends {@link WorkflowServiceTestBase} with parent-specific operations
+ * and logic to build up and run the parent service
+ */
+public abstract class ParentWorkflowTestBase extends WorkflowServiceTestBase {
+
+ /**
+ * Wait a second for the service parent to stop
+ * @param parent the service to wait for
+ */
+ protected void waitForParentToStop(ServiceParent parent) {
+ waitForParentToStop(parent, 1000);
+ }
+
+ /**
+ * Wait for the service parent to stop
+ * @param parent the service to wait for
+ * @param timeout time in milliseconds
+ */
+ protected void waitForParentToStop(ServiceParent parent, int timeout) {
+ boolean stop = parent.waitForServiceToStop(timeout);
+ if (!stop) {
+ logState(parent);
+ fail("Service failed to stop : after " + timeout + " millis " + parent);
+ }
+ }
+
+ /**
+ * Subclasses are require to implement this and return an instance of a
+ * ServiceParent
+ * @param services a possibly empty list of services
+ * @return an inited -but -not-started- service parent instance
+ */
+ protected abstract ServiceParent buildService(Service... services);
+
+ /**
+ * Use {@link #buildService(Service...)} to create service and then start it
+ * @param services
+ * @return
+ */
+ protected ServiceParent startService(Service... services) {
+ ServiceParent parent = buildService(services);
+ //expect service to start and stay started
+ parent.start();
+ return parent;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
new file mode 100644
index 0000000..4a19417
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/ProcessCommandFactory.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.workflow;
+
+import org.apache.hadoop.util.Shell;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * A source of commands, with the goal being to allow for adding different
+ * implementations for different platforms
+ */
+public class ProcessCommandFactory {
+
+ protected ProcessCommandFactory() {
+ }
+
+ /**
+ * The command to list a directory
+ * @param dir directory
+ * @return commands
+ */
+ public List<String> ls(File dir) {
+ List<String> commands;
+ if (!Shell.WINDOWS) {
+ commands = Arrays.asList("ls","-1", dir.getAbsolutePath());
+ } else {
+ commands = Arrays.asList("cmd", "/c", "dir", dir.getAbsolutePath());
+ }
+ return commands;
+ }
+
+ /**
+ * Echo some text to stdout
+ * @param text text
+ * @return commands
+ */
+ public List<String> echo(String text) {
+ List<String> commands = new ArrayList<String>(5);
+ commands.add("echo");
+ commands.add(text);
+ return commands;
+ }
+
+ /**
+ * print env variables
+ * @return commands
+ */
+ public List<String> env() {
+ List<String> commands;
+ if (!Shell.WINDOWS) {
+ commands = Arrays.asList("env");
+ } else {
+ commands = Arrays.asList("cmd", "/c", "set");
+ }
+ return commands;
+ }
+
+ /**
+ * execute a command that returns with an error code that will
+ * be converted into a number
+ * @return commands
+ */
+ public List<String> exitFalse() {
+ List<String> commands = new ArrayList<String>(2);
+ commands.add("false");
+ return commands;
+ }
+
+ /**
+ * Create a process command factory for this OS
+ * @return
+ */
+ public static ProcessCommandFactory createProcessCommandFactory() {
+ return new ProcessCommandFactory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/SimpleRunnable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/SimpleRunnable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/SimpleRunnable.java
new file mode 100644
index 0000000..1f330f4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/SimpleRunnable.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.workflow;
+
+/**
+ * Test runnable that can be made to exit, or throw an exception
+ * during its run
+ */
+class SimpleRunnable implements Runnable {
+ boolean throwException = false;
+
+
+ SimpleRunnable() {
+ }
+
+ SimpleRunnable(boolean throwException) {
+ this.throwException = throwException;
+ }
+
+ @Override
+ public synchronized void run() {
+ try {
+ if (throwException) {
+ throw new RuntimeException("SimpleRunnable");
+ }
+ } finally {
+ this.notify();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowClosingService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowClosingService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowClosingService.java
new file mode 100644
index 0000000..39516b7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowClosingService.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.workflow;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+public class TestWorkflowClosingService extends WorkflowServiceTestBase {
+
+ @Test
+ public void testSimpleClose() throws Throwable {
+ ClosingService<OpenClose> svc = instance(false);
+ OpenClose openClose = svc.getCloseable();
+ assertFalse(openClose.closed);
+ svc.stop();
+ assertTrue(openClose.closed);
+ }
+
+ @Test
+ public void testNullClose() throws Throwable {
+ ClosingService<OpenClose> svc = new ClosingService<OpenClose>("", null);
+ svc.init(new Configuration());
+ svc.start();
+ assertNull(svc.getCloseable());
+ svc.stop();
+ }
+
+ @Test
+ public void testFailingClose() throws Throwable {
+ ClosingService<OpenClose> svc = instance(false);
+ OpenClose openClose = svc.getCloseable();
+ openClose.raiseExceptionOnClose = true;
+ svc.stop();
+ assertTrue(openClose.closed);
+ Throwable cause = svc.getFailureCause();
+ assertNotNull(cause);
+
+ //retry should be a no-op
+ svc.close();
+ }
+
+ @Test
+ public void testDoubleClose() throws Throwable {
+ ClosingService<OpenClose> svc = instance(false);
+ OpenClose openClose = svc.getCloseable();
+ openClose.raiseExceptionOnClose = true;
+ svc.stop();
+ assertTrue(openClose.closed);
+ Throwable cause = svc.getFailureCause();
+ assertNotNull(cause);
+ openClose.closed = false;
+ svc.stop();
+ assertEquals(cause, svc.getFailureCause());
+ }
+
+ /**
+ * This does not recurse forever, as the service has already entered the
+ * STOPPED state before the inner close tries to stop it -that operation
+ * is a no-op
+ * @throws Throwable
+ */
+ @Test
+ public void testCloseSelf() throws Throwable {
+ ClosingService<ClosingService> svc =
+ new ClosingService<ClosingService>("");
+ svc.setCloseable(svc);
+ svc.stop();
+ }
+
+
+ private ClosingService<OpenClose> instance(boolean raiseExceptionOnClose) {
+ ClosingService<OpenClose> svc = new ClosingService<OpenClose>(new OpenClose(
+ raiseExceptionOnClose));
+ svc.init(new Configuration());
+ svc.start();
+ return svc;
+ }
+
+ private static class OpenClose implements Closeable {
+ public boolean closed = false;
+ public boolean raiseExceptionOnClose;
+
+ private OpenClose(boolean raiseExceptionOnClose) {
+ this.raiseExceptionOnClose = raiseExceptionOnClose;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (!closed) {
+ closed = true;
+ if (raiseExceptionOnClose) {
+ throw new IOException("OpenClose");
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowCompositeService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowCompositeService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowCompositeService.java
new file mode 100644
index 0000000..5780149
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowCompositeService.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.workflow;
+
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.Service;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestWorkflowCompositeService extends ParentWorkflowTestBase {
+ private static final Logger
+ log = LoggerFactory.getLogger(TestWorkflowCompositeService.class);
+
+ @Test
+ public void testSingleChild() throws Throwable {
+ Service parent = startService(new MockService());
+ parent.stop();
+ }
+
+ @Test
+ public void testSingleChildTerminating() throws Throwable {
+ ServiceParent parent =
+ startService(new MockService("1", false, 100));
+ waitForParentToStop(parent);
+ }
+
+ @Test
+ public void testSingleChildFailing() throws Throwable {
+ ServiceParent parent =
+ startService(new MockService("1", true, 100));
+ waitForParentToStop(parent);
+ assert parent.getFailureCause() != null;
+ }
+
+ @Test
+ public void testTwoChildren() throws Throwable {
+ MockService one = new MockService("one", false, 100);
+ MockService two = new MockService("two", false, 100);
+ ServiceParent parent = startService(one, two);
+ waitForParentToStop(parent);
+ assertStopped(one);
+ assertStopped(two);
+ }
+
+ @Test
+ public void testCallableChild() throws Throwable {
+
+ MockService one = new MockService("one", false, 100);
+ CallableHandler handler = new CallableHandler("hello");
+ WorkflowCallbackService<String> ens =
+ new WorkflowCallbackService<String>("handler", handler, 100, true);
+ MockService two = new MockService("two", false, 100);
+ ServiceParent parent = startService(one, ens, two);
+ waitForParentToStop(parent);
+ assertStopped(one);
+ assertStopped(ens);
+ assertStopped(two);
+ assertTrue(handler.notified);
+ String s = ens.getScheduledFuture().get();
+ assertEquals("hello", s);
+ }
+
+ @Test
+ public void testNestedComposite() throws Throwable {
+ MockService one = new MockService("one", false, 100);
+ MockService two = new MockService("two", false, 100);
+ ServiceParent parent = buildService(one, two);
+ ServiceParent outer = startService(parent);
+ assertTrue(outer.waitForServiceToStop(1000));
+ assertStopped(one);
+ assertStopped(two);
+ }
+
+ @Test
+ public void testFailingComposite() throws Throwable {
+ MockService one = new MockService("one", true, 10);
+ MockService two = new MockService("two", false, 1000);
+ ServiceParent parent = startService(one, two);
+ waitForParentToStop(parent);
+ assertStopped(one);
+ assertStopped(two);
+ assertNotNull(one.getFailureCause());
+ assertNotNull(parent.getFailureCause());
+ assertEquals(one.getFailureCause(), parent.getFailureCause());
+ }
+
+ @Override
+ public ServiceParent buildService(Service... services) {
+ ServiceParent parent =
+ new WorkflowCompositeService("test", services);
+ parent.init(new Configuration());
+ return parent;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowExecutorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowExecutorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowExecutorService.java
new file mode 100644
index 0000000..dc160d9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowExecutorService.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.workflow;
+
+import org.junit.Test;
+
+import java.util.concurrent.ExecutorService;
+
+
+/**
+ * Basic tests for executor service
+ */
+public class TestWorkflowExecutorService extends WorkflowServiceTestBase {
+
+ @Test
+ public void testAsyncRun() throws Throwable {
+
+ ExecutorSvc svc = run(new ExecutorSvc());
+ ServiceTerminatingRunnable runnable = new ServiceTerminatingRunnable(svc,
+ new SimpleRunnable());
+
+ // synchronous in-thread execution
+ svc.execute(runnable);
+ Thread.sleep(1000);
+ assertStopped(svc);
+ }
+
+ @Test
+ public void testFailureRun() throws Throwable {
+
+ ExecutorSvc svc = run(new ExecutorSvc());
+ ServiceTerminatingRunnable runnable = new ServiceTerminatingRunnable(svc,
+ new SimpleRunnable(true));
+
+ // synchronous in-thread execution
+ svc.execute(runnable);
+ Thread.sleep(1000);
+ assertStopped(svc);
+ assertNotNull(runnable.getException());
+ }
+
+ private static class ExecutorSvc
+ extends WorkflowExecutorService<ExecutorService> {
+ private ExecutorSvc() {
+ super("ExecutorService",
+ ServiceThreadFactory.singleThreadExecutor("test", true));
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowRpcService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowRpcService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowRpcService.java
new file mode 100644
index 0000000..38cd9e1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowRpcService.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.workflow;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.Server;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+public class TestWorkflowRpcService extends WorkflowServiceTestBase {
+
+ @Test
+ public void testCreateMockRPCService() throws Throwable {
+ MockRPC rpc = new MockRPC();
+ rpc.start();
+ assertTrue(rpc.started);
+ rpc.getListenerAddress();
+ rpc.stop();
+ assertTrue(rpc.stopped);
+ }
+
+ @Test
+ public void testLifecycle() throws Throwable {
+ MockRPC rpc = new MockRPC();
+ WorkflowRpcService svc = new WorkflowRpcService("test", rpc);
+ run(svc);
+ assertTrue(rpc.started);
+ svc.getConnectAddress();
+ svc.stop();
+ assertTrue(rpc.stopped);
+ }
+
+ @Test
+ public void testStartFailure() throws Throwable {
+ MockRPC rpc = new MockRPC();
+ rpc.failOnStart = true;
+ WorkflowRpcService svc = new WorkflowRpcService("test", rpc);
+ svc.init(new Configuration());
+ try {
+ svc.start();
+ fail("expected an exception");
+ } catch (RuntimeException e) {
+ assertEquals("failOnStart", e.getMessage());
+ }
+ svc.stop();
+ assertTrue(rpc.stopped);
+ }
+
+ private static class MockRPC extends Server {
+
+ public boolean stopped;
+ public boolean started;
+ public boolean failOnStart;
+
+ private MockRPC() throws IOException {
+ super("localhost", 0, null, 1, new Configuration());
+ }
+
+ @Override
+ public synchronized void start() {
+ if (failOnStart) {
+ throw new RuntimeException("failOnStart");
+ }
+ started = true;
+ super.start();
+ }
+
+ @Override
+ public synchronized void stop() {
+ stopped = true;
+ super.stop();
+ }
+
+ @Override
+ public synchronized InetSocketAddress getListenerAddress() {
+ return super.getListenerAddress();
+ }
+
+ @Override
+ public Writable call(RPC.RpcKind rpcKind,
+ String protocol,
+ Writable param,
+ long receiveTime) throws Exception {
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowSequenceService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowSequenceService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowSequenceService.java
new file mode 100644
index 0000000..581e3ed
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowSequenceService.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.workflow;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.Service;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestWorkflowSequenceService extends ParentWorkflowTestBase {
+ private static final Logger
+ log = LoggerFactory.getLogger(TestWorkflowSequenceService.class);
+
+ @Test
+ public void testSingleSequence() throws Throwable {
+ ServiceParent parent = startService(new MockService());
+ parent.stop();
+ }
+
+ @Test
+ public void testEmptySequence() throws Throwable {
+ ServiceParent parent = startService();
+ waitForParentToStop(parent);
+ }
+
+ @Test
+ public void testSequence() throws Throwable {
+ MockService one = new MockService("one", false, 100);
+ MockService two = new MockService("two", false, 100);
+ ServiceParent parent = startService(one, two);
+ waitForParentToStop(parent);
+ assertStopped(one);
+ assertStopped(two);
+ assert ((WorkflowSequenceService) parent).getPreviousService().equals(two);
+ }
+
+ @Test
+ public void testCallableChild() throws Throwable {
+
+ MockService one = new MockService("one", false, 100);
+ CallableHandler handler = new CallableHandler("hello");
+ WorkflowCallbackService<String> ens =
+ new WorkflowCallbackService<String>("handler", handler, 100, true);
+ MockService two = new MockService("two", false, 100);
+ ServiceParent parent = startService(one, ens, two);
+ waitForParentToStop(parent);
+ assertStopped(one);
+ assertStopped(ens);
+ assertStopped(two);
+ assertTrue(handler.notified);
+ String s = ens.getScheduledFuture().get();
+ assertEquals("hello", s);
+ }
+
+
+ @Test
+ public void testFailingSequence() throws Throwable {
+ MockService one = new MockService("one", true, 100);
+ MockService two = new MockService("two", false, 100);
+ WorkflowSequenceService parent =
+ (WorkflowSequenceService) startService(one, two);
+ waitForParentToStop(parent);
+ assertStopped(one);
+ assertInState(two, Service.STATE.NOTINITED);
+ assertEquals(one, parent.getPreviousService());
+ }
+
+
+ @Test
+ public void testFailInStartNext() throws Throwable {
+ MockService one = new MockService("one", false, 100);
+ MockService two = new MockService("two", true, 0);
+ MockService three = new MockService("3", false, 0);
+ ServiceParent parent = startService(one, two, three);
+ waitForParentToStop(parent);
+ assertStopped(one);
+ assertStopped(two);
+ Throwable failureCause = two.getFailureCause();
+ assertNotNull(failureCause);
+ Throwable parentFailureCause = parent.getFailureCause();
+ assertNotNull(parentFailureCause);
+ assertEquals(parentFailureCause, failureCause);
+ assertInState(three, Service.STATE.NOTINITED);
+ }
+
+ @Test
+ public void testSequenceInSequence() throws Throwable {
+ MockService one = new MockService("one", false, 100);
+ MockService two = new MockService("two", false, 100);
+ ServiceParent parent = buildService(one, two);
+ ServiceParent outer = startService(parent);
+ waitForParentToStop(parent);
+ assertStopped(one);
+ assertStopped(two);
+ }
+
+ @Test
+ public void testVarargsConstructor() throws Throwable {
+ MockService one = new MockService("one", false, 100);
+ MockService two = new MockService("two", false, 100);
+ ServiceParent parent = new WorkflowSequenceService("test", one, two);
+ parent.init(new Configuration());
+ parent.start();
+ waitForParentToStop(parent);
+ assertStopped(one);
+ assertStopped(two);
+ }
+
+
+ @Test
+ public void testAddChild() throws Throwable {
+ MockService one = new MockService("one", false, 5000);
+ MockService two = new MockService("two", false, 100);
+ ServiceParent parent = startService(one, two);
+ CallableHandler handler = new CallableHandler("hello");
+ WorkflowCallbackService<String> ens =
+ new WorkflowCallbackService<String>("handler", handler, 100, true);
+ parent.addService(ens);
+ waitForParentToStop(parent, 10000);
+ assertStopped(one);
+ assertStopped(two);
+ assertStopped(ens);
+ assertStopped(two);
+ assertEquals("hello", ens.getScheduledFuture().get());
+ }
+
+ public WorkflowSequenceService buildService(Service... services) {
+ WorkflowSequenceService parent =
+ new WorkflowSequenceService("test", services);
+ parent.init(new Configuration());
+ return parent;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowServiceTerminatingRunnable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowServiceTerminatingRunnable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowServiceTerminatingRunnable.java
new file mode 100644
index 0000000..5b7a6f9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowServiceTerminatingRunnable.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.workflow;
+
+import org.junit.Test;
+
+
+public class TestWorkflowServiceTerminatingRunnable extends WorkflowServiceTestBase {
+
+ @Test
+ public void testNoservice() throws Throwable {
+
+ try {
+ new ServiceTerminatingRunnable(null, new SimpleRunnable());
+ fail("unexpected ");
+ } catch (IllegalArgumentException e) {
+
+ // expected
+ }
+ }
+
+
+ @Test
+ public void testBasicRun() throws Throwable {
+
+ WorkflowCompositeService svc = run(new WorkflowCompositeService());
+ ServiceTerminatingRunnable runnable = new ServiceTerminatingRunnable(svc,
+ new SimpleRunnable());
+
+ // synchronous in-thread execution
+ runnable.run();
+ assertStopped(svc);
+ }
+
+ @Test
+ public void testFailureRun() throws Throwable {
+
+ WorkflowCompositeService svc = run(new WorkflowCompositeService());
+ ServiceTerminatingRunnable runnable =
+ new ServiceTerminatingRunnable(svc, new SimpleRunnable(true));
+
+ // synchronous in-thread execution
+ runnable.run();
+ assertStopped(svc);
+ assertNotNull(runnable.getException());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4826d902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
new file mode 100644
index 0000000..f38bd9d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.services.workflow;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.Service;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.TestName;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Locale;
+import java.util.concurrent.Callable;
+
+/**
+ * Test base for workflow service tests.
+ */
+public abstract class WorkflowServiceTestBase extends Assert {
+ private static final Logger
+ log = LoggerFactory.getLogger(WorkflowServiceTestBase.class);
+
+ /**
+ * Set the timeout for every test
+ */
+ @Rule
+ public Timeout testTimeout = new Timeout(15000);
+
+ @Rule
+ public TestName name = new TestName();
+
+ @Before
+ public void nameThread() {
+ Thread.currentThread().setName("JUnit");
+ }
+
+
+ protected void assertInState(Service service, Service.STATE expected) {
+ Service.STATE actual = service.getServiceState();
+ if (actual != expected) {
+ fail("Service " + service.getName() + " in state " + actual
+ + " -expected " + expected);
+ }
+ }
+
+ protected void assertStopped(Service service) {
+ assertInState(service, Service.STATE.STOPPED);
+ }
+
+ protected void logState(ServiceParent p) {
+ logService(p);
+ for (Service s : p.getServices()) {
+ logService(s);
+ }
+ }
+
+ protected void logService(Service s) {
+ log.info(s.toString());
+ Throwable failureCause = s.getFailureCause();
+ if (failureCause != null) {
+ log.info("Failed in state {} with {}", s.getFailureState(),
+ failureCause);
+ }
+ }
+
+ /**
+ * Init and start a service
+ * @param svc the service
+ * @return the service
+ */
+ protected <S extends Service> S run(S svc) {
+ svc.init(new Configuration());
+ svc.start();
+ return svc;
+ }
+
+ /**
+ * Handler for callable events
+ */
+ public static class CallableHandler implements Callable<String> {
+ public volatile boolean notified = false;
+ public final String result;
+
+ public CallableHandler(String result) {
+ this.result = result;
+ }
+
+ @Override
+ public String call() throws Exception {
+ log.info("CallableHandler::call");
+ notified = true;
+ return result;
+ }
+ }
+
+ /**
+ * Assert that a string is in an output list. Fails fast if the output
+ * list is empty
+ * @param text text to scan for
+ * @param output list of output lines.
+ */
+ public void assertStringInOutput(String text, List<String> output) {
+ assertTrue("Empty output list", !output.isEmpty());
+ boolean found = false;
+ StringBuilder builder = new StringBuilder();
+ for (String s : output) {
+ builder.append(s.toLowerCase(Locale.ENGLISH)).append('\n');
+ if (s.contains(text)) {
+ found = true;
+ break;
+ }
+ }
+
+ if (!found) {
+ String message =
+ "Text \"" + text + "\" not found in " + output.size() + " lines\n";
+ fail(message + builder.toString());
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/51] [abbrv] hadoop git commit: YARN-5769. Integrate update app
lifetime using feature implemented in YARN-5611. Contributed by Jian He
Posted by ji...@apache.org.
YARN-5769. Integrate update app lifetime using feature implemented in YARN-5611. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23395f78
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23395f78
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23395f78
Branch: refs/heads/yarn-native-services
Commit: 23395f78bdc5ce23caeb003a3334fd9f2b171640
Parents: 076ae6b
Author: Gour Saha <go...@apache.org>
Authored: Wed Nov 30 23:24:45 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../api/impl/ApplicationApiService.java | 25 +++++++++++++--
.../org/apache/slider/client/SliderClient.java | 33 +++++++++++++++++---
.../AbstractClusterBuildingActionArgs.java | 3 +-
3 files changed, 53 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23395f78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index 6db69ac..c4f5d43 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -77,6 +77,7 @@ import org.apache.slider.common.params.ActionFreezeArgs;
import org.apache.slider.common.params.ActionListArgs;
import org.apache.slider.common.params.ActionRegistryArgs;
import org.apache.slider.common.params.ActionThawArgs;
+import org.apache.slider.common.params.ActionUpdateArgs;
import org.apache.slider.common.params.ComponentArgsDelegate;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.common.tools.SliderVersionInfo;
@@ -1398,14 +1399,34 @@ public class ApplicationApiService implements ApplicationApi {
}
// If new lifetime value specified then update it
- if (updateAppData.getLifetime() != null) {
- // TODO: Once YARN-3813 and YARN-4205 are available
+ if (updateAppData.getLifetime() != null
+ && updateAppData.getLifetime() > 0) {
+ try {
+ updateAppLifetime(appName, updateAppData.getLifetime());
+ } catch (Exception e) {
+ logger.error("Failed to update application (" + appName + ") lifetime ("
+ + updateAppData.getLifetime() + ")", e);
+ return Response.status(Status.INTERNAL_SERVER_ERROR).build();
+ }
}
// If nothing happens consider it a no-op
return Response.status(Status.NO_CONTENT).build();
}
+ private Void updateAppLifetime(String appName, long lifetime)
+ throws InterruptedException, YarnException, IOException {
+ return invokeSliderClientRunnable(new SliderClientContextRunnable<Void>() {
+ @Override public Void run(SliderClient sliderClient)
+ throws YarnException, IOException, InterruptedException {
+ ActionUpdateArgs args = new ActionUpdateArgs();
+ args.lifetime = lifetime;
+ sliderClient.actionUpdate(appName, args);
+ return null;
+ }
+ });
+ }
+
// create default component and initialize with app level global values
private List<Component> getDefaultComponentAsList(Application app) {
List<Component> comps = getDefaultComponentAsList();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23395f78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 3f2df0a..12f7870 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
@@ -65,6 +66,7 @@ import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Times;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.SliderApplicationApi;
@@ -177,14 +179,12 @@ import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.BufferedReader;
import java.io.Console;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.InputStreamReader;
import java.io.InterruptedIOException;
import java.io.PrintStream;
import java.io.PrintWriter;
@@ -1602,8 +1602,33 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
public int actionUpdate(String clustername,
AbstractClusterBuildingActionArgs buildInfo) throws
YarnException, IOException {
- buildInstanceDefinition(clustername, buildInfo, true, true);
- return EXIT_SUCCESS;
+ if (buildInfo.lifetime > 0) {
+ updateLifetime(clustername, buildInfo.lifetime);
+ } else {
+ buildInstanceDefinition(clustername, buildInfo, true, true);
+ }
+ return EXIT_SUCCESS;
+ }
+
+ public void updateLifetime(String appName, long lifetime)
+ throws YarnException, IOException {
+ ApplicationReport report = findInstance(appName);
+ if (report == null) {
+ throw new YarnException("Application not found for " + appName);
+ }
+ ApplicationId appId = report.getApplicationId();
+ log.info("Updating lifetime of an application: appName = " + appName
+ + ", appId = " + appId+ ", lifetime = " + lifetime);
+ Map<ApplicationTimeoutType, String> map = new HashMap<>();
+ String newTimeout =
+ Times.formatISO8601(System.currentTimeMillis() + lifetime * 1000);
+ map.put(ApplicationTimeoutType.LIFETIME, newTimeout);
+ UpdateApplicationTimeoutsRequest request =
+ UpdateApplicationTimeoutsRequest.newInstance(appId, map);
+ yarnClient.updateApplicationTimeouts(request);
+ log.info("Successfully updated lifetime for an application: appName = "
+ + appName + ", appId = " + appId
+ + ". New expiry time in ISO8601 format is " + newTimeout);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23395f78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
index 3cb75e1..20a9989 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
@@ -103,8 +103,7 @@ public abstract class AbstractClusterBuildingActionArgs extends
public String queue;
@Parameter(names = {ARG_LIFETIME},
- description = "Life time of the application since application started at"
- + " running state")
+ description = "Lifetime of the application from the time of request")
public long lifetime;
@ParametersDelegate
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/51] [abbrv] hadoop git commit: YARN-5689. Update native services
REST API to use agentless docker provider. Contributed by Billie Rinaldi &
Gour Saha
Posted by ji...@apache.org.
YARN-5689. Update native services REST API to use agentless docker provider. Contributed by Billie Rinaldi & Gour Saha
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3862e38b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3862e38b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3862e38b
Branch: refs/heads/yarn-native-services
Commit: 3862e38be9ec5d1f42b4cfd7177bd79592584d7e
Parents: dc02649
Author: Jian He <ji...@apache.org>
Authored: Thu Oct 13 11:34:58 2016 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 22 11:09:38 2016 -0800
----------------------------------------------------------------------
.../api/impl/ApplicationApiService.java | 251 ++++++-------------
.../yarn/services/utils/RestApiConstants.java | 3 -
.../api/impl/TestApplicationApiService.java | 6 +-
3 files changed, 79 insertions(+), 181 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3862e38b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index 9645696..0a62629 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -50,7 +50,6 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.Status;
-import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.SerializationUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.security.UserGroupInformation;
@@ -68,10 +67,12 @@ import org.apache.hadoop.yarn.services.resource.Configuration;
import org.apache.hadoop.yarn.services.resource.Container;
import org.apache.hadoop.yarn.services.resource.ContainerState;
import org.apache.hadoop.yarn.services.resource.Resource;
+import org.apache.slider.api.OptionKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.StateValues;
import org.apache.slider.client.SliderClient;
import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.params.ActionCreateArgs;
import org.apache.slider.common.params.ActionFlexArgs;
import org.apache.slider.common.params.ActionFreezeArgs;
@@ -88,12 +89,11 @@ import org.apache.slider.core.exceptions.NotFoundException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.exceptions.UnknownApplicationInstanceException;
import org.apache.slider.core.registry.docstore.ConfigFormat;
+import org.apache.slider.providers.docker.DockerKeys;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonNull;
import com.google.gson.JsonObject;
@@ -211,7 +211,8 @@ public class ApplicationApiService implements ApplicationApi {
application.setConfiguration(new Configuration());
}
addPropertyToConfiguration(application.getConfiguration(),
- PROPERTY_COMPONENT_TYPE, COMPONENT_TYPE_EXTERNAL);
+ SliderKeys.COMPONENT_TYPE_KEY,
+ SliderKeys.COMPONENT_TYPE_EXTERNAL_APP);
}
// resource
validateApplicationResource(application.getResource(), null, application
@@ -249,7 +250,8 @@ public class ApplicationApiService implements ApplicationApi {
comp.setConfiguration(new Configuration());
}
addPropertyToConfiguration(comp.getConfiguration(),
- PROPERTY_COMPONENT_TYPE, COMPONENT_TYPE_EXTERNAL);
+ SliderKeys.COMPONENT_TYPE_KEY,
+ SliderKeys.COMPONENT_TYPE_EXTERNAL_APP);
compNameArtifactIdMap.put(comp.getName(), comp.getArtifact().getId());
comp.setName(comp.getArtifact().getId());
}
@@ -339,9 +341,9 @@ public class ApplicationApiService implements ApplicationApi {
final ActionCreateArgs createArgs = new ActionCreateArgs();
addAppConfOptions(createArgs, application, compNameArtifactIdMap);
addResourceOptions(createArgs, application);
- String metainfoJson = getMetainfoJson(application, compNameArtifactIdMap);
- createArgs.appMetaInfoJson = metainfoJson;
+ createArgs.provider = DockerKeys.PROVIDER_DOCKER;
+
if (queueName != null && queueName.trim().length() > 0) {
createArgs.queue = queueName.trim();
}
@@ -388,17 +390,76 @@ public class ApplicationApiService implements ApplicationApi {
// }
Set<String> uniqueGlobalPropertyCache = new HashSet<>();
- if (application.getConfiguration() != null
- && application.getConfiguration().getProperties() != null) {
- for (Map.Entry<String, String> propEntry : application.getConfiguration()
- .getProperties().entrySet()) {
+ if (application.getConfiguration() != null) {
+ if (application.getConfiguration().getProperties() != null) {
+ for (Map.Entry<String, String> propEntry : application
+ .getConfiguration().getProperties().entrySet()) {
+ addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache,
+ propEntry.getKey(), propEntry.getValue());
+ }
+ }
+ List<ConfigFile> configFiles = application.getConfiguration().getFiles();
+ if (configFiles != null && !configFiles.isEmpty()) {
addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache,
- propEntry.getKey(), propEntry.getValue());
+ SliderKeys.AM_CONFIG_GENERATION, "true");
+ for (ConfigFile configFile : configFiles) {
+ addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache,
+ OptionKeys.CONF_FILE_PREFIX + configFile.getSrcFile() +
+ OptionKeys.NAME_SUFFIX, configFile.getDestFile());
+ addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache,
+ OptionKeys.CONF_FILE_PREFIX + configFile.getSrcFile() +
+ OptionKeys.TYPE_SUFFIX, configFile.getType().toString());
+ }
}
}
if (application.getComponents() != null) {
+
+ Map<String, String> appQuicklinks = application.getQuicklinks();
+ if (appQuicklinks != null) {
+ for (Map.Entry<String, String> quicklink : appQuicklinks.entrySet()) {
+ addOptionsIfNotPresent(appOptions, uniqueGlobalPropertyCache,
+ OptionKeys.EXPORT_PREFIX + quicklink.getKey(),
+ quicklink.getValue());
+ }
+ }
+
Map<String, String> placeholders = new HashMap<>();
+ placeholders.put(PLACEHOLDER_APP_NAME, application.getName());
for (Component comp : application.getComponents()) {
+ placeholders.put(PLACEHOLDER_APP_COMPONENT_NAME, comp.getName());
+ if (comp.getArtifact().getType() == Artifact.TypeEnum.DOCKER) {
+ appCompOptionTriples.addAll(Arrays.asList(comp.getName(),
+ DockerKeys.DOCKER_IMAGE, comp.getArtifact().getId() == null ?
+ application.getArtifact().getId() : comp.getArtifact().getId()));
+ appCompOptionTriples.addAll(Arrays.asList(comp.getName(),
+ DockerKeys.DOCKER_START_COMMAND, comp.getLaunchCommand() == null ?
+ replacePlaceholders(application.getLaunchCommand(), placeholders)
+ : replacePlaceholders(comp.getLaunchCommand(), placeholders)));
+ appCompOptionTriples.addAll(Arrays.asList(comp.getName(),
+ DockerKeys.DOCKER_NETWORK, DockerKeys.DEFAULT_DOCKER_NETWORK));
+ if (comp.getRunPrivilegedContainer() != null) {
+ appCompOptionTriples.addAll(Arrays.asList(comp.getName(),
+ DockerKeys.DOCKER_USE_PRIVILEGED,
+ comp.getRunPrivilegedContainer().toString()));
+ }
+ }
+
+ if (comp.getConfiguration() != null) {
+ List<ConfigFile> configFiles = comp.getConfiguration().getFiles();
+ if (configFiles != null && !configFiles.isEmpty()) {
+ appCompOptionTriples.addAll(Arrays.asList(comp.getName(),
+ SliderKeys.AM_CONFIG_GENERATION, "true"));
+ for (ConfigFile configFile : configFiles) {
+ appCompOptionTriples.addAll(Arrays.asList(comp.getName(),
+ OptionKeys.CONF_FILE_PREFIX + configFile.getSrcFile() +
+ OptionKeys.NAME_SUFFIX, configFile.getDestFile()));
+ appCompOptionTriples.addAll(Arrays.asList(comp.getName(),
+ OptionKeys.CONF_FILE_PREFIX + configFile.getSrcFile() +
+ OptionKeys.TYPE_SUFFIX, configFile.getType().toString()));
+ }
+ }
+ }
+
if (Boolean.TRUE.equals(comp.getUniqueComponentSupport())) {
for (int i = 1; i <= comp.getNumberOfContainers(); i++) {
placeholders.put(PLACEHOLDER_COMPONENT_ID, Integer.toString(i));
@@ -526,8 +587,8 @@ public class ApplicationApiService implements ApplicationApi {
if (application.getComponents() != null) {
int priority = 1;
for (Component comp : application.getComponents()) {
- if (hasPropertyWithValue(comp, PROPERTY_COMPONENT_TYPE,
- COMPONENT_TYPE_EXTERNAL)) {
+ if (hasPropertyWithValue(comp, SliderKeys.COMPONENT_TYPE_KEY,
+ SliderKeys.COMPONENT_TYPE_EXTERNAL_APP)) {
continue;
}
if (Boolean.TRUE.equals(comp.getUniqueComponentSupport())) {
@@ -595,168 +656,6 @@ public class ApplicationApiService implements ApplicationApi {
return resCompOptTriples;
}
- private String getMetainfoJson(Application application,
- Map<String, String> compNameArtifactIdMap) throws SliderException,
- IOException {
- JsonObject rootObj = new JsonObject();
- rootObj.addProperty("schemaVersion", METAINFO_SCHEMA_VERSION);
- JsonObject applicationObj = new JsonObject();
- rootObj.add("application", applicationObj);
- applicationObj.addProperty("name", application.getName().toUpperCase());
- JsonArray componentsArray = new JsonArray();
- applicationObj.add("components", componentsArray);
- JsonArray commandOrdersArray = new JsonArray();
- applicationObj.add("commandOrders", commandOrdersArray);
-
- JsonArray exportGroupsArray = new JsonArray();
- applicationObj.add("exportGroups", exportGroupsArray);
- // Use only one export group
- JsonObject exportGroup = new JsonObject();
- exportGroup.addProperty("name", EXPORT_GROUP_NAME);
- exportGroupsArray.add(exportGroup);
- JsonArray exportsArray = new JsonArray();
- exportGroup.add("exports", exportsArray);
-
- if (application.getComponents() != null) {
-
- // Set exports at application level
- Map<String, String> appQuicklinks = application.getQuicklinks();
- Map<String, String> placeholders = new HashMap<>();
- placeholders.put(PLACEHOLDER_APP_NAME, application.getName());
- if (appQuicklinks != null) {
- for (Map.Entry<String, String> quicklink : appQuicklinks.entrySet()) {
- JsonObject export = new JsonObject();
- export.addProperty("name", quicklink.getKey());
- export.addProperty("value",
- replacePlaceholders(quicklink.getValue(), placeholders));
- exportsArray.add(export);
- }
- }
-
- for (Component comp : application.getComponents()) {
- JsonObject compObject = null;
- if (!hasPropertyWithValue(comp, PROPERTY_COMPONENT_TYPE,
- COMPONENT_TYPE_EXTERNAL)) {
- if (Boolean.TRUE.equals(comp.getUniqueComponentSupport())) {
- for (int i = 1; i <= comp.getNumberOfContainers(); i++) {
- // we also need the capability to specify ports and mount points
- // sometime
- compObject = createMetainfoComponent(comp, application,
- comp.getName() + i);
- componentsArray.add(compObject);
- }
- } else {
- compObject = createMetainfoComponent(comp, application,
- comp.getName());
- componentsArray.add(compObject);
- }
- }
-
- // Translate dependencies into command orders
- List<String> dependencies = comp.getDependencies();
- if (dependencies != null && !dependencies.isEmpty()) {
- JsonObject commandOrder = new JsonObject();
- commandOrder.addProperty("command", comp.getName()
- + COMMAND_ORDER_SUFFIX_START);
- for (String dependency : dependencies) {
- // If APPLICATION type artifact then map component name dependencies
- // to artifact id
- if (comp.getArtifact().getType() == Artifact.TypeEnum.APPLICATION) {
- dependency = compNameArtifactIdMap.get(dependency);
- }
- commandOrder.addProperty("requires", dependency
- + COMMAND_ORDER_SUFFIX_STARTED);
- }
- commandOrdersArray.add(commandOrder);
- }
-
- // Quicklinks need to be added as appExports and componentExports at the
- // component level
- List<String> compQuicklinks = comp.getQuicklinks();
- if (compQuicklinks != null && !compQuicklinks.isEmpty()) {
- if (MapUtils.isEmpty(appQuicklinks)) {
- throw new SliderException(ERROR_QUICKLINKS_FOR_COMP_INVALID);
- }
- List<String> appExports = new ArrayList<>();
- JsonArray compExportsArray = new JsonArray();
- compObject.add("componentExports", compExportsArray);
-
- for (String quicklink : compQuicklinks) {
- appExports.add(EXPORT_GROUP_NAME + "-" + quicklink);
-
- JsonObject compExport = new JsonObject();
- compExport.addProperty("name", quicklink);
- compExport.addProperty("value", appQuicklinks.get(quicklink));
- compExportsArray.add(compExport);
- }
- compObject.addProperty("appExports",
- StringUtils.join(appExports, ","));
- // specify that there are published configs for this component
- compObject.addProperty("publishConfig", "true");
- }
- }
- }
-
- String jsonString = new GsonBuilder().setPrettyPrinting().create()
- .toJson(rootObj);
- logger.info("Metainfo = \n{}", jsonString);
- return jsonString;
- }
-
- private JsonObject createMetainfoComponent(Component comp,
- Application application, String compName) {
- JsonObject compObj = new JsonObject();
- compObj.addProperty("name", compName);
- // below is diff for each type
- if (comp.getArtifact() != null && comp.getArtifact().getType() != null
- && comp.getArtifact().getType() == Artifact.TypeEnum.DOCKER) {
- compObj.addProperty("type", COMPONENT_TYPE_YARN_DOCKER);
- JsonArray dockerContainerArray = new JsonArray();
- compObj.add("dockerContainers", dockerContainerArray);
- JsonObject dockerContainerObj = new JsonObject();
- dockerContainerArray.add(dockerContainerObj);
- dockerContainerObj.addProperty("name", compName.toLowerCase());
- // if image not specified, then use global value
- dockerContainerObj.addProperty("image",
- comp.getArtifact().getId() == null ? application.getArtifact()
- .getId() : comp.getArtifact().getId());
- // If launch command not specified, then use global value. Resolve all
- // placeholders.
- Map<String, String> placeholders = new HashMap<>();
- placeholders.put(PLACEHOLDER_APP_NAME, application.getName());
- placeholders.put(PLACEHOLDER_APP_COMPONENT_NAME, compName);
- dockerContainerObj.addProperty(
- "startCommand",
- comp.getLaunchCommand() == null ? replacePlaceholders(
- application.getLaunchCommand(), placeholders)
- : replacePlaceholders(comp.getLaunchCommand(), placeholders));
- dockerContainerObj.addProperty("network", DEFAULT_NETWORK);
- dockerContainerObj.addProperty("commandPath", DEFAULT_COMMAND_PATH);
- // TODO: What to do with privContainer ?
- dockerContainerObj.addProperty("runPrivilegedContainer",
- comp.getRunPrivilegedContainer());
- if (comp.getConfiguration() != null) {
- List<ConfigFile> configFiles = comp.getConfiguration().getFiles();
- if (configFiles != null && !configFiles.isEmpty()) {
- JsonArray configFileArray = new JsonArray();
- for (ConfigFile configFile : configFiles) {
- JsonObject configFileObj = new JsonObject();
- configFileObj.addProperty("type", configFile.getType().toString());
- configFileObj.addProperty("fileName", configFile.getDestFile());
- // TODO: add all properties which should include dictionaryName
- configFileObj.addProperty("dictionaryName",
- configFile.getSrcFile());
- configFileArray.add(configFileObj);
- }
- dockerContainerObj.add("configFiles", configFileArray);
- }
- }
- // we also need to specify artifact_management_service sometime
- }
- // we also need the capability to specify ports and mount points sometime
- return compObj;
- }
-
private static UserGroupInformation getSliderUser() {
if (SLIDER_USER != null) {
return SLIDER_USER;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3862e38b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java
index 4c16546..23b7ad4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/utils/RestApiConstants.java
@@ -49,11 +49,8 @@ public interface RestApiConstants {
Integer GET_APPLICATIONS_THREAD_POOL_SIZE = 200;
String PROPERTY_PYTHON_PATH = "python.path";
- String PROPERTY_COMPONENT_TYPE = "site.global.component_type";
String PROPERTY_DNS_DEPENDENCY = "site.global.dns.dependency";
- String COMPONENT_TYPE_EXTERNAL = "external";
-
String COMMAND_ORDER_SUFFIX_START = "-START";
String COMMAND_ORDER_SUFFIX_STARTED = "-STARTED";
String EXPORT_GROUP_NAME = "QuickLinks";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3862e38b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java
index a03ab69..fdf1419 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java
@@ -26,6 +26,7 @@ import java.util.Map;
import org.apache.hadoop.yarn.services.resource.Application;
import org.apache.hadoop.yarn.services.resource.Artifact;
import org.apache.hadoop.yarn.services.resource.Resource;
+import org.apache.slider.common.SliderKeys;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -124,8 +125,9 @@ public class TestApplicationApiService {
DEFAULT_COMPONENT_NAME);
Assert.assertEquals(app.getLifetime(), DEFAULT_UNLIMITED_LIFETIME);
Assert.assertEquals("Property not set",
- app.getConfiguration().getProperties().get(PROPERTY_COMPONENT_TYPE),
- COMPONENT_TYPE_EXTERNAL);
+ app.getConfiguration().getProperties()
+ .get(SliderKeys.COMPONENT_TYPE_KEY),
+ SliderKeys.COMPONENT_TYPE_EXTERNAL_APP);
// unset artifact type, default component and no of containers to test other
// validation logic
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/51] [abbrv] hadoop git commit: YARN-5961. Generate native
services protobuf classes during build. Contributed by Billie Rinaldi
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62a4905/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/Messages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
deleted file mode 100644
index 373d64d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
+++ /dev/null
@@ -1,34473 +0,0 @@
-// Generated by the protocol buffer compiler. DO NOT EDIT!
-// source: SliderClusterMessages.proto
-
-package org.apache.slider.api.proto;
-
-public final class Messages {
- private Messages() {}
- public static void registerAllExtensions(
- com.google.protobuf.ExtensionRegistry registry) {
- }
- public interface RoleInstanceStateOrBuilder
- extends com.google.protobuf.MessageOrBuilder {
-
- // required string name = 1;
- /**
- * <code>required string name = 1;</code>
- */
- boolean hasName();
- /**
- * <code>required string name = 1;</code>
- */
- java.lang.String getName();
- /**
- * <code>required string name = 1;</code>
- */
- com.google.protobuf.ByteString
- getNameBytes();
-
- // optional string role = 2;
- /**
- * <code>optional string role = 2;</code>
- */
- boolean hasRole();
- /**
- * <code>optional string role = 2;</code>
- */
- java.lang.String getRole();
- /**
- * <code>optional string role = 2;</code>
- */
- com.google.protobuf.ByteString
- getRoleBytes();
-
- // required uint32 state = 4;
- /**
- * <code>required uint32 state = 4;</code>
- */
- boolean hasState();
- /**
- * <code>required uint32 state = 4;</code>
- */
- int getState();
-
- // required uint32 exitCode = 5;
- /**
- * <code>required uint32 exitCode = 5;</code>
- */
- boolean hasExitCode();
- /**
- * <code>required uint32 exitCode = 5;</code>
- */
- int getExitCode();
-
- // optional string command = 6;
- /**
- * <code>optional string command = 6;</code>
- */
- boolean hasCommand();
- /**
- * <code>optional string command = 6;</code>
- */
- java.lang.String getCommand();
- /**
- * <code>optional string command = 6;</code>
- */
- com.google.protobuf.ByteString
- getCommandBytes();
-
- // optional string diagnostics = 7;
- /**
- * <code>optional string diagnostics = 7;</code>
- */
- boolean hasDiagnostics();
- /**
- * <code>optional string diagnostics = 7;</code>
- */
- java.lang.String getDiagnostics();
- /**
- * <code>optional string diagnostics = 7;</code>
- */
- com.google.protobuf.ByteString
- getDiagnosticsBytes();
-
- // repeated string output = 8;
- /**
- * <code>repeated string output = 8;</code>
- */
- java.util.List<java.lang.String>
- getOutputList();
- /**
- * <code>repeated string output = 8;</code>
- */
- int getOutputCount();
- /**
- * <code>repeated string output = 8;</code>
- */
- java.lang.String getOutput(int index);
- /**
- * <code>repeated string output = 8;</code>
- */
- com.google.protobuf.ByteString
- getOutputBytes(int index);
-
- // repeated string environment = 9;
- /**
- * <code>repeated string environment = 9;</code>
- */
- java.util.List<java.lang.String>
- getEnvironmentList();
- /**
- * <code>repeated string environment = 9;</code>
- */
- int getEnvironmentCount();
- /**
- * <code>repeated string environment = 9;</code>
- */
- java.lang.String getEnvironment(int index);
- /**
- * <code>repeated string environment = 9;</code>
- */
- com.google.protobuf.ByteString
- getEnvironmentBytes(int index);
-
- // required uint32 roleId = 10;
- /**
- * <code>required uint32 roleId = 10;</code>
- */
- boolean hasRoleId();
- /**
- * <code>required uint32 roleId = 10;</code>
- */
- int getRoleId();
-
- // required bool released = 11;
- /**
- * <code>required bool released = 11;</code>
- */
- boolean hasReleased();
- /**
- * <code>required bool released = 11;</code>
- */
- boolean getReleased();
-
- // required int64 createTime = 12;
- /**
- * <code>required int64 createTime = 12;</code>
- */
- boolean hasCreateTime();
- /**
- * <code>required int64 createTime = 12;</code>
- */
- long getCreateTime();
-
- // required int64 startTime = 13;
- /**
- * <code>required int64 startTime = 13;</code>
- */
- boolean hasStartTime();
- /**
- * <code>required int64 startTime = 13;</code>
- */
- long getStartTime();
-
- // required string host = 14;
- /**
- * <code>required string host = 14;</code>
- */
- boolean hasHost();
- /**
- * <code>required string host = 14;</code>
- */
- java.lang.String getHost();
- /**
- * <code>required string host = 14;</code>
- */
- com.google.protobuf.ByteString
- getHostBytes();
-
- // required string hostURL = 15;
- /**
- * <code>required string hostURL = 15;</code>
- */
- boolean hasHostURL();
- /**
- * <code>required string hostURL = 15;</code>
- */
- java.lang.String getHostURL();
- /**
- * <code>required string hostURL = 15;</code>
- */
- com.google.protobuf.ByteString
- getHostURLBytes();
-
- // optional string appVersion = 16;
- /**
- * <code>optional string appVersion = 16;</code>
- */
- boolean hasAppVersion();
- /**
- * <code>optional string appVersion = 16;</code>
- */
- java.lang.String getAppVersion();
- /**
- * <code>optional string appVersion = 16;</code>
- */
- com.google.protobuf.ByteString
- getAppVersionBytes();
- }
- /**
- * Protobuf type {@code org.apache.slider.api.RoleInstanceState}
- */
- public static final class RoleInstanceState extends
- com.google.protobuf.GeneratedMessage
- implements RoleInstanceStateOrBuilder {
- // Use RoleInstanceState.newBuilder() to construct.
- private RoleInstanceState(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
- super(builder);
- this.unknownFields = builder.getUnknownFields();
- }
- private RoleInstanceState(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
- private static final RoleInstanceState defaultInstance;
- public static RoleInstanceState getDefaultInstance() {
- return defaultInstance;
- }
-
- public RoleInstanceState getDefaultInstanceForType() {
- return defaultInstance;
- }
-
- private final com.google.protobuf.UnknownFieldSet unknownFields;
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
- private RoleInstanceState(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- initFields();
- int mutable_bitField0_ = 0;
- com.google.protobuf.UnknownFieldSet.Builder unknownFields =
- com.google.protobuf.UnknownFieldSet.newBuilder();
- try {
- boolean done = false;
- while (!done) {
- int tag = input.readTag();
- switch (tag) {
- case 0:
- done = true;
- break;
- default: {
- if (!parseUnknownField(input, unknownFields,
- extensionRegistry, tag)) {
- done = true;
- }
- break;
- }
- case 10: {
- bitField0_ |= 0x00000001;
- name_ = input.readBytes();
- break;
- }
- case 18: {
- bitField0_ |= 0x00000002;
- role_ = input.readBytes();
- break;
- }
- case 32: {
- bitField0_ |= 0x00000004;
- state_ = input.readUInt32();
- break;
- }
- case 40: {
- bitField0_ |= 0x00000008;
- exitCode_ = input.readUInt32();
- break;
- }
- case 50: {
- bitField0_ |= 0x00000010;
- command_ = input.readBytes();
- break;
- }
- case 58: {
- bitField0_ |= 0x00000020;
- diagnostics_ = input.readBytes();
- break;
- }
- case 66: {
- if (!((mutable_bitField0_ & 0x00000040) == 0x00000040)) {
- output_ = new com.google.protobuf.LazyStringArrayList();
- mutable_bitField0_ |= 0x00000040;
- }
- output_.add(input.readBytes());
- break;
- }
- case 74: {
- if (!((mutable_bitField0_ & 0x00000080) == 0x00000080)) {
- environment_ = new com.google.protobuf.LazyStringArrayList();
- mutable_bitField0_ |= 0x00000080;
- }
- environment_.add(input.readBytes());
- break;
- }
- case 80: {
- bitField0_ |= 0x00000040;
- roleId_ = input.readUInt32();
- break;
- }
- case 88: {
- bitField0_ |= 0x00000080;
- released_ = input.readBool();
- break;
- }
- case 96: {
- bitField0_ |= 0x00000100;
- createTime_ = input.readInt64();
- break;
- }
- case 104: {
- bitField0_ |= 0x00000200;
- startTime_ = input.readInt64();
- break;
- }
- case 114: {
- bitField0_ |= 0x00000400;
- host_ = input.readBytes();
- break;
- }
- case 122: {
- bitField0_ |= 0x00000800;
- hostURL_ = input.readBytes();
- break;
- }
- case 130: {
- bitField0_ |= 0x00001000;
- appVersion_ = input.readBytes();
- break;
- }
- }
- }
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- throw e.setUnfinishedMessage(this);
- } catch (java.io.IOException e) {
- throw new com.google.protobuf.InvalidProtocolBufferException(
- e.getMessage()).setUnfinishedMessage(this);
- } finally {
- if (((mutable_bitField0_ & 0x00000040) == 0x00000040)) {
- output_ = new com.google.protobuf.UnmodifiableLazyStringList(output_);
- }
- if (((mutable_bitField0_ & 0x00000080) == 0x00000080)) {
- environment_ = new com.google.protobuf.UnmodifiableLazyStringList(environment_);
- }
- this.unknownFields = unknownFields.build();
- makeExtensionsImmutable();
- }
- }
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_RoleInstanceState_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_RoleInstanceState_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.slider.api.proto.Messages.RoleInstanceState.class, org.apache.slider.api.proto.Messages.RoleInstanceState.Builder.class);
- }
-
- public static com.google.protobuf.Parser<RoleInstanceState> PARSER =
- new com.google.protobuf.AbstractParser<RoleInstanceState>() {
- public RoleInstanceState parsePartialFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return new RoleInstanceState(input, extensionRegistry);
- }
- };
-
- @java.lang.Override
- public com.google.protobuf.Parser<RoleInstanceState> getParserForType() {
- return PARSER;
- }
-
- private int bitField0_;
- // required string name = 1;
- public static final int NAME_FIELD_NUMBER = 1;
- private java.lang.Object name_;
- /**
- * <code>required string name = 1;</code>
- */
- public boolean hasName() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>required string name = 1;</code>
- */
- public java.lang.String getName() {
- java.lang.Object ref = name_;
- if (ref instanceof java.lang.String) {
- return (java.lang.String) ref;
- } else {
- com.google.protobuf.ByteString bs =
- (com.google.protobuf.ByteString) ref;
- java.lang.String s = bs.toStringUtf8();
- if (bs.isValidUtf8()) {
- name_ = s;
- }
- return s;
- }
- }
- /**
- * <code>required string name = 1;</code>
- */
- public com.google.protobuf.ByteString
- getNameBytes() {
- java.lang.Object ref = name_;
- if (ref instanceof java.lang.String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- name_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
-
- // optional string role = 2;
- public static final int ROLE_FIELD_NUMBER = 2;
- private java.lang.Object role_;
- /**
- * <code>optional string role = 2;</code>
- */
- public boolean hasRole() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
- }
- /**
- * <code>optional string role = 2;</code>
- */
- public java.lang.String getRole() {
- java.lang.Object ref = role_;
- if (ref instanceof java.lang.String) {
- return (java.lang.String) ref;
- } else {
- com.google.protobuf.ByteString bs =
- (com.google.protobuf.ByteString) ref;
- java.lang.String s = bs.toStringUtf8();
- if (bs.isValidUtf8()) {
- role_ = s;
- }
- return s;
- }
- }
- /**
- * <code>optional string role = 2;</code>
- */
- public com.google.protobuf.ByteString
- getRoleBytes() {
- java.lang.Object ref = role_;
- if (ref instanceof java.lang.String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- role_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
-
- // required uint32 state = 4;
- public static final int STATE_FIELD_NUMBER = 4;
- private int state_;
- /**
- * <code>required uint32 state = 4;</code>
- */
- public boolean hasState() {
- return ((bitField0_ & 0x00000004) == 0x00000004);
- }
- /**
- * <code>required uint32 state = 4;</code>
- */
- public int getState() {
- return state_;
- }
-
- // required uint32 exitCode = 5;
- public static final int EXITCODE_FIELD_NUMBER = 5;
- private int exitCode_;
- /**
- * <code>required uint32 exitCode = 5;</code>
- */
- public boolean hasExitCode() {
- return ((bitField0_ & 0x00000008) == 0x00000008);
- }
- /**
- * <code>required uint32 exitCode = 5;</code>
- */
- public int getExitCode() {
- return exitCode_;
- }
-
- // optional string command = 6;
- public static final int COMMAND_FIELD_NUMBER = 6;
- private java.lang.Object command_;
- /**
- * <code>optional string command = 6;</code>
- */
- public boolean hasCommand() {
- return ((bitField0_ & 0x00000010) == 0x00000010);
- }
- /**
- * <code>optional string command = 6;</code>
- */
- public java.lang.String getCommand() {
- java.lang.Object ref = command_;
- if (ref instanceof java.lang.String) {
- return (java.lang.String) ref;
- } else {
- com.google.protobuf.ByteString bs =
- (com.google.protobuf.ByteString) ref;
- java.lang.String s = bs.toStringUtf8();
- if (bs.isValidUtf8()) {
- command_ = s;
- }
- return s;
- }
- }
- /**
- * <code>optional string command = 6;</code>
- */
- public com.google.protobuf.ByteString
- getCommandBytes() {
- java.lang.Object ref = command_;
- if (ref instanceof java.lang.String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- command_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
-
- // optional string diagnostics = 7;
- public static final int DIAGNOSTICS_FIELD_NUMBER = 7;
- private java.lang.Object diagnostics_;
- /**
- * <code>optional string diagnostics = 7;</code>
- */
- public boolean hasDiagnostics() {
- return ((bitField0_ & 0x00000020) == 0x00000020);
- }
- /**
- * <code>optional string diagnostics = 7;</code>
- */
- public java.lang.String getDiagnostics() {
- java.lang.Object ref = diagnostics_;
- if (ref instanceof java.lang.String) {
- return (java.lang.String) ref;
- } else {
- com.google.protobuf.ByteString bs =
- (com.google.protobuf.ByteString) ref;
- java.lang.String s = bs.toStringUtf8();
- if (bs.isValidUtf8()) {
- diagnostics_ = s;
- }
- return s;
- }
- }
- /**
- * <code>optional string diagnostics = 7;</code>
- */
- public com.google.protobuf.ByteString
- getDiagnosticsBytes() {
- java.lang.Object ref = diagnostics_;
- if (ref instanceof java.lang.String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- diagnostics_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
-
- // repeated string output = 8;
- public static final int OUTPUT_FIELD_NUMBER = 8;
- private com.google.protobuf.LazyStringList output_;
- /**
- * <code>repeated string output = 8;</code>
- */
- public java.util.List<java.lang.String>
- getOutputList() {
- return output_;
- }
- /**
- * <code>repeated string output = 8;</code>
- */
- public int getOutputCount() {
- return output_.size();
- }
- /**
- * <code>repeated string output = 8;</code>
- */
- public java.lang.String getOutput(int index) {
- return output_.get(index);
- }
- /**
- * <code>repeated string output = 8;</code>
- */
- public com.google.protobuf.ByteString
- getOutputBytes(int index) {
- return output_.getByteString(index);
- }
-
- // repeated string environment = 9;
- public static final int ENVIRONMENT_FIELD_NUMBER = 9;
- private com.google.protobuf.LazyStringList environment_;
- /**
- * <code>repeated string environment = 9;</code>
- */
- public java.util.List<java.lang.String>
- getEnvironmentList() {
- return environment_;
- }
- /**
- * <code>repeated string environment = 9;</code>
- */
- public int getEnvironmentCount() {
- return environment_.size();
- }
- /**
- * <code>repeated string environment = 9;</code>
- */
- public java.lang.String getEnvironment(int index) {
- return environment_.get(index);
- }
- /**
- * <code>repeated string environment = 9;</code>
- */
- public com.google.protobuf.ByteString
- getEnvironmentBytes(int index) {
- return environment_.getByteString(index);
- }
-
- // required uint32 roleId = 10;
- public static final int ROLEID_FIELD_NUMBER = 10;
- private int roleId_;
- /**
- * <code>required uint32 roleId = 10;</code>
- */
- public boolean hasRoleId() {
- return ((bitField0_ & 0x00000040) == 0x00000040);
- }
- /**
- * <code>required uint32 roleId = 10;</code>
- */
- public int getRoleId() {
- return roleId_;
- }
-
- // required bool released = 11;
- public static final int RELEASED_FIELD_NUMBER = 11;
- private boolean released_;
- /**
- * <code>required bool released = 11;</code>
- */
- public boolean hasReleased() {
- return ((bitField0_ & 0x00000080) == 0x00000080);
- }
- /**
- * <code>required bool released = 11;</code>
- */
- public boolean getReleased() {
- return released_;
- }
-
- // required int64 createTime = 12;
- public static final int CREATETIME_FIELD_NUMBER = 12;
- private long createTime_;
- /**
- * <code>required int64 createTime = 12;</code>
- */
- public boolean hasCreateTime() {
- return ((bitField0_ & 0x00000100) == 0x00000100);
- }
- /**
- * <code>required int64 createTime = 12;</code>
- */
- public long getCreateTime() {
- return createTime_;
- }
-
- // required int64 startTime = 13;
- public static final int STARTTIME_FIELD_NUMBER = 13;
- private long startTime_;
- /**
- * <code>required int64 startTime = 13;</code>
- */
- public boolean hasStartTime() {
- return ((bitField0_ & 0x00000200) == 0x00000200);
- }
- /**
- * <code>required int64 startTime = 13;</code>
- */
- public long getStartTime() {
- return startTime_;
- }
-
- // required string host = 14;
- public static final int HOST_FIELD_NUMBER = 14;
- private java.lang.Object host_;
- /**
- * <code>required string host = 14;</code>
- */
- public boolean hasHost() {
- return ((bitField0_ & 0x00000400) == 0x00000400);
- }
- /**
- * <code>required string host = 14;</code>
- */
- public java.lang.String getHost() {
- java.lang.Object ref = host_;
- if (ref instanceof java.lang.String) {
- return (java.lang.String) ref;
- } else {
- com.google.protobuf.ByteString bs =
- (com.google.protobuf.ByteString) ref;
- java.lang.String s = bs.toStringUtf8();
- if (bs.isValidUtf8()) {
- host_ = s;
- }
- return s;
- }
- }
- /**
- * <code>required string host = 14;</code>
- */
- public com.google.protobuf.ByteString
- getHostBytes() {
- java.lang.Object ref = host_;
- if (ref instanceof java.lang.String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- host_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
-
- // required string hostURL = 15;
- public static final int HOSTURL_FIELD_NUMBER = 15;
- private java.lang.Object hostURL_;
- /**
- * <code>required string hostURL = 15;</code>
- */
- public boolean hasHostURL() {
- return ((bitField0_ & 0x00000800) == 0x00000800);
- }
- /**
- * <code>required string hostURL = 15;</code>
- */
- public java.lang.String getHostURL() {
- java.lang.Object ref = hostURL_;
- if (ref instanceof java.lang.String) {
- return (java.lang.String) ref;
- } else {
- com.google.protobuf.ByteString bs =
- (com.google.protobuf.ByteString) ref;
- java.lang.String s = bs.toStringUtf8();
- if (bs.isValidUtf8()) {
- hostURL_ = s;
- }
- return s;
- }
- }
- /**
- * <code>required string hostURL = 15;</code>
- */
- public com.google.protobuf.ByteString
- getHostURLBytes() {
- java.lang.Object ref = hostURL_;
- if (ref instanceof java.lang.String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- hostURL_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
-
- // optional string appVersion = 16;
- public static final int APPVERSION_FIELD_NUMBER = 16;
- private java.lang.Object appVersion_;
- /**
- * <code>optional string appVersion = 16;</code>
- */
- public boolean hasAppVersion() {
- return ((bitField0_ & 0x00001000) == 0x00001000);
- }
- /**
- * <code>optional string appVersion = 16;</code>
- */
- public java.lang.String getAppVersion() {
- java.lang.Object ref = appVersion_;
- if (ref instanceof java.lang.String) {
- return (java.lang.String) ref;
- } else {
- com.google.protobuf.ByteString bs =
- (com.google.protobuf.ByteString) ref;
- java.lang.String s = bs.toStringUtf8();
- if (bs.isValidUtf8()) {
- appVersion_ = s;
- }
- return s;
- }
- }
- /**
- * <code>optional string appVersion = 16;</code>
- */
- public com.google.protobuf.ByteString
- getAppVersionBytes() {
- java.lang.Object ref = appVersion_;
- if (ref instanceof java.lang.String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- appVersion_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
-
- private void initFields() {
- name_ = "";
- role_ = "";
- state_ = 0;
- exitCode_ = 0;
- command_ = "";
- diagnostics_ = "";
- output_ = com.google.protobuf.LazyStringArrayList.EMPTY;
- environment_ = com.google.protobuf.LazyStringArrayList.EMPTY;
- roleId_ = 0;
- released_ = false;
- createTime_ = 0L;
- startTime_ = 0L;
- host_ = "";
- hostURL_ = "";
- appVersion_ = "";
- }
- private byte memoizedIsInitialized = -1;
- public final boolean isInitialized() {
- byte isInitialized = memoizedIsInitialized;
- if (isInitialized != -1) return isInitialized == 1;
-
- if (!hasName()) {
- memoizedIsInitialized = 0;
- return false;
- }
- if (!hasState()) {
- memoizedIsInitialized = 0;
- return false;
- }
- if (!hasExitCode()) {
- memoizedIsInitialized = 0;
- return false;
- }
- if (!hasRoleId()) {
- memoizedIsInitialized = 0;
- return false;
- }
- if (!hasReleased()) {
- memoizedIsInitialized = 0;
- return false;
- }
- if (!hasCreateTime()) {
- memoizedIsInitialized = 0;
- return false;
- }
- if (!hasStartTime()) {
- memoizedIsInitialized = 0;
- return false;
- }
- if (!hasHost()) {
- memoizedIsInitialized = 0;
- return false;
- }
- if (!hasHostURL()) {
- memoizedIsInitialized = 0;
- return false;
- }
- memoizedIsInitialized = 1;
- return true;
- }
-
- public void writeTo(com.google.protobuf.CodedOutputStream output)
- throws java.io.IOException {
- getSerializedSize();
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- output.writeBytes(1, getNameBytes());
- }
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- output.writeBytes(2, getRoleBytes());
- }
- if (((bitField0_ & 0x00000004) == 0x00000004)) {
- output.writeUInt32(4, state_);
- }
- if (((bitField0_ & 0x00000008) == 0x00000008)) {
- output.writeUInt32(5, exitCode_);
- }
- if (((bitField0_ & 0x00000010) == 0x00000010)) {
- output.writeBytes(6, getCommandBytes());
- }
- if (((bitField0_ & 0x00000020) == 0x00000020)) {
- output.writeBytes(7, getDiagnosticsBytes());
- }
- for (int i = 0; i < output_.size(); i++) {
- output.writeBytes(8, output_.getByteString(i));
- }
- for (int i = 0; i < environment_.size(); i++) {
- output.writeBytes(9, environment_.getByteString(i));
- }
- if (((bitField0_ & 0x00000040) == 0x00000040)) {
- output.writeUInt32(10, roleId_);
- }
- if (((bitField0_ & 0x00000080) == 0x00000080)) {
- output.writeBool(11, released_);
- }
- if (((bitField0_ & 0x00000100) == 0x00000100)) {
- output.writeInt64(12, createTime_);
- }
- if (((bitField0_ & 0x00000200) == 0x00000200)) {
- output.writeInt64(13, startTime_);
- }
- if (((bitField0_ & 0x00000400) == 0x00000400)) {
- output.writeBytes(14, getHostBytes());
- }
- if (((bitField0_ & 0x00000800) == 0x00000800)) {
- output.writeBytes(15, getHostURLBytes());
- }
- if (((bitField0_ & 0x00001000) == 0x00001000)) {
- output.writeBytes(16, getAppVersionBytes());
- }
- getUnknownFields().writeTo(output);
- }
-
- private int memoizedSerializedSize = -1;
- public int getSerializedSize() {
- int size = memoizedSerializedSize;
- if (size != -1) return size;
-
- size = 0;
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(1, getNameBytes());
- }
- if (((bitField0_ & 0x00000002) == 0x00000002)) {
- size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(2, getRoleBytes());
- }
- if (((bitField0_ & 0x00000004) == 0x00000004)) {
- size += com.google.protobuf.CodedOutputStream
- .computeUInt32Size(4, state_);
- }
- if (((bitField0_ & 0x00000008) == 0x00000008)) {
- size += com.google.protobuf.CodedOutputStream
- .computeUInt32Size(5, exitCode_);
- }
- if (((bitField0_ & 0x00000010) == 0x00000010)) {
- size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(6, getCommandBytes());
- }
- if (((bitField0_ & 0x00000020) == 0x00000020)) {
- size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(7, getDiagnosticsBytes());
- }
- {
- int dataSize = 0;
- for (int i = 0; i < output_.size(); i++) {
- dataSize += com.google.protobuf.CodedOutputStream
- .computeBytesSizeNoTag(output_.getByteString(i));
- }
- size += dataSize;
- size += 1 * getOutputList().size();
- }
- {
- int dataSize = 0;
- for (int i = 0; i < environment_.size(); i++) {
- dataSize += com.google.protobuf.CodedOutputStream
- .computeBytesSizeNoTag(environment_.getByteString(i));
- }
- size += dataSize;
- size += 1 * getEnvironmentList().size();
- }
- if (((bitField0_ & 0x00000040) == 0x00000040)) {
- size += com.google.protobuf.CodedOutputStream
- .computeUInt32Size(10, roleId_);
- }
- if (((bitField0_ & 0x00000080) == 0x00000080)) {
- size += com.google.protobuf.CodedOutputStream
- .computeBoolSize(11, released_);
- }
- if (((bitField0_ & 0x00000100) == 0x00000100)) {
- size += com.google.protobuf.CodedOutputStream
- .computeInt64Size(12, createTime_);
- }
- if (((bitField0_ & 0x00000200) == 0x00000200)) {
- size += com.google.protobuf.CodedOutputStream
- .computeInt64Size(13, startTime_);
- }
- if (((bitField0_ & 0x00000400) == 0x00000400)) {
- size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(14, getHostBytes());
- }
- if (((bitField0_ & 0x00000800) == 0x00000800)) {
- size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(15, getHostURLBytes());
- }
- if (((bitField0_ & 0x00001000) == 0x00001000)) {
- size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(16, getAppVersionBytes());
- }
- size += getUnknownFields().getSerializedSize();
- memoizedSerializedSize = size;
- return size;
- }
-
- private static final long serialVersionUID = 0L;
- @java.lang.Override
- protected java.lang.Object writeReplace()
- throws java.io.ObjectStreamException {
- return super.writeReplace();
- }
-
- @java.lang.Override
- public boolean equals(final java.lang.Object obj) {
- if (obj == this) {
- return true;
- }
- if (!(obj instanceof org.apache.slider.api.proto.Messages.RoleInstanceState)) {
- return super.equals(obj);
- }
- org.apache.slider.api.proto.Messages.RoleInstanceState other = (org.apache.slider.api.proto.Messages.RoleInstanceState) obj;
-
- boolean result = true;
- result = result && (hasName() == other.hasName());
- if (hasName()) {
- result = result && getName()
- .equals(other.getName());
- }
- result = result && (hasRole() == other.hasRole());
- if (hasRole()) {
- result = result && getRole()
- .equals(other.getRole());
- }
- result = result && (hasState() == other.hasState());
- if (hasState()) {
- result = result && (getState()
- == other.getState());
- }
- result = result && (hasExitCode() == other.hasExitCode());
- if (hasExitCode()) {
- result = result && (getExitCode()
- == other.getExitCode());
- }
- result = result && (hasCommand() == other.hasCommand());
- if (hasCommand()) {
- result = result && getCommand()
- .equals(other.getCommand());
- }
- result = result && (hasDiagnostics() == other.hasDiagnostics());
- if (hasDiagnostics()) {
- result = result && getDiagnostics()
- .equals(other.getDiagnostics());
- }
- result = result && getOutputList()
- .equals(other.getOutputList());
- result = result && getEnvironmentList()
- .equals(other.getEnvironmentList());
- result = result && (hasRoleId() == other.hasRoleId());
- if (hasRoleId()) {
- result = result && (getRoleId()
- == other.getRoleId());
- }
- result = result && (hasReleased() == other.hasReleased());
- if (hasReleased()) {
- result = result && (getReleased()
- == other.getReleased());
- }
- result = result && (hasCreateTime() == other.hasCreateTime());
- if (hasCreateTime()) {
- result = result && (getCreateTime()
- == other.getCreateTime());
- }
- result = result && (hasStartTime() == other.hasStartTime());
- if (hasStartTime()) {
- result = result && (getStartTime()
- == other.getStartTime());
- }
- result = result && (hasHost() == other.hasHost());
- if (hasHost()) {
- result = result && getHost()
- .equals(other.getHost());
- }
- result = result && (hasHostURL() == other.hasHostURL());
- if (hasHostURL()) {
- result = result && getHostURL()
- .equals(other.getHostURL());
- }
- result = result && (hasAppVersion() == other.hasAppVersion());
- if (hasAppVersion()) {
- result = result && getAppVersion()
- .equals(other.getAppVersion());
- }
- result = result &&
- getUnknownFields().equals(other.getUnknownFields());
- return result;
- }
-
- private int memoizedHashCode = 0;
- @java.lang.Override
- public int hashCode() {
- if (memoizedHashCode != 0) {
- return memoizedHashCode;
- }
- int hash = 41;
- hash = (19 * hash) + getDescriptorForType().hashCode();
- if (hasName()) {
- hash = (37 * hash) + NAME_FIELD_NUMBER;
- hash = (53 * hash) + getName().hashCode();
- }
- if (hasRole()) {
- hash = (37 * hash) + ROLE_FIELD_NUMBER;
- hash = (53 * hash) + getRole().hashCode();
- }
- if (hasState()) {
- hash = (37 * hash) + STATE_FIELD_NUMBER;
- hash = (53 * hash) + getState();
- }
- if (hasExitCode()) {
- hash = (37 * hash) + EXITCODE_FIELD_NUMBER;
- hash = (53 * hash) + getExitCode();
- }
- if (hasCommand()) {
- hash = (37 * hash) + COMMAND_FIELD_NUMBER;
- hash = (53 * hash) + getCommand().hashCode();
- }
- if (hasDiagnostics()) {
- hash = (37 * hash) + DIAGNOSTICS_FIELD_NUMBER;
- hash = (53 * hash) + getDiagnostics().hashCode();
- }
- if (getOutputCount() > 0) {
- hash = (37 * hash) + OUTPUT_FIELD_NUMBER;
- hash = (53 * hash) + getOutputList().hashCode();
- }
- if (getEnvironmentCount() > 0) {
- hash = (37 * hash) + ENVIRONMENT_FIELD_NUMBER;
- hash = (53 * hash) + getEnvironmentList().hashCode();
- }
- if (hasRoleId()) {
- hash = (37 * hash) + ROLEID_FIELD_NUMBER;
- hash = (53 * hash) + getRoleId();
- }
- if (hasReleased()) {
- hash = (37 * hash) + RELEASED_FIELD_NUMBER;
- hash = (53 * hash) + hashBoolean(getReleased());
- }
- if (hasCreateTime()) {
- hash = (37 * hash) + CREATETIME_FIELD_NUMBER;
- hash = (53 * hash) + hashLong(getCreateTime());
- }
- if (hasStartTime()) {
- hash = (37 * hash) + STARTTIME_FIELD_NUMBER;
- hash = (53 * hash) + hashLong(getStartTime());
- }
- if (hasHost()) {
- hash = (37 * hash) + HOST_FIELD_NUMBER;
- hash = (53 * hash) + getHost().hashCode();
- }
- if (hasHostURL()) {
- hash = (37 * hash) + HOSTURL_FIELD_NUMBER;
- hash = (53 * hash) + getHostURL().hashCode();
- }
- if (hasAppVersion()) {
- hash = (37 * hash) + APPVERSION_FIELD_NUMBER;
- hash = (53 * hash) + getAppVersion().hashCode();
- }
- hash = (29 * hash) + getUnknownFields().hashCode();
- memoizedHashCode = hash;
- return hash;
- }
-
- public static org.apache.slider.api.proto.Messages.RoleInstanceState parseFrom(
- com.google.protobuf.ByteString data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.slider.api.proto.Messages.RoleInstanceState parseFrom(
- com.google.protobuf.ByteString data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.slider.api.proto.Messages.RoleInstanceState parseFrom(byte[] data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.slider.api.proto.Messages.RoleInstanceState parseFrom(
- byte[] data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.slider.api.proto.Messages.RoleInstanceState parseFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.slider.api.proto.Messages.RoleInstanceState parseFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
- public static org.apache.slider.api.proto.Messages.RoleInstanceState parseDelimitedFrom(java.io.InputStream input)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input);
- }
- public static org.apache.slider.api.proto.Messages.RoleInstanceState parseDelimitedFrom(
- java.io.InputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseDelimitedFrom(input, extensionRegistry);
- }
- public static org.apache.slider.api.proto.Messages.RoleInstanceState parseFrom(
- com.google.protobuf.CodedInputStream input)
- throws java.io.IOException {
- return PARSER.parseFrom(input);
- }
- public static org.apache.slider.api.proto.Messages.RoleInstanceState parseFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- return PARSER.parseFrom(input, extensionRegistry);
- }
-
- public static Builder newBuilder() { return Builder.create(); }
- public Builder newBuilderForType() { return newBuilder(); }
- public static Builder newBuilder(org.apache.slider.api.proto.Messages.RoleInstanceState prototype) {
- return newBuilder().mergeFrom(prototype);
- }
- public Builder toBuilder() { return newBuilder(this); }
-
- @java.lang.Override
- protected Builder newBuilderForType(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- Builder builder = new Builder(parent);
- return builder;
- }
- /**
- * Protobuf type {@code org.apache.slider.api.RoleInstanceState}
- */
- public static final class Builder extends
- com.google.protobuf.GeneratedMessage.Builder<Builder>
- implements org.apache.slider.api.proto.Messages.RoleInstanceStateOrBuilder {
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_RoleInstanceState_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_RoleInstanceState_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.slider.api.proto.Messages.RoleInstanceState.class, org.apache.slider.api.proto.Messages.RoleInstanceState.Builder.class);
- }
-
- // Construct using org.apache.slider.api.proto.Messages.RoleInstanceState.newBuilder()
- private Builder() {
- maybeForceBuilderInitialization();
- }
-
- private Builder(
- com.google.protobuf.GeneratedMessage.BuilderParent parent) {
- super(parent);
- maybeForceBuilderInitialization();
- }
- private void maybeForceBuilderInitialization() {
- if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
- }
- }
- private static Builder create() {
- return new Builder();
- }
-
- public Builder clear() {
- super.clear();
- name_ = "";
- bitField0_ = (bitField0_ & ~0x00000001);
- role_ = "";
- bitField0_ = (bitField0_ & ~0x00000002);
- state_ = 0;
- bitField0_ = (bitField0_ & ~0x00000004);
- exitCode_ = 0;
- bitField0_ = (bitField0_ & ~0x00000008);
- command_ = "";
- bitField0_ = (bitField0_ & ~0x00000010);
- diagnostics_ = "";
- bitField0_ = (bitField0_ & ~0x00000020);
- output_ = com.google.protobuf.LazyStringArrayList.EMPTY;
- bitField0_ = (bitField0_ & ~0x00000040);
- environment_ = com.google.protobuf.LazyStringArrayList.EMPTY;
- bitField0_ = (bitField0_ & ~0x00000080);
- roleId_ = 0;
- bitField0_ = (bitField0_ & ~0x00000100);
- released_ = false;
- bitField0_ = (bitField0_ & ~0x00000200);
- createTime_ = 0L;
- bitField0_ = (bitField0_ & ~0x00000400);
- startTime_ = 0L;
- bitField0_ = (bitField0_ & ~0x00000800);
- host_ = "";
- bitField0_ = (bitField0_ & ~0x00001000);
- hostURL_ = "";
- bitField0_ = (bitField0_ & ~0x00002000);
- appVersion_ = "";
- bitField0_ = (bitField0_ & ~0x00004000);
- return this;
- }
-
- public Builder clone() {
- return create().mergeFrom(buildPartial());
- }
-
- public com.google.protobuf.Descriptors.Descriptor
- getDescriptorForType() {
- return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_RoleInstanceState_descriptor;
- }
-
- public org.apache.slider.api.proto.Messages.RoleInstanceState getDefaultInstanceForType() {
- return org.apache.slider.api.proto.Messages.RoleInstanceState.getDefaultInstance();
- }
-
- public org.apache.slider.api.proto.Messages.RoleInstanceState build() {
- org.apache.slider.api.proto.Messages.RoleInstanceState result = buildPartial();
- if (!result.isInitialized()) {
- throw newUninitializedMessageException(result);
- }
- return result;
- }
-
- public org.apache.slider.api.proto.Messages.RoleInstanceState buildPartial() {
- org.apache.slider.api.proto.Messages.RoleInstanceState result = new org.apache.slider.api.proto.Messages.RoleInstanceState(this);
- int from_bitField0_ = bitField0_;
- int to_bitField0_ = 0;
- if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
- to_bitField0_ |= 0x00000001;
- }
- result.name_ = name_;
- if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
- to_bitField0_ |= 0x00000002;
- }
- result.role_ = role_;
- if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
- to_bitField0_ |= 0x00000004;
- }
- result.state_ = state_;
- if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
- to_bitField0_ |= 0x00000008;
- }
- result.exitCode_ = exitCode_;
- if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
- to_bitField0_ |= 0x00000010;
- }
- result.command_ = command_;
- if (((from_bitField0_ & 0x00000020) == 0x00000020)) {
- to_bitField0_ |= 0x00000020;
- }
- result.diagnostics_ = diagnostics_;
- if (((bitField0_ & 0x00000040) == 0x00000040)) {
- output_ = new com.google.protobuf.UnmodifiableLazyStringList(
- output_);
- bitField0_ = (bitField0_ & ~0x00000040);
- }
- result.output_ = output_;
- if (((bitField0_ & 0x00000080) == 0x00000080)) {
- environment_ = new com.google.protobuf.UnmodifiableLazyStringList(
- environment_);
- bitField0_ = (bitField0_ & ~0x00000080);
- }
- result.environment_ = environment_;
- if (((from_bitField0_ & 0x00000100) == 0x00000100)) {
- to_bitField0_ |= 0x00000040;
- }
- result.roleId_ = roleId_;
- if (((from_bitField0_ & 0x00000200) == 0x00000200)) {
- to_bitField0_ |= 0x00000080;
- }
- result.released_ = released_;
- if (((from_bitField0_ & 0x00000400) == 0x00000400)) {
- to_bitField0_ |= 0x00000100;
- }
- result.createTime_ = createTime_;
- if (((from_bitField0_ & 0x00000800) == 0x00000800)) {
- to_bitField0_ |= 0x00000200;
- }
- result.startTime_ = startTime_;
- if (((from_bitField0_ & 0x00001000) == 0x00001000)) {
- to_bitField0_ |= 0x00000400;
- }
- result.host_ = host_;
- if (((from_bitField0_ & 0x00002000) == 0x00002000)) {
- to_bitField0_ |= 0x00000800;
- }
- result.hostURL_ = hostURL_;
- if (((from_bitField0_ & 0x00004000) == 0x00004000)) {
- to_bitField0_ |= 0x00001000;
- }
- result.appVersion_ = appVersion_;
- result.bitField0_ = to_bitField0_;
- onBuilt();
- return result;
- }
-
- public Builder mergeFrom(com.google.protobuf.Message other) {
- if (other instanceof org.apache.slider.api.proto.Messages.RoleInstanceState) {
- return mergeFrom((org.apache.slider.api.proto.Messages.RoleInstanceState)other);
- } else {
- super.mergeFrom(other);
- return this;
- }
- }
-
- public Builder mergeFrom(org.apache.slider.api.proto.Messages.RoleInstanceState other) {
- if (other == org.apache.slider.api.proto.Messages.RoleInstanceState.getDefaultInstance()) return this;
- if (other.hasName()) {
- bitField0_ |= 0x00000001;
- name_ = other.name_;
- onChanged();
- }
- if (other.hasRole()) {
- bitField0_ |= 0x00000002;
- role_ = other.role_;
- onChanged();
- }
- if (other.hasState()) {
- setState(other.getState());
- }
- if (other.hasExitCode()) {
- setExitCode(other.getExitCode());
- }
- if (other.hasCommand()) {
- bitField0_ |= 0x00000010;
- command_ = other.command_;
- onChanged();
- }
- if (other.hasDiagnostics()) {
- bitField0_ |= 0x00000020;
- diagnostics_ = other.diagnostics_;
- onChanged();
- }
- if (!other.output_.isEmpty()) {
- if (output_.isEmpty()) {
- output_ = other.output_;
- bitField0_ = (bitField0_ & ~0x00000040);
- } else {
- ensureOutputIsMutable();
- output_.addAll(other.output_);
- }
- onChanged();
- }
- if (!other.environment_.isEmpty()) {
- if (environment_.isEmpty()) {
- environment_ = other.environment_;
- bitField0_ = (bitField0_ & ~0x00000080);
- } else {
- ensureEnvironmentIsMutable();
- environment_.addAll(other.environment_);
- }
- onChanged();
- }
- if (other.hasRoleId()) {
- setRoleId(other.getRoleId());
- }
- if (other.hasReleased()) {
- setReleased(other.getReleased());
- }
- if (other.hasCreateTime()) {
- setCreateTime(other.getCreateTime());
- }
- if (other.hasStartTime()) {
- setStartTime(other.getStartTime());
- }
- if (other.hasHost()) {
- bitField0_ |= 0x00001000;
- host_ = other.host_;
- onChanged();
- }
- if (other.hasHostURL()) {
- bitField0_ |= 0x00002000;
- hostURL_ = other.hostURL_;
- onChanged();
- }
- if (other.hasAppVersion()) {
- bitField0_ |= 0x00004000;
- appVersion_ = other.appVersion_;
- onChanged();
- }
- this.mergeUnknownFields(other.getUnknownFields());
- return this;
- }
-
- public final boolean isInitialized() {
- if (!hasName()) {
-
- return false;
- }
- if (!hasState()) {
-
- return false;
- }
- if (!hasExitCode()) {
-
- return false;
- }
- if (!hasRoleId()) {
-
- return false;
- }
- if (!hasReleased()) {
-
- return false;
- }
- if (!hasCreateTime()) {
-
- return false;
- }
- if (!hasStartTime()) {
-
- return false;
- }
- if (!hasHost()) {
-
- return false;
- }
- if (!hasHostURL()) {
-
- return false;
- }
- return true;
- }
-
- public Builder mergeFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws java.io.IOException {
- org.apache.slider.api.proto.Messages.RoleInstanceState parsedMessage = null;
- try {
- parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- parsedMessage = (org.apache.slider.api.proto.Messages.RoleInstanceState) e.getUnfinishedMessage();
- throw e;
- } finally {
- if (parsedMessage != null) {
- mergeFrom(parsedMessage);
- }
- }
- return this;
- }
- private int bitField0_;
-
- // required string name = 1;
- private java.lang.Object name_ = "";
- /**
- * <code>required string name = 1;</code>
- */
- public boolean hasName() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>required string name = 1;</code>
- */
- public java.lang.String getName() {
- java.lang.Object ref = name_;
- if (!(ref instanceof java.lang.String)) {
- java.lang.String s = ((com.google.protobuf.ByteString) ref)
- .toStringUtf8();
- name_ = s;
- return s;
- } else {
- return (java.lang.String) ref;
- }
- }
- /**
- * <code>required string name = 1;</code>
- */
- public com.google.protobuf.ByteString
- getNameBytes() {
- java.lang.Object ref = name_;
- if (ref instanceof String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- name_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
- /**
- * <code>required string name = 1;</code>
- */
- public Builder setName(
- java.lang.String value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000001;
- name_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required string name = 1;</code>
- */
- public Builder clearName() {
- bitField0_ = (bitField0_ & ~0x00000001);
- name_ = getDefaultInstance().getName();
- onChanged();
- return this;
- }
- /**
- * <code>required string name = 1;</code>
- */
- public Builder setNameBytes(
- com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000001;
- name_ = value;
- onChanged();
- return this;
- }
-
- // optional string role = 2;
- private java.lang.Object role_ = "";
- /**
- * <code>optional string role = 2;</code>
- */
- public boolean hasRole() {
- return ((bitField0_ & 0x00000002) == 0x00000002);
- }
- /**
- * <code>optional string role = 2;</code>
- */
- public java.lang.String getRole() {
- java.lang.Object ref = role_;
- if (!(ref instanceof java.lang.String)) {
- java.lang.String s = ((com.google.protobuf.ByteString) ref)
- .toStringUtf8();
- role_ = s;
- return s;
- } else {
- return (java.lang.String) ref;
- }
- }
- /**
- * <code>optional string role = 2;</code>
- */
- public com.google.protobuf.ByteString
- getRoleBytes() {
- java.lang.Object ref = role_;
- if (ref instanceof String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- role_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
- /**
- * <code>optional string role = 2;</code>
- */
- public Builder setRole(
- java.lang.String value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000002;
- role_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>optional string role = 2;</code>
- */
- public Builder clearRole() {
- bitField0_ = (bitField0_ & ~0x00000002);
- role_ = getDefaultInstance().getRole();
- onChanged();
- return this;
- }
- /**
- * <code>optional string role = 2;</code>
- */
- public Builder setRoleBytes(
- com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000002;
- role_ = value;
- onChanged();
- return this;
- }
-
- // required uint32 state = 4;
- private int state_ ;
- /**
- * <code>required uint32 state = 4;</code>
- */
- public boolean hasState() {
- return ((bitField0_ & 0x00000004) == 0x00000004);
- }
- /**
- * <code>required uint32 state = 4;</code>
- */
- public int getState() {
- return state_;
- }
- /**
- * <code>required uint32 state = 4;</code>
- */
- public Builder setState(int value) {
- bitField0_ |= 0x00000004;
- state_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required uint32 state = 4;</code>
- */
- public Builder clearState() {
- bitField0_ = (bitField0_ & ~0x00000004);
- state_ = 0;
- onChanged();
- return this;
- }
-
- // required uint32 exitCode = 5;
- private int exitCode_ ;
- /**
- * <code>required uint32 exitCode = 5;</code>
- */
- public boolean hasExitCode() {
- return ((bitField0_ & 0x00000008) == 0x00000008);
- }
- /**
- * <code>required uint32 exitCode = 5;</code>
- */
- public int getExitCode() {
- return exitCode_;
- }
- /**
- * <code>required uint32 exitCode = 5;</code>
- */
- public Builder setExitCode(int value) {
- bitField0_ |= 0x00000008;
- exitCode_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required uint32 exitCode = 5;</code>
- */
- public Builder clearExitCode() {
- bitField0_ = (bitField0_ & ~0x00000008);
- exitCode_ = 0;
- onChanged();
- return this;
- }
-
- // optional string command = 6;
- private java.lang.Object command_ = "";
- /**
- * <code>optional string command = 6;</code>
- */
- public boolean hasCommand() {
- return ((bitField0_ & 0x00000010) == 0x00000010);
- }
- /**
- * <code>optional string command = 6;</code>
- */
- public java.lang.String getCommand() {
- java.lang.Object ref = command_;
- if (!(ref instanceof java.lang.String)) {
- java.lang.String s = ((com.google.protobuf.ByteString) ref)
- .toStringUtf8();
- command_ = s;
- return s;
- } else {
- return (java.lang.String) ref;
- }
- }
- /**
- * <code>optional string command = 6;</code>
- */
- public com.google.protobuf.ByteString
- getCommandBytes() {
- java.lang.Object ref = command_;
- if (ref instanceof String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- command_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
- /**
- * <code>optional string command = 6;</code>
- */
- public Builder setCommand(
- java.lang.String value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000010;
- command_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>optional string command = 6;</code>
- */
- public Builder clearCommand() {
- bitField0_ = (bitField0_ & ~0x00000010);
- command_ = getDefaultInstance().getCommand();
- onChanged();
- return this;
- }
- /**
- * <code>optional string command = 6;</code>
- */
- public Builder setCommandBytes(
- com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000010;
- command_ = value;
- onChanged();
- return this;
- }
-
- // optional string diagnostics = 7;
- private java.lang.Object diagnostics_ = "";
- /**
- * <code>optional string diagnostics = 7;</code>
- */
- public boolean hasDiagnostics() {
- return ((bitField0_ & 0x00000020) == 0x00000020);
- }
- /**
- * <code>optional string diagnostics = 7;</code>
- */
- public java.lang.String getDiagnostics() {
- java.lang.Object ref = diagnostics_;
- if (!(ref instanceof java.lang.String)) {
- java.lang.String s = ((com.google.protobuf.ByteString) ref)
- .toStringUtf8();
- diagnostics_ = s;
- return s;
- } else {
- return (java.lang.String) ref;
- }
- }
- /**
- * <code>optional string diagnostics = 7;</code>
- */
- public com.google.protobuf.ByteString
- getDiagnosticsBytes() {
- java.lang.Object ref = diagnostics_;
- if (ref instanceof String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- diagnostics_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
- /**
- * <code>optional string diagnostics = 7;</code>
- */
- public Builder setDiagnostics(
- java.lang.String value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000020;
- diagnostics_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>optional string diagnostics = 7;</code>
- */
- public Builder clearDiagnostics() {
- bitField0_ = (bitField0_ & ~0x00000020);
- diagnostics_ = getDefaultInstance().getDiagnostics();
- onChanged();
- return this;
- }
- /**
- * <code>optional string diagnostics = 7;</code>
- */
- public Builder setDiagnosticsBytes(
- com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00000020;
- diagnostics_ = value;
- onChanged();
- return this;
- }
-
- // repeated string output = 8;
- private com.google.protobuf.LazyStringList output_ = com.google.protobuf.LazyStringArrayList.EMPTY;
- private void ensureOutputIsMutable() {
- if (!((bitField0_ & 0x00000040) == 0x00000040)) {
- output_ = new com.google.protobuf.LazyStringArrayList(output_);
- bitField0_ |= 0x00000040;
- }
- }
- /**
- * <code>repeated string output = 8;</code>
- */
- public java.util.List<java.lang.String>
- getOutputList() {
- return java.util.Collections.unmodifiableList(output_);
- }
- /**
- * <code>repeated string output = 8;</code>
- */
- public int getOutputCount() {
- return output_.size();
- }
- /**
- * <code>repeated string output = 8;</code>
- */
- public java.lang.String getOutput(int index) {
- return output_.get(index);
- }
- /**
- * <code>repeated string output = 8;</code>
- */
- public com.google.protobuf.ByteString
- getOutputBytes(int index) {
- return output_.getByteString(index);
- }
- /**
- * <code>repeated string output = 8;</code>
- */
- public Builder setOutput(
- int index, java.lang.String value) {
- if (value == null) {
- throw new NullPointerException();
- }
- ensureOutputIsMutable();
- output_.set(index, value);
- onChanged();
- return this;
- }
- /**
- * <code>repeated string output = 8;</code>
- */
- public Builder addOutput(
- java.lang.String value) {
- if (value == null) {
- throw new NullPointerException();
- }
- ensureOutputIsMutable();
- output_.add(value);
- onChanged();
- return this;
- }
- /**
- * <code>repeated string output = 8;</code>
- */
- public Builder addAllOutput(
- java.lang.Iterable<java.lang.String> values) {
- ensureOutputIsMutable();
- super.addAll(values, output_);
- onChanged();
- return this;
- }
- /**
- * <code>repeated string output = 8;</code>
- */
- public Builder clearOutput() {
- output_ = com.google.protobuf.LazyStringArrayList.EMPTY;
- bitField0_ = (bitField0_ & ~0x00000040);
- onChanged();
- return this;
- }
- /**
- * <code>repeated string output = 8;</code>
- */
- public Builder addOutputBytes(
- com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- ensureOutputIsMutable();
- output_.add(value);
- onChanged();
- return this;
- }
-
- // repeated string environment = 9;
- private com.google.protobuf.LazyStringList environment_ = com.google.protobuf.LazyStringArrayList.EMPTY;
- private void ensureEnvironmentIsMutable() {
- if (!((bitField0_ & 0x00000080) == 0x00000080)) {
- environment_ = new com.google.protobuf.LazyStringArrayList(environment_);
- bitField0_ |= 0x00000080;
- }
- }
- /**
- * <code>repeated string environment = 9;</code>
- */
- public java.util.List<java.lang.String>
- getEnvironmentList() {
- return java.util.Collections.unmodifiableList(environment_);
- }
- /**
- * <code>repeated string environment = 9;</code>
- */
- public int getEnvironmentCount() {
- return environment_.size();
- }
- /**
- * <code>repeated string environment = 9;</code>
- */
- public java.lang.String getEnvironment(int index) {
- return environment_.get(index);
- }
- /**
- * <code>repeated string environment = 9;</code>
- */
- public com.google.protobuf.ByteString
- getEnvironmentBytes(int index) {
- return environment_.getByteString(index);
- }
- /**
- * <code>repeated string environment = 9;</code>
- */
- public Builder setEnvironment(
- int index, java.lang.String value) {
- if (value == null) {
- throw new NullPointerException();
- }
- ensureEnvironmentIsMutable();
- environment_.set(index, value);
- onChanged();
- return this;
- }
- /**
- * <code>repeated string environment = 9;</code>
- */
- public Builder addEnvironment(
- java.lang.String value) {
- if (value == null) {
- throw new NullPointerException();
- }
- ensureEnvironmentIsMutable();
- environment_.add(value);
- onChanged();
- return this;
- }
- /**
- * <code>repeated string environment = 9;</code>
- */
- public Builder addAllEnvironment(
- java.lang.Iterable<java.lang.String> values) {
- ensureEnvironmentIsMutable();
- super.addAll(values, environment_);
- onChanged();
- return this;
- }
- /**
- * <code>repeated string environment = 9;</code>
- */
- public Builder clearEnvironment() {
- environment_ = com.google.protobuf.LazyStringArrayList.EMPTY;
- bitField0_ = (bitField0_ & ~0x00000080);
- onChanged();
- return this;
- }
- /**
- * <code>repeated string environment = 9;</code>
- */
- public Builder addEnvironmentBytes(
- com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- ensureEnvironmentIsMutable();
- environment_.add(value);
- onChanged();
- return this;
- }
-
- // required uint32 roleId = 10;
- private int roleId_ ;
- /**
- * <code>required uint32 roleId = 10;</code>
- */
- public boolean hasRoleId() {
- return ((bitField0_ & 0x00000100) == 0x00000100);
- }
- /**
- * <code>required uint32 roleId = 10;</code>
- */
- public int getRoleId() {
- return roleId_;
- }
- /**
- * <code>required uint32 roleId = 10;</code>
- */
- public Builder setRoleId(int value) {
- bitField0_ |= 0x00000100;
- roleId_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required uint32 roleId = 10;</code>
- */
- public Builder clearRoleId() {
- bitField0_ = (bitField0_ & ~0x00000100);
- roleId_ = 0;
- onChanged();
- return this;
- }
-
- // required bool released = 11;
- private boolean released_ ;
- /**
- * <code>required bool released = 11;</code>
- */
- public boolean hasReleased() {
- return ((bitField0_ & 0x00000200) == 0x00000200);
- }
- /**
- * <code>required bool released = 11;</code>
- */
- public boolean getReleased() {
- return released_;
- }
- /**
- * <code>required bool released = 11;</code>
- */
- public Builder setReleased(boolean value) {
- bitField0_ |= 0x00000200;
- released_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required bool released = 11;</code>
- */
- public Builder clearReleased() {
- bitField0_ = (bitField0_ & ~0x00000200);
- released_ = false;
- onChanged();
- return this;
- }
-
- // required int64 createTime = 12;
- private long createTime_ ;
- /**
- * <code>required int64 createTime = 12;</code>
- */
- public boolean hasCreateTime() {
- return ((bitField0_ & 0x00000400) == 0x00000400);
- }
- /**
- * <code>required int64 createTime = 12;</code>
- */
- public long getCreateTime() {
- return createTime_;
- }
- /**
- * <code>required int64 createTime = 12;</code>
- */
- public Builder setCreateTime(long value) {
- bitField0_ |= 0x00000400;
- createTime_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required int64 createTime = 12;</code>
- */
- public Builder clearCreateTime() {
- bitField0_ = (bitField0_ & ~0x00000400);
- createTime_ = 0L;
- onChanged();
- return this;
- }
-
- // required int64 startTime = 13;
- private long startTime_ ;
- /**
- * <code>required int64 startTime = 13;</code>
- */
- public boolean hasStartTime() {
- return ((bitField0_ & 0x00000800) == 0x00000800);
- }
- /**
- * <code>required int64 startTime = 13;</code>
- */
- public long getStartTime() {
- return startTime_;
- }
- /**
- * <code>required int64 startTime = 13;</code>
- */
- public Builder setStartTime(long value) {
- bitField0_ |= 0x00000800;
- startTime_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required int64 startTime = 13;</code>
- */
- public Builder clearStartTime() {
- bitField0_ = (bitField0_ & ~0x00000800);
- startTime_ = 0L;
- onChanged();
- return this;
- }
-
- // required string host = 14;
- private java.lang.Object host_ = "";
- /**
- * <code>required string host = 14;</code>
- */
- public boolean hasHost() {
- return ((bitField0_ & 0x00001000) == 0x00001000);
- }
- /**
- * <code>required string host = 14;</code>
- */
- public java.lang.String getHost() {
- java.lang.Object ref = host_;
- if (!(ref instanceof java.lang.String)) {
- java.lang.String s = ((com.google.protobuf.ByteString) ref)
- .toStringUtf8();
- host_ = s;
- return s;
- } else {
- return (java.lang.String) ref;
- }
- }
- /**
- * <code>required string host = 14;</code>
- */
- public com.google.protobuf.ByteString
- getHostBytes() {
- java.lang.Object ref = host_;
- if (ref instanceof String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- host_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
- /**
- * <code>required string host = 14;</code>
- */
- public Builder setHost(
- java.lang.String value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00001000;
- host_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required string host = 14;</code>
- */
- public Builder clearHost() {
- bitField0_ = (bitField0_ & ~0x00001000);
- host_ = getDefaultInstance().getHost();
- onChanged();
- return this;
- }
- /**
- * <code>required string host = 14;</code>
- */
- public Builder setHostBytes(
- com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00001000;
- host_ = value;
- onChanged();
- return this;
- }
-
- // required string hostURL = 15;
- private java.lang.Object hostURL_ = "";
- /**
- * <code>required string hostURL = 15;</code>
- */
- public boolean hasHostURL() {
- return ((bitField0_ & 0x00002000) == 0x00002000);
- }
- /**
- * <code>required string hostURL = 15;</code>
- */
- public java.lang.String getHostURL() {
- java.lang.Object ref = hostURL_;
- if (!(ref instanceof java.lang.String)) {
- java.lang.String s = ((com.google.protobuf.ByteString) ref)
- .toStringUtf8();
- hostURL_ = s;
- return s;
- } else {
- return (java.lang.String) ref;
- }
- }
- /**
- * <code>required string hostURL = 15;</code>
- */
- public com.google.protobuf.ByteString
- getHostURLBytes() {
- java.lang.Object ref = hostURL_;
- if (ref instanceof String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- hostURL_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
- /**
- * <code>required string hostURL = 15;</code>
- */
- public Builder setHostURL(
- java.lang.String value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00002000;
- hostURL_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>required string hostURL = 15;</code>
- */
- public Builder clearHostURL() {
- bitField0_ = (bitField0_ & ~0x00002000);
- hostURL_ = getDefaultInstance().getHostURL();
- onChanged();
- return this;
- }
- /**
- * <code>required string hostURL = 15;</code>
- */
- public Builder setHostURLBytes(
- com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00002000;
- hostURL_ = value;
- onChanged();
- return this;
- }
-
- // optional string appVersion = 16;
- private java.lang.Object appVersion_ = "";
- /**
- * <code>optional string appVersion = 16;</code>
- */
- public boolean hasAppVersion() {
- return ((bitField0_ & 0x00004000) == 0x00004000);
- }
- /**
- * <code>optional string appVersion = 16;</code>
- */
- public java.lang.String getAppVersion() {
- java.lang.Object ref = appVersion_;
- if (!(ref instanceof java.lang.String)) {
- java.lang.String s = ((com.google.protobuf.ByteString) ref)
- .toStringUtf8();
- appVersion_ = s;
- return s;
- } else {
- return (java.lang.String) ref;
- }
- }
- /**
- * <code>optional string appVersion = 16;</code>
- */
- public com.google.protobuf.ByteString
- getAppVersionBytes() {
- java.lang.Object ref = appVersion_;
- if (ref instanceof String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- appVersion_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
- /**
- * <code>optional string appVersion = 16;</code>
- */
- public Builder setAppVersion(
- java.lang.String value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00004000;
- appVersion_ = value;
- onChanged();
- return this;
- }
- /**
- * <code>optional string appVersion = 16;</code>
- */
- public Builder clearAppVersion() {
- bitField0_ = (bitField0_ & ~0x00004000);
- appVersion_ = getDefaultInstance().getAppVersion();
- onChanged();
- return this;
- }
- /**
- * <code>optional string appVersion = 16;</code>
- */
- public Builder setAppVersionBytes(
- com.google.protobuf.ByteString value) {
- if (value == null) {
- throw new NullPointerException();
- }
- bitField0_ |= 0x00004000;
- appVersion_ = value;
- onChanged();
- return this;
- }
-
- // @@protoc_insertion_point(builder_scope:org.apache.slider.api.RoleInstanceState)
- }
-
- static {
- defaultInstance = new RoleInstanceState(true);
- defaultInstance.initFields();
- }
-
- // @@protoc_insertion_point(class_scope:org.apache.slider.api.RoleInstanceState)
- }
-
- public interface StopClusterRequestProtoOrBuilder
- extends com.google.protobuf.MessageOrBuilder {
-
- // required string message = 1;
- /**
- * <code>required string message = 1;</code>
- *
- * <pre>
- **
- *message to include
- * </pre>
- */
- boolean hasMessage();
- /**
- * <code>required string message = 1;</code>
- *
- * <pre>
- **
- *message to include
- * </pre>
- */
- java.lang.String getMessage();
- /**
- * <code>required string message = 1;</code>
- *
- * <pre>
- **
- *message to include
- * </pre>
- */
- com.google.protobuf.ByteString
- getMessageBytes();
- }
- /**
- * Protobuf type {@code org.apache.slider.api.StopClusterRequestProto}
- *
- * <pre>
- **
- * stop the cluster
- * </pre>
- */
- public static final class StopClusterRequestProto extends
- com.google.protobuf.GeneratedMessage
- implements StopClusterRequestProtoOrBuilder {
- // Use StopClusterRequestProto.newBuilder() to construct.
- private StopClusterRequestProto(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
- super(builder);
- this.unknownFields = builder.getUnknownFields();
- }
- private StopClusterRequestProto(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
- private static final StopClusterRequestProto defaultInstance;
- public static StopClusterRequestProto getDefaultInstance() {
- return defaultInstance;
- }
-
- public StopClusterRequestProto getDefaultInstanceForType() {
- return defaultInstance;
- }
-
- private final com.google.protobuf.UnknownFieldSet unknownFields;
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
- private StopClusterRequestProto(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- initFields();
- int mutable_bitField0_ = 0;
- com.google.protobuf.UnknownFieldSet.Builder unknownFields =
- com.google.protobuf.UnknownFieldSet.newBuilder();
- try {
- boolean done = false;
- while (!done) {
- int tag = input.readTag();
- switch (tag) {
- case 0:
- done = true;
- break;
- default: {
- if (!parseUnknownField(input, unknownFields,
- extensionRegistry, tag)) {
- done = true;
- }
- break;
- }
- case 10: {
- bitField0_ |= 0x00000001;
- message_ = input.readBytes();
- break;
- }
- }
- }
- } catch (com.google.protobuf.InvalidProtocolBufferException e) {
- throw e.setUnfinishedMessage(this);
- } catch (java.io.IOException e) {
- throw new com.google.protobuf.InvalidProtocolBufferException(
- e.getMessage()).setUnfinishedMessage(this);
- } finally {
- this.unknownFields = unknownFields.build();
- makeExtensionsImmutable();
- }
- }
- public static final com.google.protobuf.Descriptors.Descriptor
- getDescriptor() {
- return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_StopClusterRequestProto_descriptor;
- }
-
- protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
- internalGetFieldAccessorTable() {
- return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_StopClusterRequestProto_fieldAccessorTable
- .ensureFieldAccessorsInitialized(
- org.apache.slider.api.proto.Messages.StopClusterRequestProto.class, org.apache.slider.api.proto.Messages.StopClusterRequestProto.Builder.class);
- }
-
- public static com.google.protobuf.Parser<StopClusterRequestProto> PARSER =
- new com.google.protobuf.AbstractParser<StopClusterRequestProto>() {
- public StopClusterRequestProto parsePartialFrom(
- com.google.protobuf.CodedInputStream input,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return new StopClusterRequestProto(input, extensionRegistry);
- }
- };
-
- @java.lang.Override
- public com.google.protobuf.Parser<StopClusterRequestProto> getParserForType() {
- return PARSER;
- }
-
- private int bitField0_;
- // required string message = 1;
- public static final int MESSAGE_FIELD_NUMBER = 1;
- private java.lang.Object message_;
- /**
- * <code>required string message = 1;</code>
- *
- * <pre>
- **
- *message to include
- * </pre>
- */
- public boolean hasMessage() {
- return ((bitField0_ & 0x00000001) == 0x00000001);
- }
- /**
- * <code>required string message = 1;</code>
- *
- * <pre>
- **
- *message to include
- * </pre>
- */
- public java.lang.String getMessage() {
- java.lang.Object ref = message_;
- if (ref instanceof java.lang.String) {
- return (java.lang.String) ref;
- } else {
- com.google.protobuf.ByteString bs =
- (com.google.protobuf.ByteString) ref;
- java.lang.String s = bs.toStringUtf8();
- if (bs.isValidUtf8()) {
- message_ = s;
- }
- return s;
- }
- }
- /**
- * <code>required string message = 1;</code>
- *
- * <pre>
- **
- *message to include
- * </pre>
- */
- public com.google.protobuf.ByteString
- getMessageBytes() {
- java.lang.Object ref = message_;
- if (ref instanceof java.lang.String) {
- com.google.protobuf.ByteString b =
- com.google.protobuf.ByteString.copyFromUtf8(
- (java.lang.String) ref);
- message_ = b;
- return b;
- } else {
- return (com.google.protobuf.ByteString) ref;
- }
- }
-
- private void initFields() {
- message_ = "";
- }
- private byte memoizedIsInitialized = -1;
- public final boolean isInitialized() {
- byte isInitialized = memoizedIsInitialized;
- if (isInitialized != -1) return isInitialized == 1;
-
- if (!hasMessage()) {
- memoizedIsInitialized = 0;
- return false;
- }
- memoizedIsInitialized = 1;
- return true;
- }
-
- public void writeTo(com.google.protobuf.CodedOutputStream output)
- throws java.io.IOException {
- getSerializedSize();
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- output.writeBytes(1, getMessageBytes());
- }
- getUnknownFields().writeTo(output);
- }
-
- private int memoizedSerializedSize = -1;
- public int getSerializedSize() {
- int size = memoizedSerializedSize;
- if (size != -1) return size;
-
- size = 0;
- if (((bitField0_ & 0x00000001) == 0x00000001)) {
- size += com.google.protobuf.CodedOutputStream
- .computeBytesSize(1, getMessageBytes());
- }
- size += getUnknownFields().getSerializedSize();
- memoizedSerializedSize = size;
- return size;
- }
-
- private static final long serialVersionUID = 0L;
- @java.lang.Override
- protected java.lang.Object writeReplace()
- throws java.io.ObjectStreamException {
- return super.writeReplace();
- }
-
- @java.lang.Override
- public boolean equals(final java.lang.Object obj) {
- if (obj == this) {
- return true;
- }
- if (!(obj instanceof org.apache.slider.api.proto.Messages.StopClusterRequestProto)) {
- return super.equals(obj);
- }
- org.apache.slider.api.proto.Messages.StopClusterRequestProto other = (org.apache.slider.api.proto.Messages.StopClusterRequestProto) obj;
-
- boolean result = true;
- result = result && (hasMessage() == other.hasMessage());
- if (hasMessage()) {
- result = result && getMessage()
- .equals(other.getMessage());
- }
- result = result &&
- getUnknownFields().equals(other.getUnknownFields());
- return result;
- }
-
- private int memoizedHashCode = 0;
- @java.lang.Override
- public int hashCode() {
- if (memoizedHashCode != 0) {
- return memoizedHashCode;
- }
- int hash = 41;
- hash = (19 * hash) + getDescriptorForType().hashCode();
- if (hasMessage()) {
- hash = (37 * hash) + MESSAGE_FIELD_NUMBER;
- hash = (53 * hash) + getMessage().hashCode();
- }
- hash = (29 * hash) + getUnknownFields().hashCode();
- memoizedHashCode = hash;
- return hash;
- }
-
- public static org.apache.slider.api.proto.Messages.StopClusterRequestProto parseFrom(
- com.google.protobuf.ByteString data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.slider.api.proto.Messages.StopClusterRequestProto parseFrom(
- com.google.protobuf.ByteString data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.slider.api.proto.Messages.StopClusterRequestProto parseFrom(byte[] data)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data);
- }
- public static org.apache.slider.api.proto.Messages.StopClusterRequestProto parseFrom(
- byte[] data,
- com.google.protobuf.ExtensionRegistryLite extensionRegistry)
- throws com.google.protobuf.InvalidProtocolBufferException {
- return PARSER.parseFrom(data, extensionRegistry);
- }
- public static org.apache.slider.api.proto.Messages.StopClusterRequestProto parseFrom(java.io.InputSt
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/51] [abbrv] hadoop git commit: YARN-5909. Remove agent related
code in slider AM. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/340967d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
deleted file mode 100644
index 499812e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ /dev/null
@@ -1,2850 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.types.Endpoint;
-import org.apache.hadoop.registry.client.types.ProtocolTypes;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.slider.api.ClusterDescription;
-import org.apache.slider.api.ClusterNode;
-import org.apache.slider.api.InternalKeys;
-import org.apache.slider.api.OptionKeys;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.StatusKeys;
-import org.apache.slider.common.SliderExitCodes;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.SliderXmlConfKeys;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.conf.MapOperations;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.NoSuchNodeException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.CommandLineBuilder;
-import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.slider.core.registry.docstore.ConfigFormat;
-import org.apache.slider.core.registry.docstore.ConfigUtils;
-import org.apache.slider.core.registry.docstore.ExportEntry;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.core.registry.docstore.PublishedExports;
-import org.apache.slider.core.registry.info.CustomRegistryConstants;
-import org.apache.slider.providers.AbstractProviderService;
-import org.apache.slider.providers.MonitorDetail;
-import org.apache.slider.providers.ProviderCore;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.providers.ProviderUtils;
-import org.apache.slider.providers.agent.application.metadata.AbstractComponent;
-import org.apache.slider.providers.agent.application.metadata.Application;
-import org.apache.slider.providers.agent.application.metadata.CommandOrder;
-import org.apache.slider.providers.agent.application.metadata.CommandScript;
-import org.apache.slider.providers.agent.application.metadata.Component;
-import org.apache.slider.providers.agent.application.metadata.ComponentCommand;
-import org.apache.slider.providers.agent.application.metadata.ComponentExport;
-import org.apache.slider.providers.agent.application.metadata.ComponentsInAddonPackage;
-import org.apache.slider.providers.agent.application.metadata.ConfigFile;
-import org.apache.slider.providers.agent.application.metadata.DefaultConfig;
-import org.apache.slider.providers.agent.application.metadata.DockerContainer;
-import org.apache.slider.providers.agent.application.metadata.Export;
-import org.apache.slider.providers.agent.application.metadata.ExportGroup;
-import org.apache.slider.providers.agent.application.metadata.Metainfo;
-import org.apache.slider.providers.agent.application.metadata.OSPackage;
-import org.apache.slider.providers.agent.application.metadata.OSSpecific;
-import org.apache.slider.providers.agent.application.metadata.Package;
-import org.apache.slider.providers.agent.application.metadata.PropertyInfo;
-import org.apache.slider.server.appmaster.actions.ProviderReportedContainerLoss;
-import org.apache.slider.server.appmaster.actions.RegisterComponentInstance;
-import org.apache.slider.server.appmaster.state.ContainerPriority;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
-import org.apache.slider.server.appmaster.web.rest.agent.CommandReport;
-import org.apache.slider.server.appmaster.web.rest.agent.ComponentStatus;
-import org.apache.slider.server.appmaster.web.rest.agent.ExecutionCommand;
-import org.apache.slider.server.appmaster.web.rest.agent.HeartBeat;
-import org.apache.slider.server.appmaster.web.rest.agent.HeartBeatResponse;
-import org.apache.slider.server.appmaster.web.rest.agent.Register;
-import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse;
-import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus;
-import org.apache.slider.server.appmaster.web.rest.agent.StatusCommand;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Scanner;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.regex.Pattern;
-
-import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_AGENTS;
-
-/**
- * This class implements the server-side logic for application deployment through Slider application package
- */
-public class AgentProviderService extends AbstractProviderService implements
- ProviderCore,
- AgentKeys,
- SliderKeys, AgentRestOperations {
-
-
- protected static final Logger log =
- LoggerFactory.getLogger(AgentProviderService.class);
- private static final ProviderUtils providerUtils = new ProviderUtils(log);
- private static final String LABEL_MAKER = "___";
- private static final String CONTAINER_ID = "container_id";
- private static final String GLOBAL_CONFIG_TAG = "global";
- private static final String COMPONENT_TAG = "component";
- private static final String APPLICATION_TAG = "application";
- private static final String COMPONENT_DATA_TAG = "ComponentInstanceData";
- private static final String SHARED_PORT_TAG = "SHARED";
- private static final String PER_CONTAINER_TAG = "{PER_CONTAINER}";
- private static final int MAX_LOG_ENTRIES = 40;
- private static final int DEFAULT_HEARTBEAT_MONITOR_INTERVAL = 60 * 1000;
-
- private final Object syncLock = new Object();
- private final ComponentTagProvider tags = new ComponentTagProvider();
- private int heartbeatMonitorInterval = 0;
- private AgentClientProvider clientProvider;
- private AtomicInteger taskId = new AtomicInteger(0);
- private volatile Map<String, MetainfoHolder> metaInfoMap = new HashMap<>();
- private SliderFileSystem fileSystem = null;
- private Map<String, DefaultConfig> defaultConfigs = null;
- private ComponentCommandOrder commandOrder = new ComponentCommandOrder();
- private HeartbeatMonitor monitor;
- private Boolean canAnyMasterPublish = null;
- private AgentLaunchParameter agentLaunchParameter = null;
- private String clusterName = null;
- private boolean isInUpgradeMode;
- private Set<String> upgradeContainers = new HashSet<String>();
- private boolean appStopInitiated;
-
- private final Map<String, ComponentInstanceState> componentStatuses =
- new ConcurrentHashMap<String, ComponentInstanceState>();
- private final Map<String, Map<String, String>> componentInstanceData =
- new ConcurrentHashMap<String, Map<String, String>>();
- private final Map<String, Map<String, List<ExportEntry>>> exportGroups =
- new ConcurrentHashMap<String, Map<String, List<ExportEntry>>>();
- private final Map<String, Map<String, String>> allocatedPorts =
- new ConcurrentHashMap<String, Map<String, String>>();
- private final Map<String, Metainfo> packageMetainfo =
- new ConcurrentHashMap<String, Metainfo>();
-
- private final Map<String, ExportEntry> logFolderExports =
- Collections.synchronizedMap(new LinkedHashMap<String, ExportEntry>(MAX_LOG_ENTRIES, 0.75f, false) {
- protected boolean removeEldestEntry(Map.Entry eldest) {
- return size() > MAX_LOG_ENTRIES;
- }
- });
- private final Map<String, ExportEntry> workFolderExports =
- Collections.synchronizedMap(new LinkedHashMap<String, ExportEntry>(MAX_LOG_ENTRIES, 0.75f, false) {
- protected boolean removeEldestEntry(Map.Entry eldest) {
- return size() > MAX_LOG_ENTRIES;
- }
- });
- private final Map<String, Set<String>> containerExportsMap =
- new HashMap<String, Set<String>>();
-
- private static class MetainfoHolder {
- Metainfo metaInfo;
- private Map<String, DefaultConfig> defaultConfigs = null;
-
- public MetainfoHolder(Metainfo metaInfo,
- Map<String, DefaultConfig> defaultConfigs) {
- this.metaInfo = metaInfo;
- this.defaultConfigs = defaultConfigs;
- }
- }
-
- /**
- * Create an instance of AgentProviderService
- */
- public AgentProviderService() {
- super("AgentProviderService");
- setAgentRestOperations(this);
- setHeartbeatMonitorInterval(DEFAULT_HEARTBEAT_MONITOR_INTERVAL);
- }
-
- @Override
- public String getHumanName() {
- return "Slider Agent";
- }
-
- @Override
- public List<ProviderRole> getRoles() {
- return AgentRoles.getRoles();
- }
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- super.serviceInit(conf);
- clientProvider = new AgentClientProvider(conf);
- }
-
- @Override
- public void validateInstanceDefinition(AggregateConf instanceDefinition)
- throws
- SliderException {
- clientProvider.validateInstanceDefinition(instanceDefinition, null);
-
- ConfTreeOperations resources =
- instanceDefinition.getResourceOperations();
-
- Set<String> names = resources.getComponentNames();
- names.remove(COMPONENT_AM);
- for (String name : names) {
- Component componentDef = getApplicationComponent(name);
- if (componentDef == null) {
- // component member is validated elsewhere, so we don't need to throw
- // an exception here
- continue;
- }
-
- MapOperations componentConfig = resources.getMandatoryComponent(name);
- int count =
- componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES);
- int definedMinCount = componentDef.getMinInstanceCountInt();
- int definedMaxCount = componentDef.getMaxInstanceCountInt();
- if (count < definedMinCount || count > definedMaxCount) {
- throw new BadConfigException("Component %s, %s value %d out of range. "
- + "Expected minimum is %d and maximum is %d",
- name,
- ResourceKeys.COMPONENT_INSTANCES,
- count,
- definedMinCount,
- definedMaxCount);
- }
- }
- }
-
- // Reads the metainfo.xml in the application package and loads it
- private void buildMetainfo(AggregateConf instanceDefinition,
- SliderFileSystem fileSystem,
- String roleGroup)
- throws IOException, SliderException {
- String mapKey = instanceDefinition.getAppConfOperations()
- .getComponentOpt(roleGroup, ROLE_PREFIX, DEFAULT_METAINFO_MAP_KEY);
- String appDef = SliderUtils.getApplicationDefinitionPath(
- instanceDefinition.getAppConfOperations(), roleGroup);
- MapOperations component = null;
- if (roleGroup != null) {
- component = instanceDefinition.getAppConfOperations().getComponent(roleGroup);
- }
-
- MetainfoHolder metaInfoHolder = metaInfoMap.get(mapKey);
- if (metaInfoHolder == null) {
- synchronized (syncLock) {
- if (this.fileSystem == null) {
- this.fileSystem = fileSystem;
- }
- metaInfoHolder = metaInfoMap.get(mapKey);
- if (metaInfoHolder == null) {
- readAndSetHeartbeatMonitoringInterval(instanceDefinition);
- initializeAgentDebugCommands(instanceDefinition);
-
- Metainfo metaInfo = getApplicationMetainfo(fileSystem, appDef, false);
- log.info("Master package metainfo: {}", metaInfo.toString());
- if (metaInfo == null || metaInfo.getApplication() == null) {
- log.error("metainfo.xml is unavailable or malformed at {}.", appDef);
- throw new SliderException(
- "metainfo.xml is required in app package.");
- }
- List<CommandOrder> commandOrders = metaInfo.getApplication()
- .getCommandOrders();
- if (!DEFAULT_METAINFO_MAP_KEY.equals(mapKey)) {
- for (Component comp : metaInfo.getApplication().getComponents()) {
- comp.setName(mapKey + comp.getName());
- log.info("Modifying external metainfo component name to {}",
- comp.getName());
- }
- for (CommandOrder co : commandOrders) {
- log.info("Adding prefix {} to command order {}",
- mapKey, co);
- co.setCommand(mapKey + co.getCommand());
- co.setRequires(mapKey + co.getRequires());
- }
- }
- log.debug("Merging command orders {} for {}", commandOrders,
- roleGroup);
- commandOrder.mergeCommandOrders(commandOrders,
- instanceDefinition.getResourceOperations());
- Map<String, DefaultConfig> defaultConfigs =
- initializeDefaultConfigs(fileSystem, appDef, metaInfo);
- metaInfoMap.put(mapKey, new MetainfoHolder(metaInfo, defaultConfigs));
- monitor = new HeartbeatMonitor(this, getHeartbeatMonitorInterval());
- monitor.start();
-
- // build a map from component to metainfo
- String addonAppDefString = instanceDefinition.getAppConfOperations()
- .getGlobalOptions().getOption(ADDONS, null);
- if (component != null) {
- addonAppDefString = component.getOption(ADDONS, addonAppDefString);
- }
- log.debug("All addon appdefs: {}", addonAppDefString);
- if (addonAppDefString != null) {
- Scanner scanner = new Scanner(addonAppDefString).useDelimiter(",");
- while (scanner.hasNext()) {
- String addonAppDef = scanner.next();
- String addonAppDefPath = instanceDefinition
- .getAppConfOperations().getGlobalOptions().get(addonAppDef);
- if (component != null) {
- addonAppDefPath = component.getOption(addonAppDef, addonAppDefPath);
- }
- log.debug("Addon package {} is stored at: {}", addonAppDef
- + addonAppDefPath);
- Metainfo addonMetaInfo = getApplicationMetainfo(fileSystem,
- addonAppDefPath, true);
- addonMetaInfo.validate();
- packageMetainfo.put(addonMetaInfo.getApplicationPackage()
- .getName(), addonMetaInfo);
- }
- log.info("Metainfo map for master and addon: {}",
- packageMetainfo.toString());
- }
- }
- }
- }
- }
-
- @Override
- public void initializeApplicationConfiguration(
- AggregateConf instanceDefinition, SliderFileSystem fileSystem,
- String roleGroup)
- throws IOException, SliderException {
- buildMetainfo(instanceDefinition, fileSystem, roleGroup);
- }
-
- @Override
- public void buildContainerLaunchContext(ContainerLauncher launcher,
- AggregateConf instanceDefinition,
- Container container,
- ProviderRole providerRole,
- SliderFileSystem fileSystem,
- Path generatedConfPath,
- MapOperations resourceComponent,
- MapOperations appComponent,
- Path containerTmpDirPath) throws
- IOException,
- SliderException {
-
- String roleName = providerRole.name;
- String roleGroup = providerRole.group;
- String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition
- .getAppConfOperations(), roleGroup);
-
- initializeApplicationConfiguration(instanceDefinition, fileSystem, roleGroup);
-
- log.info("Build launch context for Agent");
- log.debug(instanceDefinition.toString());
-
- //if we are launching docker based app on yarn, then we need to pass docker image
- if (isYarnDockerContainer(roleGroup)) {
- launcher.setYarnDockerMode(true);
- launcher.setDockerImage(getConfigFromMetaInfo(roleGroup, "image"));
- launcher.setDockerNetwork(getConfigFromMetaInfo(roleGroup, "network"));
- launcher.setRunPrivilegedContainer(getConfigFromMetaInfo(roleGroup, "runPriviledgedContainer"));
- launcher
- .setYarnContainerMountPoints(getConfigFromMetaInfoWithAppConfigOverriding(
- roleGroup, "yarn.container.mount.points"));
- }
-
- // Set the environment
- launcher.putEnv(SliderUtils.buildEnvMap(appComponent,
- providerUtils.getStandardTokenMap(getAmState().getAppConfSnapshot(),
- getAmState().getInternalsSnapshot(), roleName, roleGroup,
- getClusterName())));
-
- String workDir = ApplicationConstants.Environment.PWD.$();
- launcher.setEnv("AGENT_WORK_ROOT", workDir);
- log.info("AGENT_WORK_ROOT set to {}", workDir);
- String logDir = ApplicationConstants.LOG_DIR_EXPANSION_VAR;
- launcher.setEnv("AGENT_LOG_ROOT", logDir);
- log.info("AGENT_LOG_ROOT set to {}", logDir);
- if (System.getenv(HADOOP_USER_NAME) != null) {
- launcher.setEnv(HADOOP_USER_NAME, System.getenv(HADOOP_USER_NAME));
- }
- // for 2-Way SSL
- launcher.setEnv(SLIDER_PASSPHRASE, instanceDefinition.getPassphrase());
- //add english env
- launcher.setEnv("LANG", "en_US.UTF-8");
- launcher.setEnv("LC_ALL", "en_US.UTF-8");
- launcher.setEnv("LANGUAGE", "en_US.UTF-8");
-
- //local resources
-
- // TODO: Should agent need to support App Home
- String scriptPath = new File(AGENT_MAIN_SCRIPT_ROOT, AGENT_MAIN_SCRIPT).getPath();
- String appHome = instanceDefinition.getAppConfOperations().
- getGlobalOptions().get(PACKAGE_PATH);
- if (SliderUtils.isSet(appHome)) {
- scriptPath = new File(appHome, AGENT_MAIN_SCRIPT).getPath();
- }
-
- // set PYTHONPATH
- List<String> pythonPaths = new ArrayList<String>();
- pythonPaths.add(AGENT_MAIN_SCRIPT_ROOT);
- pythonPaths.add(AGENT_JINJA2_ROOT);
- String pythonPath = StringUtils.join(File.pathSeparator, pythonPaths);
- launcher.setEnv(PYTHONPATH, pythonPath);
- log.info("PYTHONPATH set to {}", pythonPath);
-
- Path agentImagePath = null;
- String agentImage = instanceDefinition.getInternalOperations().
- get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
- if (SliderUtils.isUnset(agentImage)) {
- agentImagePath =
- new Path(new Path(new Path(instanceDefinition.getInternalOperations().get(InternalKeys.INTERNAL_TMP_DIR),
- container.getId().getApplicationAttemptId().getApplicationId().toString()),
- PROVIDER_AGENT),
- AGENT_TAR);
- } else {
- agentImagePath = new Path(agentImage);
- }
-
- if (fileSystem.getFileSystem().exists(agentImagePath)) {
- LocalResource agentImageRes = fileSystem.createAmResource(agentImagePath, LocalResourceType.ARCHIVE);
- launcher.addLocalResource(AGENT_INSTALL_DIR, agentImageRes);
- } else {
- String msg =
- String.format("Required agent image slider-agent.tar.gz is unavailable at %s", agentImagePath.toString());
- MapOperations compOps = appComponent;
- boolean relaxVerificationForTest = compOps != null ? Boolean.valueOf(compOps.
- getOptionBool(TEST_RELAX_VERIFICATION, false)) : false;
- log.error(msg);
-
- if (!relaxVerificationForTest) {
- throw new SliderException(SliderExitCodes.EXIT_DEPLOYMENT_FAILED, msg);
- }
- }
-
- log.info("Using {} for agent.", scriptPath);
- LocalResource appDefRes = fileSystem.createAmResource(
- fileSystem.getFileSystem().resolvePath(new Path(appDef)),
- LocalResourceType.ARCHIVE);
- launcher.addLocalResource(APP_DEFINITION_DIR, appDefRes);
-
- for (Package pkg : getMetaInfo(roleGroup).getApplication().getPackages()) {
- Path pkgPath = fileSystem.buildResourcePath(pkg.getName());
- if (!fileSystem.isFile(pkgPath)) {
- pkgPath = fileSystem.buildResourcePath(getClusterName(),
- pkg.getName());
- }
- if (!fileSystem.isFile(pkgPath)) {
- throw new IOException("Package doesn't exist as a resource: " +
- pkg.getName());
- }
- log.info("Adding resource {}", pkg.getName());
- LocalResourceType type = LocalResourceType.FILE;
- if ("archive".equals(pkg.getType())) {
- type = LocalResourceType.ARCHIVE;
- }
- LocalResource packageResource = fileSystem.createAmResource(
- pkgPath, type);
- launcher.addLocalResource(APP_PACKAGES_DIR, packageResource);
- }
-
- String agentConf = instanceDefinition.getAppConfOperations().
- getGlobalOptions().getOption(AGENT_CONF, "");
- if (SliderUtils.isSet(agentConf)) {
- LocalResource agentConfRes = fileSystem.createAmResource(fileSystem
- .getFileSystem().resolvePath(new Path(agentConf)),
- LocalResourceType.FILE);
- launcher.addLocalResource(AGENT_CONFIG_FILE, agentConfRes);
- }
-
- String agentVer = instanceDefinition.getAppConfOperations().
- getGlobalOptions().getOption(AGENT_VERSION, null);
- if (agentVer != null) {
- LocalResource agentVerRes = fileSystem.createAmResource(
- fileSystem.getFileSystem().resolvePath(new Path(agentVer)),
- LocalResourceType.FILE);
- launcher.addLocalResource(AGENT_VERSION_FILE, agentVerRes);
- }
-
- if (SliderUtils.isHadoopClusterSecure(getConfig())) {
- providerUtils.localizeServiceKeytabs(launcher, instanceDefinition,
- fileSystem, getClusterName());
- }
-
- MapOperations amComponent = instanceDefinition.
- getAppConfOperations().getComponent(COMPONENT_AM);
- if (providerUtils.hasTwoWaySSLEnabled(amComponent)) {
- providerUtils.localizeContainerSSLResources(launcher, container,
- fileSystem, getClusterName());
- }
-
- if (providerUtils.areStoresRequested(appComponent)) {
- providerUtils.localizeContainerSecurityStores(launcher, container,
- roleName, fileSystem, instanceDefinition, appComponent,
- getClusterName());
- }
-
- //add the configuration resources
- launcher.addLocalResources(fileSystem.submitDirectory(
- generatedConfPath,
- PROPAGATED_CONF_DIR_NAME));
-
- if (appComponent.getOptionBool(AM_CONFIG_GENERATION, false)) {
- // build and localize configuration files
- Map<String, Map<String, String>> configurations =
- buildCommandConfigurations(instanceDefinition.getAppConfOperations(),
- instanceDefinition.getInternalOperations(),
- container.getId().toString(), roleName, roleGroup);
- for (ConfigFile configFile : getMetaInfo(roleGroup)
- .getComponentConfigFiles(roleGroup)) {
- localizeConfigFile(launcher, roleName, roleGroup, configFile,
- configurations, launcher.getEnv(), fileSystem);
- }
- }
-
- String label = getContainerLabel(container, roleName, roleGroup);
- CommandLineBuilder operation = new CommandLineBuilder();
-
- String pythonExec = instanceDefinition.getAppConfOperations()
- .getGlobalOptions().getOption(SliderXmlConfKeys.PYTHON_EXECUTABLE_PATH,
- PYTHON_EXE);
-
- operation.add(pythonExec);
-
- operation.add(scriptPath);
- operation.add(ARG_LABEL, label);
- operation.add(ARG_ZOOKEEPER_QUORUM);
- operation.add(getClusterOptionPropertyValue(OptionKeys.ZOOKEEPER_QUORUM));
- operation.add(ARG_ZOOKEEPER_REGISTRY_PATH);
- operation.add(getZkRegistryPath());
-
- String debugCmd = agentLaunchParameter.getNextLaunchParameter(roleGroup);
- if (SliderUtils.isSet(debugCmd)) {
- operation.add(ARG_DEBUG);
- operation.add(debugCmd);
- }
-
- operation.add("> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/"
- + AGENT_OUT_FILE + " 2>&1");
-
- launcher.addCommand(operation.build());
-
- // localize addon package
- String addonAppDefString = instanceDefinition.getAppConfOperations()
- .getGlobalOptions().getOption(ADDONS, null);
- log.debug("All addon appdefs: {}", addonAppDefString);
- if (addonAppDefString != null) {
- Scanner scanner = new Scanner(addonAppDefString).useDelimiter(",");
- while (scanner.hasNext()) {
- String addonAppDef = scanner.next();
- String addonAppDefPath = instanceDefinition
- .getAppConfOperations().getGlobalOptions().get(addonAppDef);
- log.debug("Addon package {} is stored at: {}", addonAppDef, addonAppDefPath);
- LocalResource addonPkgRes = fileSystem.createAmResource(
- fileSystem.getFileSystem().resolvePath(new Path(addonAppDefPath)),
- LocalResourceType.ARCHIVE);
- launcher.addLocalResource(ADDON_DEFINITION_DIR + "/" + addonAppDef, addonPkgRes);
- }
- log.debug("Metainfo map for master and addon: {}",
- packageMetainfo.toString());
- }
-
- // Additional files to localize in addition to the application def
- String appResourcesString = instanceDefinition.getAppConfOperations()
- .getGlobalOptions().getOption(APP_RESOURCES, null);
- log.info("Configuration value for extra resources to localize: {}", appResourcesString);
- if (null != appResourcesString) {
- try (Scanner scanner = new Scanner(appResourcesString).useDelimiter(",")) {
- while (scanner.hasNext()) {
- String resource = scanner.next();
- Path resourcePath = new Path(resource);
- LocalResource extraResource = fileSystem.createAmResource(
- fileSystem.getFileSystem().resolvePath(resourcePath),
- LocalResourceType.FILE);
- String destination = APP_RESOURCES_DIR + "/" + resourcePath.getName();
- log.info("Localizing {} to {}", resourcePath, destination);
- // TODO Can we try harder to avoid collisions?
- launcher.addLocalResource(destination, extraResource);
- }
- }
- }
-
- // initialize addon pkg states for all componentInstanceStatus
- Map<String, State> pkgStatuses = new TreeMap<>();
- for (Metainfo appPkg : packageMetainfo.values()) {
- // check each component of that addon to see if they apply to this
- // component 'role'
- for (ComponentsInAddonPackage comp : appPkg.getApplicationPackage()
- .getComponents()) {
- log.debug("Current component: {} component in metainfo: {}", roleName,
- comp.getName());
- if (comp.getName().equals(roleGroup)
- || comp.getName().equals(ADDON_FOR_ALL_COMPONENTS)) {
- pkgStatuses.put(appPkg.getApplicationPackage().getName(), State.INIT);
- }
- }
- }
- log.debug("For component: {} pkg status map: {}", roleName,
- pkgStatuses.toString());
-
- // initialize the component instance state
- getComponentStatuses().put(label,
- new ComponentInstanceState(
- roleGroup,
- container.getId(),
- getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME),
- pkgStatuses));
- }
-
- @VisibleForTesting
- protected void localizeConfigFile(ContainerLauncher launcher,
- String roleName, String roleGroup,
- ConfigFile configFile,
- Map<String, Map<String, String>> configs,
- MapOperations env,
- SliderFileSystem fileSystem)
- throws IOException {
- ConfigFormat configFormat = ConfigFormat.resolve(configFile.getType());
- providerUtils.localizeConfigFile(launcher, roleName, roleGroup,
- configFile.getDictionaryName(), configFormat, configFile.getFileName(),
- configs, env, fileSystem, getClusterName());
- }
-
- /**
- * build the zookeeper registry path.
- *
- * @return the path the service registered at
- * @throws NullPointerException if the service has not yet registered
- */
- private String getZkRegistryPath() {
- Preconditions.checkNotNull(yarnRegistry, "Yarn registry not bound");
- String path = yarnRegistry.getAbsoluteSelfRegistrationPath();
- Preconditions.checkNotNull(path, "Service record path not defined");
- return path;
- }
-
- @Override
- public void rebuildContainerDetails(List<Container> liveContainers,
- String applicationId, Map<Integer, ProviderRole> providerRoleMap) {
- for (Container container : liveContainers) {
- // get the role name and label
- ProviderRole role = providerRoleMap.get(ContainerPriority
- .extractRole(container));
- if (role != null) {
- String roleName = role.name;
- String roleGroup = role.group;
- String label = getContainerLabel(container, roleName, roleGroup);
- log.info("Rebuilding in-memory: container {} in role {} in cluster {}",
- container.getId(), roleName, applicationId);
- getComponentStatuses().put(label,
- new ComponentInstanceState(roleGroup, container.getId(),
- applicationId));
- } else {
- log.warn("Role not found for container {} in cluster {}",
- container.getId(), applicationId);
- }
- }
- }
-
- @Override
- public boolean isSupportedRole(String role) {
- return true;
- }
-
- /**
- * Handle registration calls from the agents
- *
- * @param registration registration entry
- *
- * @return response
- */
- @Override
- public RegistrationResponse handleRegistration(Register registration) {
- log.info("Handling registration: {}", registration);
- RegistrationResponse response = new RegistrationResponse();
- String label = registration.getLabel();
- String pkg = registration.getPkg();
- State agentState = registration.getActualState();
- String appVersion = registration.getAppVersion();
-
- log.info("label: {} pkg: {}", label, pkg);
-
- if (getComponentStatuses().containsKey(label)) {
- response.setResponseStatus(RegistrationStatus.OK);
- ComponentInstanceState componentStatus = getComponentStatuses().get(label);
- componentStatus.heartbeat(System.currentTimeMillis());
- updateComponentStatusWithAgentState(componentStatus, agentState);
-
- String roleName = getRoleName(label);
- String roleGroup = getRoleGroup(label);
- String containerId = getContainerId(label);
-
- if (SliderUtils.isSet(registration.getTags())) {
- tags.recordAssignedTag(roleName, containerId, registration.getTags());
- } else {
- response.setTags(tags.getTag(roleName, containerId));
- }
-
- String hostFqdn = registration.getPublicHostname();
- Map<String, String> ports = registration.getAllocatedPorts();
- if (ports != null && !ports.isEmpty()) {
- processAllocatedPorts(hostFqdn, roleName, roleGroup, containerId, ports);
- }
-
- Map<String, String> folders = registration.getLogFolders();
- if (folders != null && !folders.isEmpty()) {
- publishFolderPaths(folders, containerId, roleName, hostFqdn);
- }
-
- // Set app version if empty. It gets unset during upgrade - why?
- checkAndSetContainerAppVersion(containerId, appVersion);
- } else {
- response.setResponseStatus(RegistrationStatus.FAILED);
- response.setLog("Label not recognized.");
- log.warn("Received registration request from unknown label {}", label);
- }
- log.info("Registration response: {}", response);
- return response;
- }
-
- // Checks if app version is empty. Sets it to the version as reported by the
- // container during registration phase.
- private void checkAndSetContainerAppVersion(String containerId,
- String appVersion) {
- StateAccessForProviders amState = getAmState();
- try {
- RoleInstance role = amState.getOwnedContainer(containerId);
- if (role != null) {
- String currentAppVersion = role.appVersion;
- log.debug("Container = {}, app version current = {} new = {}",
- containerId, currentAppVersion, appVersion);
- if (currentAppVersion == null
- || currentAppVersion.equals(APP_VERSION_UNKNOWN)) {
- amState.getOwnedContainer(containerId).appVersion = appVersion;
- }
- }
- } catch (NoSuchNodeException e) {
- // ignore - there is nothing to do if we don't find a container
- log.warn("Owned container {} not found - {}", containerId, e);
- }
- }
-
- /**
- * Handle heartbeat response from agents
- *
- * @param heartBeat incoming heartbeat from Agent
- *
- * @return response to send back
- */
- @Override
- public HeartBeatResponse handleHeartBeat(HeartBeat heartBeat) {
- log.debug("Handling heartbeat: {}", heartBeat);
- HeartBeatResponse response = new HeartBeatResponse();
- long id = heartBeat.getResponseId();
- response.setResponseId(id + 1L);
-
- String label = heartBeat.getHostname();
- String pkg = heartBeat.getPackage();
-
- log.debug("package received: " + pkg);
-
- String roleName = getRoleName(label);
- String roleGroup = getRoleGroup(label);
- String containerId = getContainerId(label);
- boolean doUpgrade = false;
- if (isInUpgradeMode && upgradeContainers.contains(containerId)) {
- doUpgrade = true;
- }
-
- CommandScript cmdScript = getScriptPathForMasterPackage(roleGroup);
- List<ComponentCommand> commands = getApplicationComponent(roleGroup).getCommands();
-
- if (!isDockerContainer(roleGroup) && !isYarnDockerContainer(roleGroup)
- && (cmdScript == null || cmdScript.getScript() == null)
- && commands.size() == 0) {
- log.error(
- "role.script is unavailable for {}. Commands will not be sent.",
- roleName);
- return response;
- }
-
- String scriptPath = null;
- long timeout = 600L;
- if (cmdScript != null) {
- scriptPath = cmdScript.getScript();
- timeout = cmdScript.getTimeout();
- }
-
- if (timeout == 0L) {
- timeout = 600L;
- }
-
- if (!getComponentStatuses().containsKey(label)) {
- // container is completed but still heart-beating, send terminate signal
- log.info(
- "Sending terminate signal to completed container (still heartbeating): {}",
- label);
- response.setTerminateAgent(true);
- return response;
- }
-
- List<ComponentStatus> statuses = heartBeat.getComponentStatus();
- if (statuses != null && !statuses.isEmpty()) {
- log.info("status from agent: " + statuses.toString());
- for(ComponentStatus status : statuses){
- providerUtils.updateServiceRecord(getAmState(), yarnRegistry,
- containerId, roleName, Collections.singletonList(status.getIp()),
- status.getHostname());
- }
- }
-
- Boolean isMaster = isMaster(roleGroup);
- ComponentInstanceState componentStatus = getComponentStatuses().get(label);
- componentStatus.heartbeat(System.currentTimeMillis());
- if (doUpgrade) {
- switch (componentStatus.getState()) {
- case STARTED:
- componentStatus.setTargetState(State.UPGRADED);
- break;
- case UPGRADED:
- componentStatus.setTargetState(State.STOPPED);
- break;
- case STOPPED:
- componentStatus.setTargetState(State.TERMINATING);
- break;
- default:
- break;
- }
- log.info("Current state = {} target state {}",
- componentStatus.getState(), componentStatus.getTargetState());
- }
-
- if (appStopInitiated && !componentStatus.isStopInitiated()) {
- log.info("Stop initiated for label {}", label);
- componentStatus.setTargetState(State.STOPPED);
- componentStatus.setStopInitiated(true);
- }
-
- publishConfigAndExportGroups(heartBeat, componentStatus, roleGroup);
- CommandResult result = null;
- List<CommandReport> reports = heartBeat.getReports();
- if (SliderUtils.isNotEmpty(reports)) {
- CommandReport report = reports.get(0);
- Map<String, String> ports = report.getAllocatedPorts();
- if (SliderUtils.isNotEmpty(ports)) {
- processAllocatedPorts(heartBeat.getFqdn(), roleName, roleGroup, containerId, ports);
- }
- result = CommandResult.getCommandResult(report.getStatus());
- Command command = Command.getCommand(report.getRoleCommand());
- componentStatus.applyCommandResult(result, command, pkg);
- log.info("Component operation. Status: {}; new container state: {};"
- + " new component state: {}", result,
- componentStatus.getContainerState(), componentStatus.getState());
-
- if (command == Command.INSTALL && SliderUtils.isNotEmpty(report.getFolders())) {
- publishFolderPaths(report.getFolders(), containerId, roleName, heartBeat.getFqdn());
- }
- }
-
- int waitForCount = getAmState().getInstanceDefinitionSnapshot().
- getAppConfOperations().getComponentOptInt(roleGroup, WAIT_HEARTBEAT, 0);
-
- if (id < waitForCount) {
- log.info("Waiting until heartbeat count {}. Current val: {}", waitForCount, id);
- getComponentStatuses().put(label, componentStatus);
- return response;
- }
-
- Command command = componentStatus.getNextCommand(doUpgrade);
- try {
- if (Command.NOP != command) {
- log.debug("For comp {} pkg {} issuing {}", roleName,
- componentStatus.getNextPkgToInstall(), command.toString());
- if (command == Command.INSTALL) {
- log.info("Installing {} on {}.", roleName, containerId);
- if (isDockerContainer(roleGroup) || isYarnDockerContainer(roleGroup)){
- addInstallDockerCommand(roleName, roleGroup, containerId,
- response, null, timeout);
- } else if (scriptPath != null) {
- addInstallCommand(roleName, roleGroup, containerId, response,
- scriptPath, null, timeout, null);
- } else {
- // commands
- ComponentCommand installCmd = null;
- for (ComponentCommand compCmd : commands) {
- if (compCmd.getName().equals("INSTALL")) {
- installCmd = compCmd;
- }
- }
- addInstallCommand(roleName, roleGroup, containerId, response, null,
- installCmd, timeout, null);
- }
- componentStatus.commandIssued(command);
- } else if (command == Command.INSTALL_ADDON) {
- String nextPkgToInstall = componentStatus.getNextPkgToInstall();
- // retrieve scriptPath or command of that package for the component
- for (ComponentsInAddonPackage comp : packageMetainfo
- .get(nextPkgToInstall).getApplicationPackage().getComponents()) {
- // given nextPkgToInstall and roleName is determined, the if below
- // should only execute once per heartbeat
- log.debug("Addon component: {} pkg: {} script: {}", comp.getName(),
- nextPkgToInstall, comp.getCommandScript().getScript());
- if (comp.getName().equals(roleGroup)
- || comp.getName().equals(ADDON_FOR_ALL_COMPONENTS)) {
- scriptPath = comp.getCommandScript().getScript();
- if (scriptPath != null) {
- addInstallCommand(roleName, roleGroup, containerId, response,
- scriptPath, null, timeout, nextPkgToInstall);
- } else {
- ComponentCommand installCmd = null;
- for (ComponentCommand compCmd : comp.getCommands()) {
- if (compCmd.getName().equals("INSTALL")) {
- installCmd = compCmd;
- }
- }
- addInstallCommand(roleName, roleGroup, containerId, response,
- null, installCmd, timeout, nextPkgToInstall);
- }
- }
- }
- componentStatus.commandIssued(command);
- } else if (command == Command.START) {
- // check against dependencies
- boolean canExecute = commandOrder.canExecute(roleGroup, command, getComponentStatuses().values());
- if (canExecute) {
- log.info("Starting {} on {}.", roleName, containerId);
- if (isDockerContainer(roleGroup) || isYarnDockerContainer(roleGroup)){
- addStartDockerCommand(roleName, roleGroup, containerId,
- response, null, timeout, false);
- } else if (scriptPath != null) {
- addStartCommand(roleName,
- roleGroup,
- containerId,
- response,
- scriptPath,
- null,
- null,
- timeout,
- isMarkedAutoRestart(roleGroup));
- } else {
- ComponentCommand startCmd = null;
- for (ComponentCommand compCmd : commands) {
- if (compCmd.getName().equals("START")) {
- startCmd = compCmd;
- }
- }
- ComponentCommand stopCmd = null;
- for (ComponentCommand compCmd : commands) {
- if (compCmd.getName().equals("STOP")) {
- stopCmd = compCmd;
- }
- }
- addStartCommand(roleName, roleGroup, containerId, response, null,
- startCmd, stopCmd, timeout, false);
- }
- componentStatus.commandIssued(command);
- } else {
- log.info("Start of {} on {} delayed as dependencies have not started.", roleName, containerId);
- }
- } else if (command == Command.UPGRADE) {
- addUpgradeCommand(roleName, roleGroup, containerId, response,
- scriptPath, timeout);
- componentStatus.commandIssued(command, true);
- } else if (command == Command.STOP) {
- log.info("Stop command being sent to container with id {}",
- containerId);
- addStopCommand(roleName, roleGroup, containerId, response, scriptPath,
- timeout, doUpgrade);
- componentStatus.commandIssued(command);
- } else if (command == Command.TERMINATE) {
- log.info("A formal terminate command is being sent to container {}"
- + " in state {}", label, componentStatus.getState());
- response.setTerminateAgent(true);
- }
- }
-
- // if there is no outstanding command then retrieve config
- if (isMaster && componentStatus.getState() == State.STARTED
- && command == Command.NOP) {
- if (!componentStatus.getConfigReported()) {
- log.info("Requesting applied config for {} on {}.", roleName, containerId);
- if (isDockerContainer(roleGroup) || isYarnDockerContainer(roleGroup)){
- addGetConfigDockerCommand(roleName, roleGroup, containerId, response);
- } else {
- addGetConfigCommand(roleName, roleGroup, containerId, response);
- }
- }
- }
-
- // if restart is required then signal
- response.setRestartEnabled(false);
- if (componentStatus.getState() == State.STARTED
- && command == Command.NOP && isMarkedAutoRestart(roleGroup)) {
- response.setRestartEnabled(true);
- }
-
- //If INSTALL_FAILED and no INSTALL is scheduled let the agent fail
- if (componentStatus.getState() == State.INSTALL_FAILED
- && command == Command.NOP) {
- log.warn("Sending terminate signal to container that failed installation: {}", label);
- response.setTerminateAgent(true);
- }
-
- } catch (SliderException e) {
- log.warn("Component instance failed operation.", e);
- componentStatus.applyCommandResult(CommandResult.FAILED, command, null);
- }
-
- log.debug("Heartbeat response: " + response);
- return response;
- }
-
- private boolean isDockerContainer(String roleGroup) {
- String type = getApplicationComponent(roleGroup).getType();
- if (SliderUtils.isSet(type)) {
- return type.toLowerCase().equals(SliderUtils.DOCKER) || type.toLowerCase().equals(SliderUtils.DOCKER_YARN);
- }
- return false;
- }
-
- private boolean isYarnDockerContainer(String roleGroup) {
- String type = getApplicationComponent(roleGroup).getType();
- if (SliderUtils.isSet(type)) {
- return type.toLowerCase().equals(SliderUtils.DOCKER_YARN);
- }
- return false;
- }
-
- protected void processAllocatedPorts(String fqdn,
- String roleName,
- String roleGroup,
- String containerId,
- Map<String, String> ports) {
- RoleInstance instance;
- try {
- instance = getAmState().getOwnedContainer(containerId);
- } catch (NoSuchNodeException e) {
- log.warn("Failed to locate instance of container {}", containerId, e);
- instance = null;
- }
- for (Map.Entry<String, String> port : ports.entrySet()) {
- String portname = port.getKey();
- String portNo = port.getValue();
- log.info("Recording allocated port for {} as {}", portname, portNo);
-
- // add the allocated ports to the global list as well as per container list
- // per container allocation will over-write each other in the global
- this.getAllocatedPorts().put(portname, portNo);
- this.getAllocatedPorts(containerId).put(portname, portNo);
- if (instance != null) {
- try {
- // if the returned value is not a single port number then there are no
- // meaningful way for Slider to use it during export
- // No need to error out as it may not be the responsibility of the component
- // to allocate port or the component may need an array of ports
- instance.registerPortEndpoint(Integer.valueOf(portNo), portname);
- } catch (NumberFormatException e) {
- log.warn("Failed to parse {}", portNo, e);
- }
- }
- }
-
- processAndPublishComponentSpecificData(ports, containerId, fqdn, roleGroup);
- processAndPublishComponentSpecificExports(ports, containerId, fqdn, roleName, roleGroup);
-
- // and update registration entries
- if (instance != null) {
- queueAccess.put(new RegisterComponentInstance(instance.getId(),
- roleName, roleGroup, 0, TimeUnit.MILLISECONDS));
- }
- }
-
- private void updateComponentStatusWithAgentState(
- ComponentInstanceState componentStatus, State agentState) {
- if (agentState != null) {
- componentStatus.setState(agentState);
- }
- }
-
- @Override
- public Map<String, MonitorDetail> buildMonitorDetails(ClusterDescription clusterDesc) {
- Map<String, MonitorDetail> details = super.buildMonitorDetails(clusterDesc);
- buildRoleHostDetails(details);
- return details;
- }
-
- public void applyInitialRegistryDefinitions(URL amWebURI,
- URL agentOpsURI,
- URL agentStatusURI,
- ServiceRecord serviceRecord)
- throws IOException {
- super.applyInitialRegistryDefinitions(amWebURI,
- serviceRecord);
-
- try {
- URL restURL = new URL(agentOpsURI, SLIDER_PATH_AGENTS);
- URL agentStatusURL = new URL(agentStatusURI, SLIDER_PATH_AGENTS);
-
- serviceRecord.addInternalEndpoint(
- new Endpoint(CustomRegistryConstants.AGENT_SECURE_REST_API,
- ProtocolTypes.PROTOCOL_REST,
- restURL.toURI()));
- serviceRecord.addInternalEndpoint(
- new Endpoint(CustomRegistryConstants.AGENT_ONEWAY_REST_API,
- ProtocolTypes.PROTOCOL_REST,
- agentStatusURL.toURI()));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
-
- // identify client component
- Component client = null;
- for (Component component : getMetaInfo().getApplication().getComponents()) {
- if (component.getCategory().equals("CLIENT")) {
- client = component;
- break;
- }
- }
- if (client == null) {
- log.info("No client component specified, not publishing client configs");
- return;
- }
-
- // register AM-generated client configs
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- MapOperations clientOperations = appConf.getOrAddComponent(client.getName());
- appConf.resolve();
- if (!clientOperations.getOptionBool(AM_CONFIG_GENERATION,
- false)) {
- log.info("AM config generation is false, not publishing client configs");
- return;
- }
-
- // build and localize configuration files
- Map<String, Map<String, String>> configurations = new TreeMap<>();
- Map<String, String> tokens = providerUtils.getStandardTokenMap(appConf,
- getAmState().getInternalsSnapshot(), client.getName(),
- client.getName(), getClusterName());
-
- for (ConfigFile configFile : getMetaInfo().getComponentConfigFiles(client.getName())) {
- addNamedConfiguration(configFile.getDictionaryName(),
- appConf.getGlobalOptions().options, configurations, tokens, null,
- client.getName(), client.getName());
- if (appConf.getComponent(client.getName()) != null) {
- addNamedConfiguration(configFile.getDictionaryName(),
- appConf.getComponent(client.getName()).options, configurations,
- tokens, null, client.getName(), client.getName());
- }
- }
-
- //do a final replacement of re-used configs
- dereferenceAllConfigs(configurations);
-
- for (ConfigFile configFile : getMetaInfo().getComponentConfigFiles(client.getName())) {
- ConfigFormat configFormat = ConfigFormat.resolve(configFile.getType());
-
- Map<String, String> config = configurations.get(configFile.getDictionaryName());
- ConfigUtils.prepConfigForTemplateOutputter(configFormat, config,
- fileSystem, getClusterName(),
- new File(configFile.getFileName()).getName());
- PublishedConfiguration publishedConfiguration =
- new PublishedConfiguration(configFile.getDictionaryName(),
- config.entrySet());
- getAmState().getPublishedSliderConfigurations().put(
- configFile.getDictionaryName(), publishedConfiguration);
- log.info("Publishing AM configuration {}", configFile.getDictionaryName());
- }
- }
-
- @Override
- public void notifyContainerCompleted(ContainerId containerId) {
- // containers get allocated and free'ed without being assigned to any
- // component - so many of the data structures may not be initialized
- if (containerId != null) {
- String containerIdStr = containerId.toString();
- if (getComponentInstanceData().containsKey(containerIdStr)) {
- getComponentInstanceData().remove(containerIdStr);
- log.info("Removing container specific data for {}", containerIdStr);
- publishComponentInstanceData();
- }
-
- if (this.allocatedPorts.containsKey(containerIdStr)) {
- Map<String, String> portsByContainerId = getAllocatedPorts(containerIdStr);
- this.allocatedPorts.remove(containerIdStr);
- // free up the allocations from global as well
- // if multiple containers allocate global ports then last one
- // wins and similarly first one removes it - its not supported anyway
- for(String portName : portsByContainerId.keySet()) {
- getAllocatedPorts().remove(portName);
- }
-
- }
-
- String componentName = null;
- synchronized (this.componentStatuses) {
- for (String label : getComponentStatuses().keySet()) {
- if (label.startsWith(containerIdStr)) {
- componentName = getRoleName(label);
- log.info("Removing component status for label {}", label);
- getComponentStatuses().remove(label);
- }
- }
- }
-
- tags.releaseTag(componentName, containerIdStr);
-
- synchronized (this.containerExportsMap) {
- Set<String> containerExportSets = containerExportsMap.get(containerIdStr);
- if (containerExportSets != null) {
- for (String containerExportStr : containerExportSets) {
- String[] parts = containerExportStr.split(":");
- Map<String, List<ExportEntry>> exportGroup = getCurrentExports(parts[0]);
- List<ExportEntry> exports = exportGroup.get(parts[1]);
- List<ExportEntry> exportToRemove = new ArrayList<ExportEntry>();
- for (ExportEntry export : exports) {
- if (containerIdStr.equals(export.getContainerId())) {
- exportToRemove.add(export);
- }
- }
- exports.removeAll(exportToRemove);
- }
- log.info("Removing container exports for {}", containerIdStr);
- containerExportsMap.remove(containerIdStr);
- }
- }
- }
- }
-
- /**
- * Reads and sets the heartbeat monitoring interval. If bad value is provided then log it and set to default.
- *
- * @param instanceDefinition
- */
- private void readAndSetHeartbeatMonitoringInterval(AggregateConf instanceDefinition) {
- String hbMonitorInterval = instanceDefinition.getAppConfOperations().
- getGlobalOptions().getOption(HEARTBEAT_MONITOR_INTERVAL,
- Integer.toString(DEFAULT_HEARTBEAT_MONITOR_INTERVAL));
- try {
- setHeartbeatMonitorInterval(Integer.parseInt(hbMonitorInterval));
- } catch (NumberFormatException e) {
- log.warn(
- "Bad value {} for {}. Defaulting to ",
- hbMonitorInterval,
- HEARTBEAT_MONITOR_INTERVAL,
- DEFAULT_HEARTBEAT_MONITOR_INTERVAL);
- }
- }
-
- /**
- * Reads and sets the heartbeat monitoring interval. If bad value is provided then log it and set to default.
- *
- * @param instanceDefinition
- */
- private void initializeAgentDebugCommands(AggregateConf instanceDefinition) {
- String launchParameterStr = instanceDefinition.getAppConfOperations().
- getGlobalOptions().getOption(AGENT_INSTANCE_DEBUG_DATA, "");
- agentLaunchParameter = new AgentLaunchParameter(launchParameterStr);
- }
-
- @VisibleForTesting
- protected Map<String, ExportEntry> getLogFolderExports() {
- return logFolderExports;
- }
-
- @VisibleForTesting
- protected Map<String, ExportEntry> getWorkFolderExports() {
- return workFolderExports;
- }
-
- protected Metainfo getMetaInfo() {
- return getMetaInfo(null);
- }
-
- @VisibleForTesting
- protected Metainfo getMetaInfo(String roleGroup) {
- String mapKey = DEFAULT_METAINFO_MAP_KEY;
- if (roleGroup != null) {
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- mapKey = appConf.getComponentOpt(roleGroup, ROLE_PREFIX,
- DEFAULT_METAINFO_MAP_KEY);
- }
- MetainfoHolder mh = this.metaInfoMap.get(mapKey);
- if (mh == null) {
- return null;
- }
- return mh.metaInfo;
- }
-
- @VisibleForTesting
- protected Map<String, ComponentInstanceState> getComponentStatuses() {
- return componentStatuses;
- }
-
- @VisibleForTesting
- protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
- String appDef, boolean addonPackage) throws IOException,
- BadConfigException {
- return AgentUtils.getApplicationMetainfo(fileSystem, appDef, addonPackage);
- }
-
- @VisibleForTesting
- protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
- String appDef) throws IOException, BadConfigException {
- return getApplicationMetainfo(fileSystem, appDef, false);
- }
-
- @VisibleForTesting
- protected void setHeartbeatMonitorInterval(int heartbeatMonitorInterval) {
- this.heartbeatMonitorInterval = heartbeatMonitorInterval;
- }
-
- public void setInUpgradeMode(boolean inUpgradeMode) {
- this.isInUpgradeMode = inUpgradeMode;
- }
-
- public void addUpgradeContainers(Set<String> upgradeContainers) {
- this.upgradeContainers.addAll(upgradeContainers);
- }
-
- public void setAppStopInitiated(boolean appStopInitiated) {
- this.appStopInitiated = appStopInitiated;
- }
-
- /**
- * Read all default configs
- *
- * @param fileSystem fs
- * @param appDef app default path
- * @param metainfo metadata
- *
- * @return configuration maps
- *
- * @throws IOException
- */
- protected Map<String, DefaultConfig> initializeDefaultConfigs(SliderFileSystem fileSystem,
- String appDef, Metainfo metainfo) throws IOException {
- Map<String, DefaultConfig> defaultConfigMap = new HashMap<>();
- if (SliderUtils.isNotEmpty(metainfo.getApplication().getConfigFiles())) {
- for (ConfigFile configFile : metainfo.getApplication().getConfigFiles()) {
- DefaultConfig config = null;
- try {
- config = AgentUtils.getDefaultConfig(fileSystem, appDef, configFile.getDictionaryName() + ".xml");
- } catch (IOException e) {
- log.warn("Default config file not found. Only the config as input during create will be applied for {}",
- configFile.getDictionaryName());
- }
- if (config != null) {
- defaultConfigMap.put(configFile.getDictionaryName(), config);
- }
- }
- }
-
- return defaultConfigMap;
- }
-
- protected Map<String, DefaultConfig> getDefaultConfigs(String roleGroup) {
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- String mapKey = appConf.getComponentOpt(roleGroup, ROLE_PREFIX,
- DEFAULT_METAINFO_MAP_KEY);
- return metaInfoMap.get(mapKey).defaultConfigs;
- }
-
- private int getHeartbeatMonitorInterval() {
- return this.heartbeatMonitorInterval;
- }
-
- private String getClusterName() {
- if (SliderUtils.isUnset(clusterName)) {
- clusterName = getAmState().getInternalsSnapshot().get(OptionKeys.APPLICATION_NAME);
- }
- return clusterName;
- }
-
- @VisibleForTesting
- protected void publishApplicationInstanceData(String name, String description,
- Iterable<Map.Entry<String, String>> entries) {
- providerUtils.publishApplicationInstanceData(name, description, entries,
- getAmState());
- }
-
- /**
- * Get a list of all hosts for all role/container per role
- *
- * @return the map of role->node
- */
- protected Map<String, Map<String, ClusterNode>> getRoleClusterNodeMapping() {
- return amState.getRoleClusterNodeMapping();
- }
-
- private String getContainerLabel(Container container, String role, String group) {
- if (role.equals(group)) {
- return container.getId().toString() + LABEL_MAKER + role;
- } else {
- return container.getId().toString() + LABEL_MAKER + role + LABEL_MAKER +
- group;
- }
- }
-
- protected String getClusterInfoPropertyValue(String name) {
- StateAccessForProviders accessor = getAmState();
- assert accessor.isApplicationLive();
- ClusterDescription description = accessor.getClusterStatus();
- return description.getInfo(name);
- }
-
- protected String getClusterOptionPropertyValue(String name)
- throws BadConfigException {
- StateAccessForProviders accessor = getAmState();
- assert accessor.isApplicationLive();
- ClusterDescription description = accessor.getClusterStatus();
- return description.getMandatoryOption(name);
- }
-
- /**
- * Lost heartbeat from the container - release it and ask for a replacement (async operation)
- *
- * @param label
- * @param containerId
- */
- protected void lostContainer(
- String label,
- ContainerId containerId) {
- getComponentStatuses().remove(label);
- getQueueAccess().put(new ProviderReportedContainerLoss(containerId));
- }
-
- /**
- * Build the provider status, can be empty
- *
- * @return the provider status - map of entries to add to the info section
- */
- public Map<String, String> buildProviderStatus() {
- Map<String, String> stats = new HashMap<String, String>();
- return stats;
- }
-
- @VisibleForTesting
- protected void publishFolderPaths(
- Map<String, String> folders, String containerId, String componentName, String hostFqdn) {
- providerUtils.publishFolderPaths(folders, containerId, componentName, hostFqdn,
- getAmState(), getLogFolderExports(), getWorkFolderExports());
- }
-
- /**
- * Process return status for component instances
- *
- * @param heartBeat
- * @param componentStatus
- */
- protected void publishConfigAndExportGroups(HeartBeat heartBeat,
- ComponentInstanceState componentStatus, String componentGroup) {
- List<ComponentStatus> statuses = heartBeat.getComponentStatus();
- if (statuses != null && !statuses.isEmpty()) {
- log.info("Processing {} status reports.", statuses.size());
- for (ComponentStatus status : statuses) {
- log.info("Status report: {}", status.toString());
-
- if (status.getConfigs() != null) {
- Application application = getMetaInfo(componentGroup).getApplication();
-
- if ((!canAnyMasterPublishConfig(componentGroup) || canPublishConfig(componentGroup)) &&
- !getAmState().getAppConfSnapshot().getComponentOptBool(
- componentGroup, AM_CONFIG_GENERATION, false)) {
- // If no Master can explicitly publish then publish if its a master
- // Otherwise, wait till the master that can publish is ready
-
- Set<String> exportedConfigs = new HashSet();
- String exportedConfigsStr = application.getExportedConfigs();
- boolean exportedAllConfigs = exportedConfigsStr == null || exportedConfigsStr.isEmpty();
- if (!exportedAllConfigs) {
- for (String exportedConfig : exportedConfigsStr.split(",")) {
- if (exportedConfig.trim().length() > 0) {
- exportedConfigs.add(exportedConfig.trim());
- }
- }
- }
-
- for (String key : status.getConfigs().keySet()) {
- if ((!exportedAllConfigs && exportedConfigs.contains(key)) ||
- exportedAllConfigs) {
- Map<String, String> configs = status.getConfigs().get(key);
- publishApplicationInstanceData(key, key, configs.entrySet());
- }
- }
- }
-
- List<ExportGroup> appExportGroups = application.getExportGroups();
- boolean hasExportGroups = SliderUtils.isNotEmpty(appExportGroups);
-
- Set<String> appExports = new HashSet();
- String appExportsStr = getApplicationComponent(componentGroup).getAppExports();
- if (SliderUtils.isSet(appExportsStr)) {
- for (String appExport : appExportsStr.split(",")) {
- if (!appExport.trim().isEmpty()) {
- appExports.add(appExport.trim());
- }
- }
- }
-
- if (hasExportGroups && !appExports.isEmpty()) {
- String configKeyFormat = "${site.%s.%s}";
- String hostKeyFormat = "${%s_HOST}";
-
- // publish export groups if any
- Map<String, String> replaceTokens = new HashMap<String, String>();
- for (Map.Entry<String, Map<String, ClusterNode>> entry : getRoleClusterNodeMapping().entrySet()) {
- String hostName = providerUtils.getHostsList(
- entry.getValue().values(), true).iterator().next();
- replaceTokens.put(String.format(hostKeyFormat, entry.getKey().toUpperCase(Locale.ENGLISH)), hostName);
- }
-
- for (String key : status.getConfigs().keySet()) {
- Map<String, String> configs = status.getConfigs().get(key);
- for (String configKey : configs.keySet()) {
- String lookupKey = String.format(configKeyFormat, key, configKey);
- replaceTokens.put(lookupKey, configs.get(configKey));
- }
- }
-
- Set<String> modifiedGroups = new HashSet<String>();
- for (ExportGroup exportGroup : appExportGroups) {
- List<Export> exports = exportGroup.getExports();
- if (SliderUtils.isNotEmpty(exports)) {
- String exportGroupName = exportGroup.getName();
- ConcurrentHashMap<String, List<ExportEntry>> map =
- (ConcurrentHashMap<String, List<ExportEntry>>)getCurrentExports(exportGroupName);
- for (Export export : exports) {
- if (canBeExported(exportGroupName, export.getName(), appExports)) {
- String value = export.getValue();
- // replace host names
- for (String token : replaceTokens.keySet()) {
- if (value.contains(token)) {
- value = value.replace(token, replaceTokens.get(token));
- }
- }
- ExportEntry entry = new ExportEntry();
- entry.setLevel(APPLICATION_TAG);
- entry.setValue(value);
- entry.setUpdatedTime(new Date().toString());
- // over-write, app exports are singletons
- map.put(export.getName(), new ArrayList(Arrays.asList(entry)));
- log.info("Preparing to publish. Key {} and Value {}", export.getName(), value);
- }
- }
- modifiedGroups.add(exportGroupName);
- }
- }
- publishModifiedExportGroups(modifiedGroups);
- }
-
- log.info("Received and processed config for {}", heartBeat.getHostname());
- componentStatus.setConfigReported(true);
-
- }
- }
- }
- }
-
- private boolean canBeExported(String exportGroupName, String name, Set<String> appExports) {
- return appExports.contains(String.format("%s-%s", exportGroupName, name));
- }
-
- protected Map<String, List<ExportEntry>> getCurrentExports(String groupName) {
- if (!this.exportGroups.containsKey(groupName)) {
- synchronized (this.exportGroups) {
- if (!this.exportGroups.containsKey(groupName)) {
- this.exportGroups.put(groupName, new ConcurrentHashMap<String, List<ExportEntry>>());
- }
- }
- }
-
- return this.exportGroups.get(groupName);
- }
-
- private void publishModifiedExportGroups(Set<String> modifiedGroups) {
- for (String roleGroup : modifiedGroups) {
- Map<String, List<ExportEntry>> entries = this.exportGroups.get(roleGroup);
- // Publish in old format for the time being
- Map<String, String> simpleEntries = new HashMap<String, String>();
- for (Entry<String, List<ExportEntry>> entry : entries.entrySet()) {
- List<ExportEntry> exports = entry.getValue();
- if (SliderUtils.isNotEmpty(exports)) {
- // there is no support for multiple exports per name - so extract only the first one
- simpleEntries.put(entry.getKey(), entry.getValue().get(0).getValue());
- }
- }
- publishApplicationInstanceData(roleGroup, roleGroup,
- simpleEntries.entrySet());
-
- PublishedExports exports = new PublishedExports(roleGroup);
- exports.setUpdated(new Date().getTime());
- exports.putValues(entries.entrySet());
- getAmState().getPublishedExportsSet().put(roleGroup, exports);
- }
- }
-
- /** Publish component instance specific data if the component demands it */
- protected void processAndPublishComponentSpecificData(Map<String, String> ports,
- String containerId,
- String hostFqdn,
- String componentGroup) {
- String portVarFormat = "${site.%s}";
- String hostNamePattern = "${THIS_HOST}";
- Map<String, String> toPublish = new HashMap<String, String>();
-
- Application application = getMetaInfo(componentGroup).getApplication();
- for (Component component : application.getComponents()) {
- if (component.getName().equals(componentGroup)) {
- if (component.getComponentExports().size() > 0) {
-
- for (ComponentExport export : component.getComponentExports()) {
- String templateToExport = export.getValue();
- for (String portName : ports.keySet()) {
- boolean publishData = false;
- String portValPattern = String.format(portVarFormat, portName);
- if (templateToExport.contains(portValPattern)) {
- templateToExport = templateToExport.replace(portValPattern, ports.get(portName));
- publishData = true;
- }
- if (templateToExport.contains(hostNamePattern)) {
- templateToExport = templateToExport.replace(hostNamePattern, hostFqdn);
- publishData = true;
- }
- if (publishData) {
- toPublish.put(export.getName(), templateToExport);
- log.info("Publishing {} for name {} and container {}",
- templateToExport, export.getName(), containerId);
- }
- }
- }
- }
- }
- }
-
- if (toPublish.size() > 0) {
- Map<String, String> perContainerData = null;
- if (!getComponentInstanceData().containsKey(containerId)) {
- perContainerData = new ConcurrentHashMap<String, String>();
- } else {
- perContainerData = getComponentInstanceData().get(containerId);
- }
- perContainerData.putAll(toPublish);
- getComponentInstanceData().put(containerId, perContainerData);
- publishComponentInstanceData();
- }
- }
-
- /** Publish component instance specific data if the component demands it */
- protected void processAndPublishComponentSpecificExports(Map<String, String> ports,
- String containerId,
- String hostFqdn,
- String compName,
- String compGroup) {
- String portVarFormat = "${site.%s}";
- String hostNamePattern = "${" + compGroup + "_HOST}";
-
- List<ExportGroup> appExportGroups = getMetaInfo(compGroup).getApplication().getExportGroups();
- Component component = getApplicationComponent(compGroup);
- if (component != null && SliderUtils.isSet(component.getCompExports())
- && SliderUtils.isNotEmpty(appExportGroups)) {
-
- Set<String> compExports = new HashSet();
- String compExportsStr = component.getCompExports();
- for (String compExport : compExportsStr.split(",")) {
- if (!compExport.trim().isEmpty()) {
- compExports.add(compExport.trim());
- }
- }
-
- Date now = new Date();
- Set<String> modifiedGroups = new HashSet<String>();
- for (ExportGroup exportGroup : appExportGroups) {
- List<Export> exports = exportGroup.getExports();
- if (SliderUtils.isNotEmpty(exports)) {
- String exportGroupName = exportGroup.getName();
- ConcurrentHashMap<String, List<ExportEntry>> map =
- (ConcurrentHashMap<String, List<ExportEntry>>) getCurrentExports(exportGroupName);
- for (Export export : exports) {
- if (canBeExported(exportGroupName, export.getName(), compExports)) {
- log.info("Attempting to publish {} of group {} for component type {}",
- export.getName(), exportGroupName, compName);
- String templateToExport = export.getValue();
- for (String portName : ports.keySet()) {
- boolean publishData = false;
- String portValPattern = String.format(portVarFormat, portName);
- if (templateToExport.contains(portValPattern)) {
- templateToExport = templateToExport.replace(portValPattern, ports.get(portName));
- publishData = true;
- }
- if (templateToExport.contains(hostNamePattern)) {
- templateToExport = templateToExport.replace(hostNamePattern, hostFqdn);
- publishData = true;
- }
- if (publishData) {
- ExportEntry entryToAdd = new ExportEntry();
- entryToAdd.setLevel(COMPONENT_TAG);
- entryToAdd.setValue(templateToExport);
- entryToAdd.setUpdatedTime(now.toString());
- entryToAdd.setContainerId(containerId);
- entryToAdd.setTag(tags.getTag(compName, containerId));
-
- List<ExportEntry> existingList =
- map.putIfAbsent(export.getName(), new CopyOnWriteArrayList(Arrays.asList(entryToAdd)));
-
- // in-place edit, no lock needed
- if (existingList != null) {
- boolean updatedInPlace = false;
- for (ExportEntry entry : existingList) {
- if (containerId.toLowerCase(Locale.ENGLISH)
- .equals(entry.getContainerId())) {
- entryToAdd.setValue(templateToExport);
- entryToAdd.setUpdatedTime(now.toString());
- updatedInPlace = true;
- }
- }
- if (!updatedInPlace) {
- existingList.add(entryToAdd);
- }
- }
-
- log.info("Publishing {} for name {} and container {}",
- templateToExport, export.getName(), containerId);
- modifiedGroups.add(exportGroupName);
- synchronized (containerExportsMap) {
- if (!containerExportsMap.containsKey(containerId)) {
- containerExportsMap.put(containerId, new HashSet<String>());
- }
- Set<String> containerExportMaps = containerExportsMap.get(containerId);
- containerExportMaps.add(String.format("%s:%s", exportGroupName, export.getName()));
- }
- }
- }
- }
- }
- }
- }
- publishModifiedExportGroups(modifiedGroups);
- }
- }
-
- private void publishComponentInstanceData() {
- Map<String, String> dataToPublish = new HashMap<String, String>();
- for (String container : getComponentInstanceData().keySet()) {
- for (String prop : getComponentInstanceData().get(container).keySet()) {
- dataToPublish.put(
- container + "." + prop, getComponentInstanceData().get(container).get(prop));
- }
- }
- publishApplicationInstanceData(COMPONENT_DATA_TAG, COMPONENT_DATA_TAG, dataToPublish.entrySet());
- }
-
- /**
- * Return Component based on group
- *
- * @param roleGroup component group
- *
- * @return the component entry or null for no match
- */
- protected Component getApplicationComponent(String roleGroup) {
- Metainfo metainfo = getMetaInfo(roleGroup);
- if (metainfo == null) {
- return null;
- }
- return metainfo.getApplicationComponent(roleGroup);
- }
-
- /**
- * Extract script path from the application metainfo
- *
- * @param roleGroup component group
- * @return the script path or null for no match
- */
- protected CommandScript getScriptPathForMasterPackage(String roleGroup) {
- Component component = getApplicationComponent(roleGroup);
- if (component != null) {
- return component.getCommandScript();
- }
- return null;
- }
-
- /**
- * Is the role of type MASTER
- *
- * @param roleGroup component group
- *
- * @return true if the role category is MASTER
- */
- protected boolean isMaster(String roleGroup) {
- Component component = getApplicationComponent(roleGroup);
- if (component != null) {
- if (component.getCategory().equals("MASTER")) {
- return true;
- }
- }
- return false;
- }
-
- /**
- * Can the role publish configuration
- *
- * @param roleGroup component group
- *
- * @return true if it can be pubished
- */
- protected boolean canPublishConfig(String roleGroup) {
- Component component = getApplicationComponent(roleGroup);
- if (component != null) {
- return Boolean.TRUE.toString().equals(component.getPublishConfig());
- }
- return false;
- }
-
- /**
- * Checks if the role is marked auto-restart
- *
- * @param roleGroup component group
- *
- * @return true if it is auto-restart
- */
- protected boolean isMarkedAutoRestart(String roleGroup) {
- Component component = getApplicationComponent(roleGroup);
- if (component != null) {
- return component.getAutoStartOnFailureBoolean();
- }
- return false;
- }
-
- /**
- * Can any master publish config explicitly, if not a random master is used
- *
- * @return true if the condition holds
- */
- protected boolean canAnyMasterPublishConfig(String roleGroup) {
- if (canAnyMasterPublish == null) {
- Application application = getMetaInfo(roleGroup).getApplication();
- if (application == null) {
- log.error("Malformed app definition: Expect application as root element in the metainfo.xml");
- } else {
- for (Component component : application.getComponents()) {
- if (Boolean.TRUE.toString().equals(component.getPublishConfig()) &&
- component.getCategory().equals("MASTER")) {
- canAnyMasterPublish = true;
- }
- }
- }
- }
-
- if (canAnyMasterPublish == null) {
- canAnyMasterPublish = false;
- }
- return canAnyMasterPublish;
- }
-
- private String getRoleName(String label) {
- int index1 = label.indexOf(LABEL_MAKER);
- int index2 = label.lastIndexOf(LABEL_MAKER);
- if (index1 == index2) {
- return label.substring(index1 + LABEL_MAKER.length());
- } else {
- return label.substring(index1 + LABEL_MAKER.length(), index2);
- }
- }
-
- private String getRoleGroup(String label) {
- return label.substring(label.lastIndexOf(LABEL_MAKER) + LABEL_MAKER.length());
- }
-
- private String getContainerId(String label) {
- return label.substring(0, label.indexOf(LABEL_MAKER));
- }
-
- /**
- * Add install command to the heartbeat response
- *
- * @param roleName
- * @param roleGroup
- * @param containerId
- * @param response
- * @param scriptPath
- * @param pkg
- * when this field is null, it indicates the command is for the
- * master package; while not null, for the package named by this
- * field
- * @throws SliderException
- */
- @VisibleForTesting
- protected void addInstallCommand(String roleName,
- String roleGroup,
- String containerId,
- HeartBeatResponse response,
- String scriptPath,
- ComponentCommand compCmd,
- long timeout,
- String pkg)
- throws SliderException {
- assert getAmState().isApplicationLive();
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
-
- ExecutionCommand cmd = new ExecutionCommand(AgentCommandType.EXECUTION_COMMAND);
- prepareExecutionCommand(cmd);
- String clusterName = getClusterName();
- cmd.setClusterName(clusterName);
- cmd.setRoleCommand(Command.INSTALL.toString());
- cmd.setServiceName(clusterName);
- cmd.setComponentName(roleName);
- cmd.setRole(roleName);
- cmd.setPkg(pkg);
- Map<String, String> hostLevelParams = new TreeMap<String, String>();
- hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir()));
- hostLevelParams.put(PACKAGE_LIST, getPackageList(roleGroup));
- hostLevelParams.put(CONTAINER_ID, containerId);
- cmd.setHostLevelParams(hostLevelParams);
-
- Map<String, Map<String, String>> configurations =
- buildCommandConfigurations(appConf, getAmState().getInternalsSnapshot(),
- containerId, roleName, roleGroup);
- cmd.setConfigurations(configurations);
- Map<String, Map<String, String>> componentConfigurations = buildComponentConfigurations(appConf);
- cmd.setComponentConfigurations(componentConfigurations);
-
- if (SliderUtils.isSet(scriptPath)) {
- cmd.setCommandParams(commandParametersSet(scriptPath, timeout, false));
- } else {
- // assume it to be default shell command
- ComponentCommand effectiveCommand = compCmd;
- if (effectiveCommand == null) {
- effectiveCommand = ComponentCommand.getDefaultComponentCommand("INSTALL");
- }
- cmd.setCommandParams(commandParametersSet(effectiveCommand, timeout, false));
- configurations.get("global").put("exec_cmd", effectiveCommand.getExec());
- }
-
- cmd.setHostname(getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME));
-
- response.addExecutionCommand(cmd);
-
- log.debug("command looks like: {} ", cmd);
- }
-
- @VisibleForTesting
- protected void addInstallDockerCommand(String roleName,
- String roleGroup,
- String containerId,
- HeartBeatResponse response,
- ComponentCommand compCmd,
- long timeout)
- throws SliderException {
- assert getAmState().isApplicationLive();
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
-
- ExecutionCommand cmd = new ExecutionCommand(AgentCommandType.EXECUTION_COMMAND);
- prepareExecutionCommand(cmd);
- String clusterName = getClusterName();
- cmd.setClusterName(clusterName);
- cmd.setRoleCommand(Command.INSTALL.toString());
- cmd.setServiceName(clusterName);
- cmd.setComponentName(roleName);
- cmd.setRole(roleName);
- Map<String, String> hostLevelParams = new TreeMap<String, String>();
- hostLevelParams.put(PACKAGE_LIST, getPackageList(roleGroup));
- hostLevelParams.put(CONTAINER_ID, containerId);
- cmd.setHostLevelParams(hostLevelParams);
-
- Map<String, Map<String, String>> configurations = buildCommandConfigurations(
- appConf, getAmState().getInternalsSnapshot(), containerId, roleName,
- roleGroup);
- cmd.setConfigurations(configurations);
- Map<String, Map<String, String>> componentConfigurations = buildComponentConfigurations(appConf);
- cmd.setComponentConfigurations(componentConfigurations);
-
- ComponentCommand effectiveCommand = compCmd;
- if (compCmd == null) {
- effectiveCommand = new ComponentCommand();
- effectiveCommand.setName("INSTALL");
- effectiveCommand.setExec("DEFAULT");
- }
- cmd.setCommandParams(setCommandParameters(effectiveCommand, timeout, false));
- configurations.get("global").put("exec_cmd", effectiveCommand.getExec());
-
- cmd.setHostname(getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME));
- cmd.addContainerDetails(roleGroup, getMetaInfo(roleGroup));
-
- Map<String, String> dockerConfig = new HashMap<String, String>();
- if(isYarnDockerContainer(roleGroup)){
- //put nothing
- cmd.setYarnDockerMode(true);
- } else {
- dockerConfig.put(
- "docker.command_path",
- getConfigFromMetaInfoWithAppConfigOverriding(roleGroup,
- "commandPath"));
- dockerConfig.put("docker.image_name",
- getConfigFromMetaInfo(roleGroup, "image"));
- }
- configurations.put("docker", dockerConfig);
-
- log.debug("Docker- command: {}", cmd.toString());
-
- response.addExecutionCommand(cmd);
- }
-
- private Map<String, String> setCommandParameters(String scriptPath,
- long timeout, boolean recordConfig) {
- Map<String, String> cmdParams = new TreeMap<String, String>();
- cmdParams.put("service_package_folder",
- "${AGENT_WORK_ROOT}/work/app/definition/package");
- cmdParams.put("script", scriptPath);
- cmdParams.put("schema_version", "2.0");
- cmdParams.put("command_timeout", Long.toString(timeout));
- cmdParams.put("script_type", AbstractComponent.TYPE_PYTHON);
- cmdParams.put("record_config", Boolean.toString(recordConfig));
- return cmdParams;
- }
-
- private Map<String, String> setCommandParameters(ComponentCommand compCmd,
- long timeout, boolean recordConfig) {
- Map<String, String> cmdParams = new TreeMap<String, String>();
- cmdParams.put("service_package_folder",
- "${AGENT_WORK_ROOT}/work/app/definition/package");
- cmdParams.put("command", compCmd.getExec());
- cmdParams.put("schema_version", "2.0");
- cmdParams.put("command_timeout", Long.toString(timeout));
- cmdParams.put("script_type", compCmd.getType());
- cmdParams.put("record_config", Boolean.toString(recordConfig));
- return cmdParams;
- }
-
- private Map<String, Map<String, String>> buildComponentConfigurations(
- ConfTreeOperations appConf) {
- return appConf.getComponents();
- }
-
- protected static String getPackageListFromApplication(Application application) {
- String pkgFormatString = "{\"type\":\"%s\",\"name\":\"%s\"}";
- String pkgListFormatString = "[%s]";
- List<
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/51] [abbrv] hadoop git commit: YARN-5967. Fix slider core module
findbugs warnings. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/MiniZooKeeperCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/MiniZooKeeperCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/MiniZooKeeperCluster.java
index c8b3adb..1af883e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/MiniZooKeeperCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/MiniZooKeeperCluster.java
@@ -137,8 +137,6 @@ public class MiniZooKeeperCluster extends AbstractService {
}
/**
- * @param baseDir
- * @param numZooKeeperServers
* @return ClientPort server bound to, -1 if there was a
* binding problem and we couldn't pick another port.
* @throws IOException
@@ -229,17 +227,6 @@ public class MiniZooKeeperCluster extends AbstractService {
}
}
- /**
- * Delete the basedir
- */
- private void deleteBaseDir() {
- if (baseDir != null) {
- baseDir.delete();
- baseDir = null;
- }
-
- }
-
@Override
protected void serviceStop() throws Exception {
@@ -359,7 +346,7 @@ public class MiniZooKeeperCluster extends AbstractService {
try {
sock = new Socket("localhost", port);
OutputStream outstream = sock.getOutputStream();
- outstream.write("stat".getBytes());
+ outstream.write("stat".getBytes("UTF-8"));
outstream.flush();
} finally {
IOUtils.closeSocket(sock);
@@ -387,10 +374,10 @@ public class MiniZooKeeperCluster extends AbstractService {
BufferedReader reader = null;
try {
OutputStream outstream = sock.getOutputStream();
- outstream.write("stat".getBytes());
+ outstream.write("stat".getBytes("UTF-8"));
outstream.flush();
- Reader isr = new InputStreamReader(sock.getInputStream());
+ Reader isr = new InputStreamReader(sock.getInputStream(), "UTF-8");
reader = new BufferedReader(isr);
String line = reader.readLine();
if (line != null && line.startsWith("Zookeeper version:")) {
@@ -412,12 +399,4 @@ public class MiniZooKeeperCluster extends AbstractService {
}
return false;
}
-
- public int getClientPort() {
- return clientPort;
- }
-
- public String getZkQuorum() {
- return zkQuorum;
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
index 6ed58d5..ef96c9b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
@@ -43,14 +43,14 @@ public class ZKIntegration implements Watcher, Closeable {
/**
* Base path for services
*/
- public static String ZK_SERVICES = "services";
+ public static final String ZK_SERVICES = "services";
/**
* Base path for all Slider references
*/
- public static String ZK_SLIDER = "slider";
- public static String ZK_USERS = "users";
- public static String SVC_SLIDER = "/" + ZK_SERVICES + "/" + ZK_SLIDER;
- public static String SVC_SLIDER_USERS = SVC_SLIDER + "/" + ZK_USERS;
+ public static final String ZK_SLIDER = "slider";
+ public static final String ZK_USERS = "users";
+ public static final String SVC_SLIDER = "/" + ZK_SERVICES + "/" + ZK_SLIDER;
+ public static final String SVC_SLIDER_USERS = SVC_SLIDER + "/" + ZK_USERS;
public static final List<String> ZK_USERS_PATH_LIST = new ArrayList<String>();
static {
@@ -59,7 +59,7 @@ public class ZKIntegration implements Watcher, Closeable {
ZK_USERS_PATH_LIST.add(ZK_USERS);
}
- public static int SESSION_TIMEOUT = 30000;
+ public static final int SESSION_TIMEOUT = 30000;
protected static final Logger log =
LoggerFactory.getLogger(ZKIntegration.class);
private ZooKeeper zookeeper;
@@ -279,14 +279,6 @@ public class ZKIntegration implements Watcher, Closeable {
}
}
-/**
- * Blocking enum of users
- * @return an unordered list of clusters under a user
- */
- public List<String> getClusters() throws KeeperException, InterruptedException {
- return zookeeper.getChildren(userPath, null);
- }
-
/**
* Delete a node, does not throw an exception if the path is not fond
* @param path path to delete
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index cff5ed8..0d9ddec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -735,11 +735,11 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
Map<String, Map<String, String>> configurations) {
Map<String, String> allConfigs = new HashMap<>();
String lookupFormat = "${@//site/%s/%s}";
- for (String configType : configurations.keySet()) {
- Map<String, String> configBucket = configurations.get(configType);
- for (String configName : configBucket.keySet()) {
- allConfigs.put(String.format(lookupFormat, configType, configName),
- configBucket.get(configName));
+ for (Map.Entry<String, Map<String, String>> entry : configurations.entrySet()) {
+ Map<String, String> configBucket = entry.getValue();
+ for(Map.Entry<String, String> config: configBucket.entrySet()) {
+ allConfigs.put(String.format(lookupFormat, entry.getKey(), config.getKey()),
+ config.getValue());
}
}
@@ -758,15 +758,15 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
configValue = configValue.replace(lookUpKey, lookUpValue);
}
}
- if (!configValue.equals(entry.getValue())) {
+ if (configValue != null && !configValue.equals(entry.getValue())) {
finished = false;
allConfigs.put(entry.getKey(), configValue);
}
}
}
-
- for (String configType : configurations.keySet()) {
- Map<String, String> configBucket = configurations.get(configType);
+ for (Map.Entry<String, Map<String, String>> configEntry : configurations
+ .entrySet()) {
+ Map<String, String> configBucket = configEntry.getValue();
for (Map.Entry<String, String> entry: configBucket.entrySet()) {
String configName = entry.getKey();
String configValue = entry.getValue();
@@ -817,7 +817,8 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
*/
private void addConfsToList(Map<String, String> confMap,
Set<String> confList, String prefix, String suffix) {
- for (String key : confMap.keySet()) {
+ for (Entry<String, String> entry : confMap.entrySet()) {
+ String key = entry.getKey();
if (key.startsWith(prefix) && key.endsWith(suffix)) {
String confName = key.substring(prefix.length(),
key.length() - suffix.length());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
index 1482062..5edfa6a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
@@ -257,19 +257,20 @@ public class DockerProviderService extends AbstractProviderService implements
.getInternalsSnapshot(), null, getClusterName(), clientName,
clientName, getAmState());
- for (String configFileDN : configurations.keySet()) {
+ for (Map.Entry<String, Map<String, String>> entry : configurations.entrySet()) {
+ String configFileDN = entry.getKey();
String configFileName = appConf.getComponentOpt(clientName,
OptionKeys.CONF_FILE_PREFIX + configFileDN + OptionKeys
.NAME_SUFFIX, null);
String configFileType = appConf.getComponentOpt(clientName,
OptionKeys.CONF_FILE_PREFIX + configFileDN + OptionKeys
.TYPE_SUFFIX, null);
- if (configFileName == null && configFileType == null) {
+ if (configFileName == null || configFileType == null) {
continue;
}
ConfigFormat configFormat = ConfigFormat.resolve(configFileType);
- Map<String, String> config = configurations.get(configFileDN);
+ Map<String, String> config = entry.getValue();
ConfigUtils.prepConfigForTemplateOutputter(configFormat, config,
fileSystem, getClusterName(),
new File(configFileName).getName());
@@ -365,9 +366,10 @@ public class DockerProviderService extends AbstractProviderService implements
for (Entry<String, String> export : exports.entrySet()) {
String value = export.getValue();
// replace host names and site properties
- for (String token : replaceTokens.keySet()) {
+ for (Map.Entry<String, String> entry : replaceTokens.entrySet()) {
+ String token = entry.getKey();
if (value.contains(token)) {
- value = value.replaceAll(Pattern.quote(token), replaceTokens.get(token));
+ value = value.replaceAll(Pattern.quote(token), entry.getValue());
}
}
ExportEntry entry = new ExportEntry();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetric.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetric.java
index 33f8d85..89dfbfd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetric.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetric.java
@@ -47,21 +47,6 @@ public class BoolMetric implements Metric, Gauge<Integer> {
return value.get() ? 1 : 0;
}
- /**
- * Evaluate from a string. Returns true if the string is considered to match 'true',
- * false otherwise.
- * @param s source
- * @return true if the input parses to an integer other than 0. False if it doesn't parse
- * or parses to 0.
- */
- public static boolean fromString(String s) {
- try {
- return Integer.valueOf(s) != 0;
- } catch (NumberFormatException e) {
- return false;
- }
- }
-
@Override
public String toString() {
return value.toString();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
index 37a8935..1fe8ea6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
@@ -58,11 +58,6 @@ public class MetricsAndMonitoring extends CompositeService {
private final List<MetricSet> metricSets = new ArrayList<>();
- /**
- * List of recorded events
- */
- private final List<RecordedEvent> eventHistory = new ArrayList<>(100);
-
public static final int EVENT_LIMIT = 1000;
public MetricRegistry getMetrics() {
@@ -139,26 +134,6 @@ public class MetricsAndMonitoring extends CompositeService {
return register(MetricRegistry.name(klass, names), metric);
}
-
- /**
- * Add an event (synchronized)
- * @param event event
- */
- public synchronized void noteEvent(RecordedEvent event) {
- if (eventHistory.size() > EVENT_LIMIT) {
- eventHistory.remove(0);
- }
- eventHistory.add(event);
- }
-
- /**
- * Clone the event history; blocks for the duration of the copy operation.
- * @return a new list
- */
- public synchronized List<RecordedEvent> cloneEventHistory() {
- return new ArrayList<>(eventHistory);
- }
-
/**
* Add a metric set for registering and deregistration on service stop
* @param metricSet metric set
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/RangeLimitedCounter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/RangeLimitedCounter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/RangeLimitedCounter.java
deleted file mode 100644
index 80e88fc..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/RangeLimitedCounter.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.management;
-
-import com.codahale.metrics.Counter;
-import com.codahale.metrics.Counting;
-import com.codahale.metrics.Metric;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * This is a counter whose range can be given a min and a max
- */
-public class RangeLimitedCounter implements Metric, Counting {
-
- private final AtomicLong value;
- private final long min, max;
-
- /**
- * Instantiate
- * @param val current value
- * @param min minimum value
- * @param max max value (or 0 for no max)
- */
- public RangeLimitedCounter(long val, long min, long max) {
- this.value = new AtomicLong(val);
- this.min = min;
- this.max = max;
- }
-
- /**
- * Set to a new value. If below the min, set to the minimum. If the max is non
- * zero and the value is above that maximum, set it to the maximum instead.
- * @param val value
- */
- public synchronized void set(long val) {
- if (val < min) {
- val = min;
- } else if (max > 0 && val > max) {
- val = max;
- }
- value.set(val);
- }
-
- public void inc() {
- inc(1);
- }
-
- public void dec() {
- dec(1);
- }
-
- public synchronized void inc(int delta) {
- set(value.get() + delta);
- }
-
- public synchronized void dec(int delta) {
- set(value.get() - delta);
- }
-
- public long get() {
- return value.get();
- }
-
- @Override
- public long getCount() {
- return value.get();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/RecordedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/RecordedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/RecordedEvent.java
deleted file mode 100644
index d48d337..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/RecordedEvent.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.management;
-
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.text.DateFormat;
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class RecordedEvent {
- private static final DateFormat dateFormat = DateFormat.getDateInstance();
- public long id;
- public String name;
- public long timestamp;
- public String time;
- public String category;
- public String host;
- public int role;
- public String text;
-
- public RecordedEvent() {
- }
-
- /**
- * Create an event. The timestamp is also converted to a time string
- * @param id id counter
- * @param name event name
- * @param timestamp timestamp. If non-zero, is used to build the {@code time} text field.
- * @param category even category
- * @param text arbitrary text
- */
- public RecordedEvent(long id, String name, long timestamp, String category, String text) {
- this.id = id;
- this.name = name;
- this.timestamp = timestamp;
- this.time = timestamp > 0 ? dateFormat.format(timestamp) : "";
- this.category = category;
- this.text = text;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java
index 00910a4..70c2f05 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java
@@ -59,7 +59,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
-import static org.apache.slider.api.proto.RestTypeMarshalling.marshall;
+import static org.apache.slider.api.types.RestTypeMarshalling.marshall;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
index cc17cf0..fd60d7d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
@@ -253,7 +253,7 @@ public class NodeInstance {
new StringBuilder(toString());
sb.append("{ ");
for (NodeEntry entry : nodeEntries) {
- sb.append(String.format("\n [%02d] ", entry.rolePriority));
+ sb.append(String.format("%n [%02d] ", entry.rolePriority));
sb.append(entry.toString());
}
sb.append("} ");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
index c16aa3c..dbdf8ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
@@ -232,7 +232,7 @@ public class OutstandingRequestTracker {
* the most recent one is picked first. This operation <i>does not</i>
* change the role history, though it queries it.
*/
- static class newerThan implements Comparator<Container>, Serializable {
+ static class newerThan implements Comparator<Container> {
private RoleHistory rh;
public newerThan(RoleHistory rh) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index 0a3a3c9..98557ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -321,10 +321,6 @@ public final class RoleStatus implements Cloneable, MetricSet {
return completed.get();
}
- public synchronized void setCompleted(int completed) {
- this.completed.set(completed);
- }
-
public long incCompleted() {
return completed.incrementAndGet();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilter.java
index d5b6b36..b4a92ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilter.java
@@ -69,7 +69,6 @@ public class InsecureAmFilter extends AmIpFilter {
FilterChain chain) throws IOException, ServletException {
rejectNonHttpRequests(req);
HttpServletRequest httpReq = (HttpServletRequest) req;
- HttpServletResponse httpResp = (HttpServletResponse) resp;
String requestURI = httpReq.getRequestURI();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
index c727581..b67f069 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
@@ -48,6 +48,7 @@ import java.net.URLClassLoader;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -122,9 +123,9 @@ public class PublisherResource extends AbstractSliderResource {
@GET
@Path(CLASSPATH)
@Produces({MediaType.APPLICATION_JSON})
- public Set<URL> getAMClassPath() {
+ public List<URL> getAMClassPath() {
URL[] urls = ((URLClassLoader) getClass().getClassLoader()).getURLs();
- return new LinkedHashSet<URL>(Arrays.asList(urls));
+ return Arrays.asList(urls);
}
@GET
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
index 56285c2..8b7d695 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
@@ -32,6 +32,7 @@ import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.web.WebAppApi;
+import javax.annotation.Nonnull;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
@@ -163,7 +164,7 @@ public class ContainerStatsBlock extends SliderHamletBlock {
protected static <T> Function<Entry<String,T>,Entry<TableContent,T>> toTableContentFunction() {
return new Function<Entry<String,T>,Entry<TableContent,T>>() {
@Override
- public Entry<TableContent,T> apply(Entry<String,T> input) {
+ public Entry<TableContent,T> apply(@Nonnull Entry<String,T> input) {
return Maps.immutableEntry(new TableContent(input.getKey()), input.getValue());
}
};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeStatus.java
index 653f479..b4deabc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/ProbeStatus.java
@@ -25,6 +25,7 @@ import java.util.Date;
* Had better be unserializable at the far end if that is to work.
*/
public final class ProbeStatus implements Serializable {
+ private static final long serialVersionUID = 165468L;
private long timestamp;
private String timestampText;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
index 1622309..43f0e4e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
@@ -41,7 +41,6 @@ public abstract class AbstractSliderLaunchedService extends
protected AbstractSliderLaunchedService(String name) {
super(name);
// make sure all the yarn configs get loaded
- YarnConfiguration conf = new YarnConfiguration();
ConfigHelper.registerDeprecatedConfigItems();
}
@@ -74,7 +73,6 @@ public abstract class AbstractSliderLaunchedService extends
throws BadConfigException {
// push back the slider registry entry if needed
- String quorum = lookupZKQuorum();
RegistryOperations registryWriterService =
createRegistryOperationsInstance();
deployChildService(registryWriterService);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
index 9e9e7ac..90a8d40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
@@ -27,6 +27,7 @@ import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedList;
@@ -527,9 +528,9 @@ public class LongLivedProcess implements Runnable {
StringBuilder errorLine = new StringBuilder(LINE_LENGTH);
try {
errReader = new BufferedReader(
- new InputStreamReader(process.getErrorStream()));
+ new InputStreamReader(process.getErrorStream(), "UTF-8"));
outReader = new BufferedReader(
- new InputStreamReader(process.getInputStream()));
+ new InputStreamReader(process.getInputStream(), "UTF-8"));
while (!finished.get()) {
boolean processed = false;
if (readAnyLine(errReader, errorLine, LINE_LENGTH)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
index 20e72c0..0df6047 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
@@ -153,7 +153,7 @@ public class TestSliderUtils {
@Test
public void testWrite() throws IOException {
File testWriteFile = folder.newFile("testWrite");
- SliderUtils.write(testWriteFile, "test".getBytes("UTF-8"), true);
+ SliderUtils.write(testWriteFile, "test".getBytes("UTF-8"));
Assert.assertTrue(FileUtils.readFileToString(testWriteFile, "UTF-8").equals("test"));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562c8263/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/MiniZooKeeperCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/MiniZooKeeperCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/MiniZooKeeperCluster.java
deleted file mode 100644
index d739324..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/MiniZooKeeperCluster.java
+++ /dev/null
@@ -1,395 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.test;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.zookeeper.server.NIOServerCnxnFactory;
-import org.apache.zookeeper.server.ZooKeeperServer;
-import org.apache.zookeeper.server.persistence.FileTxnLog;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.io.Reader;
-import java.net.BindException;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-
-/**
- * This is a version of the HBase ZK cluster cut out to be standalone
- */
-public class MiniZooKeeperCluster {
- private static final Logger LOG = LoggerFactory.getLogger(
- MiniZooKeeperCluster.class);
-
- private static final int TICK_TIME = 2000;
- private static final int CONNECTION_TIMEOUT = 30000;
- public static final int MAX_CLIENT_CONNECTIONS = 1000;
-
- private boolean started;
-
- /** The default port. If zero, we use a random port. */
- private int defaultClientPort = 0;
-
- private int clientPort;
-
- private List<NIOServerCnxnFactory> standaloneServerFactoryList;
- private List<ZooKeeperServer> zooKeeperServers;
- private List<Integer> clientPortList;
-
- private int activeZKServerIndex;
- private int tickTime = 0;
-
- private Configuration configuration;
-
- public MiniZooKeeperCluster() {
- this(new Configuration());
- }
-
- public MiniZooKeeperCluster(Configuration configuration) {
- this.started = false;
- this.configuration = configuration;
- activeZKServerIndex = -1;
- zooKeeperServers = new ArrayList<ZooKeeperServer>();
- clientPortList = new ArrayList<Integer>();
- standaloneServerFactoryList = new ArrayList<NIOServerCnxnFactory>();
- }
-
- public void setDefaultClientPort(int clientPort) {
- if (clientPort <= 0) {
- throw new IllegalArgumentException("Invalid default ZK client port: "
- + clientPort);
- }
- this.defaultClientPort = clientPort;
- }
-
- /**
- * Selects a ZK client port. Returns the default port if specified.
- * Otherwise, returns a random port. The random port is selected from the
- * range between 49152 to 65535. These ports cannot be registered with IANA
- * and are intended for dynamic allocation (see http://bit.ly/dynports).
- */
- private int selectClientPort() {
- if (defaultClientPort > 0) {
- return defaultClientPort;
- }
- return 0xc000 + new Random().nextInt(0x3f00);
- }
-
- public void setTickTime(int tickTime) {
- this.tickTime = tickTime;
- }
-
- public int getBackupZooKeeperServerNum() {
- return zooKeeperServers.size() - 1;
- }
-
- public int getZooKeeperServerNum() {
- return zooKeeperServers.size();
- }
-
- // / XXX: From o.a.zk.t.ClientBase
- private static void setupTestEnv() {
- // during the tests we run with 100K prealloc in the logs.
- // on windows systems prealloc of 64M was seen to take ~15seconds
- // resulting in test failure (client timeout on first session).
- // set env and directly in order to handle static init/gc issues
- System.setProperty("zookeeper.preAllocSize", "100");
- FileTxnLog.setPreallocSize(100 * 1024);
- }
-
- public int startup(File baseDir) throws IOException, InterruptedException {
- return startup(baseDir, 1);
- }
-
- /**
- * @param baseDir
- * @param numZooKeeperServers
- * @return ClientPort server bound to, -1 if there was a
- * binding problem and we couldn't pick another port.
- * @throws IOException
- * @throws InterruptedException
- */
- public int startup(File baseDir, int numZooKeeperServers) throws IOException,
- InterruptedException {
- if (numZooKeeperServers <= 0)
- return -1;
-
- setupTestEnv();
- shutdown();
-
- int tentativePort = selectClientPort();
-
- // running all the ZK servers
- for (int i = 0; i < numZooKeeperServers; i++) {
- File dir = new File(baseDir, "zookeeper_" + i).getAbsoluteFile();
- recreateDir(dir);
- int tickTimeToUse;
- if (this.tickTime > 0) {
- tickTimeToUse = this.tickTime;
- } else {
- tickTimeToUse = TICK_TIME;
- }
- ZooKeeperServer server = new ZooKeeperServer(dir, dir, tickTimeToUse);
- NIOServerCnxnFactory standaloneServerFactory;
- while (true) {
- try {
- standaloneServerFactory = new NIOServerCnxnFactory();
- standaloneServerFactory.configure(
- new InetSocketAddress(tentativePort),
- MAX_CLIENT_CONNECTIONS
- );
- } catch (BindException e) {
- LOG.debug("Failed binding ZK Server to client port: " +
- tentativePort, e);
- // We're told to use some port but it's occupied, fail
- if (defaultClientPort > 0) return -1;
- // This port is already in use, try to use another.
- tentativePort = selectClientPort();
- continue;
- }
- break;
- }
-
- // Start up this ZK server
- standaloneServerFactory.startup(server);
- if (!waitForServerUp(tentativePort, CONNECTION_TIMEOUT)) {
- throw new IOException("Waiting for startup of standalone server");
- }
-
- // We have selected this port as a client port.
- clientPortList.add(tentativePort);
- standaloneServerFactoryList.add(standaloneServerFactory);
- zooKeeperServers.add(server);
- tentativePort++; //for the next server
- }
-
- // set the first one to be active ZK; Others are backups
- activeZKServerIndex = 0;
- started = true;
- clientPort = clientPortList.get(activeZKServerIndex);
- LOG.info("Started MiniZK Cluster and connect 1 ZK server " +
- "on client port: " + clientPort);
- return clientPort;
- }
-
- private void recreateDir(File dir) throws IOException {
- if (dir.exists()) {
- if (!FileUtil.fullyDelete(dir)) {
- throw new IOException("Could not delete zk base directory: " + dir);
- }
- }
- try {
- dir.mkdirs();
- } catch (SecurityException e) {
- throw new IOException("creating dir: " + dir, e);
- }
- }
-
- /**
- * @throws IOException
- */
- public void shutdown() throws IOException {
- if (!started) {
- return;
- }
-
- // shut down all the zk servers
- for (int i = 0; i < standaloneServerFactoryList.size(); i++) {
- NIOServerCnxnFactory standaloneServerFactory =
- standaloneServerFactoryList.get(i);
- int clientPort = clientPortList.get(i);
-
- standaloneServerFactory.shutdown();
- if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
- throw new IOException("Waiting for shutdown of standalone server");
- }
- }
- for (ZooKeeperServer zkServer : zooKeeperServers) {
- //explicitly close ZKDatabase since ZookeeperServer does not close them
- zkServer.getZKDatabase().close();
- }
-
- // clear everything
- started = false;
- activeZKServerIndex = 0;
- standaloneServerFactoryList.clear();
- clientPortList.clear();
- zooKeeperServers.clear();
-
- LOG.info("Shutdown MiniZK cluster with all ZK servers");
- }
-
- /**@return clientPort return clientPort if there is another ZK backup can run
- * when killing the current active; return -1, if there is no backups.
- * @throws IOException
- * @throws InterruptedException
- */
- public int killCurrentActiveZooKeeperServer() throws IOException,
- InterruptedException {
- if (!started || activeZKServerIndex < 0) {
- return -1;
- }
-
- // Shutdown the current active one
- NIOServerCnxnFactory standaloneServerFactory =
- standaloneServerFactoryList.get(activeZKServerIndex);
- int clientPort = clientPortList.get(activeZKServerIndex);
-
- standaloneServerFactory.shutdown();
- if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
- throw new IOException("Waiting for shutdown of standalone server");
- }
-
- zooKeeperServers.get(activeZKServerIndex).getZKDatabase().close();
-
- // remove the current active zk server
- standaloneServerFactoryList.remove(activeZKServerIndex);
- clientPortList.remove(activeZKServerIndex);
- zooKeeperServers.remove(activeZKServerIndex);
- LOG.info("Kill the current active ZK servers in the cluster " +
- "on client port: " + clientPort);
-
- if (standaloneServerFactoryList.size() == 0) {
- // there is no backup servers;
- return -1;
- }
- clientPort = clientPortList.get(activeZKServerIndex);
- LOG.info("Activate a backup zk server in the cluster " +
- "on client port: " + clientPort);
- // return the next back zk server's port
- return clientPort;
- }
-
- /**
- * Kill one back up ZK servers
- * @throws IOException
- * @throws InterruptedException
- */
- public void killOneBackupZooKeeperServer() throws IOException,
- InterruptedException {
- if (!started || activeZKServerIndex < 0 ||
- standaloneServerFactoryList.size() <= 1) {
- return;
- }
-
- int backupZKServerIndex = activeZKServerIndex + 1;
- // Shutdown the current active one
- NIOServerCnxnFactory standaloneServerFactory =
- standaloneServerFactoryList.get(backupZKServerIndex);
- int clientPort = clientPortList.get(backupZKServerIndex);
-
- standaloneServerFactory.shutdown();
- if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
- throw new IOException("Waiting for shutdown of standalone server");
- }
-
- zooKeeperServers.get(backupZKServerIndex).getZKDatabase().close();
-
- // remove this backup zk server
- standaloneServerFactoryList.remove(backupZKServerIndex);
- clientPortList.remove(backupZKServerIndex);
- zooKeeperServers.remove(backupZKServerIndex);
- LOG.info("Kill one backup ZK servers in the cluster " +
- "on client port: " + clientPort);
- }
-
- // XXX: From o.a.zk.t.ClientBase
- private static boolean waitForServerDown(int port, long timeout) {
- long start = System.currentTimeMillis();
- while (true) {
- try {
- Socket sock = new Socket("localhost", port);
- try {
- OutputStream outstream = sock.getOutputStream();
- outstream.write("stat".getBytes());
- outstream.flush();
- } finally {
- sock.close();
- }
- } catch (IOException e) {
- return true;
- }
-
- if (System.currentTimeMillis() > start + timeout) {
- break;
- }
- try {
- Thread.sleep(250);
- } catch (InterruptedException e) {
- // ignore
- }
- }
- return false;
- }
-
- // XXX: From o.a.zk.t.ClientBase
- private static boolean waitForServerUp(int port, long timeout) {
- long start = System.currentTimeMillis();
- while (true) {
- try {
- Socket sock = new Socket("localhost", port);
- BufferedReader reader = null;
- try {
- OutputStream outstream = sock.getOutputStream();
- outstream.write("stat".getBytes());
- outstream.flush();
-
- Reader isr = new InputStreamReader(sock.getInputStream());
- reader = new BufferedReader(isr);
- String line = reader.readLine();
- if (line != null && line.startsWith("Zookeeper version:")) {
- return true;
- }
- } finally {
- sock.close();
- if (reader != null) {
- reader.close();
- }
- }
- } catch (IOException e) {
- // ignore as this is expected
- LOG.info("server localhost:" + port + " not up " + e);
- }
-
- if (System.currentTimeMillis() > start + timeout) {
- break;
- }
- try {
- Thread.sleep(250);
- } catch (InterruptedException e) {
- // ignore
- }
- }
- return false;
- }
-
- public int getClientPort() {
- return clientPort;
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/51] [abbrv] hadoop git commit: YARN-5975. Remove the agent -
slider AM ssl related code. Contributed by Jian He
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
deleted file mode 100644
index e82ad84..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
+++ /dev/null
@@ -1,256 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.services.security;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.SliderXmlConfKeys;
-import org.apache.slider.core.conf.MapOperations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-//import java.nio.file.Files;
-//import java.nio.file.Path;
-//import java.nio.file.Paths;
-//import java.nio.file.attribute.PosixFilePermission;
-//import java.nio.file.attribute.PosixFilePermissions;
-
-
-/**
- *
- */
-public class SecurityUtils {
- private static final Logger LOG =
- LoggerFactory.getLogger(SecurityUtils.class);
-
- private static String CA_CONFIG_CONTENTS = "HOME = .\n"
- + "RANDFILE = $ENV::HOME/.rnd\n\n"
- + "[ ca ]\n"
- + "default_ca = CA_CLIENT\n"
- + "[ CA_CLIENT ]\n"
- + "dir = ${SEC_DIR}/db\n"
- + "certs = $dir/certs\n"
- + "new_certs_dir = $dir/newcerts\n"
- + "\n"
- + "database = $dir/index.txt\n"
- + "serial = $dir/serial\n"
- + "default_days = 365 \n"
- + "\n"
- + "default_crl_days = 7 \n"
- + "default_md = sha256 \n"
- + "\n"
- + "policy = policy_anything \n"
- + "\n"
- + "[ policy_anything ]\n"
- + "countryName = optional\n"
- + "stateOrProvinceName = optional\n"
- + "localityName = optional\n"
- + "organizationName = optional\n"
- + "organizationalUnitName = optional\n"
- + "commonName = optional\n"
- + "emailAddress = optional\n"
- + "\n"
- + "[req]\n"
- + "distinguished_name = req_distinguished_name\n"
- + "\n"
- + "[ req_distinguished_name ]\n"
- + "\n"
- + "[ jdk7_ca ]\n"
- + "subjectKeyIdentifier = hash\n"
- + "authorityKeyIdentifier = keyid:always,issuer:always\n"
- + "basicConstraints = CA:true\n";
-
- private static final String PASS_TOKEN = "pass:";
- private static String keystorePass;
- private static String securityDir;
-
- public static void logOpenSslExitCode(String command, int exitCode) {
- if (exitCode == 0) {
- LOG.info(getOpenSslCommandResult(command, exitCode));
- } else {
- LOG.warn(getOpenSslCommandResult(command, exitCode));
- }
-
- }
-
- public static String hideOpenSslPassword(String command){
- int start = command.indexOf(PASS_TOKEN);
- while (start >= 0) {
- start += PASS_TOKEN.length();
- CharSequence cs = command.subSequence(start, command.indexOf(" ", start));
- command = command.replace(cs, "****");
- start = command.indexOf(PASS_TOKEN, start + 1);
- }
- return command;
- }
-
- public static String getOpenSslCommandResult(String command, int exitCode) {
- return new StringBuilder().append("Command ")
- .append(hideOpenSslPassword(command))
- .append(" was finished with exit code: ")
- .append(exitCode).append(" - ")
- .append(getOpenSslExitCodeDescription(exitCode)).toString();
- }
-
- private static String getOpenSslExitCodeDescription(int exitCode) {
- switch (exitCode) {
- case 0: {
- return "the operation was completed successfully.";
- }
- case 1: {
- return "an error occurred parsing the command options.";
- }
- case 2: {
- return "one of the input files could not be read.";
- }
- case 3: {
- return "an error occurred creating the PKCS#7 file or when reading the MIME message.";
- }
- case 4: {
- return "an error occurred decrypting or verifying the message.";
- }
- case 5: {
- return "the message was verified correctly but an error occurred writing out the signers certificates.";
- }
- default:
- return "unsupported code";
- }
- }
-
- public static void writeCaConfigFile(String path) throws IOException {
- String contents = CA_CONFIG_CONTENTS.replace("${SEC_DIR}", path);
- FileUtils.writeStringToFile(new File(path, "ca.config"), contents);
- }
-
- public static String getKeystorePass() {
- return keystorePass;
- }
-
- public static String getSecurityDir() {
- return securityDir;
- }
-
- public static void initializeSecurityParameters(MapOperations configMap) {
- initializeSecurityParameters(configMap, false);
- }
-
- public static void initializeSecurityParameters(MapOperations configMap,
- boolean persistPassword) {
- String keyStoreLocation = configMap.getOption(
- SliderXmlConfKeys.KEY_KEYSTORE_LOCATION, getDefaultKeystoreLocation());
- if (keyStoreLocation == null) {
- LOG.error(SliderXmlConfKeys.KEY_KEYSTORE_LOCATION
- + " is not specified. Unable to initialize security params.");
- return;
- }
- File secDirFile = new File(keyStoreLocation).getParentFile();
- if (!secDirFile.exists()) {
- // create entire required directory structure
- File dbDir = new File(secDirFile, "db");
- File newCertsDir = new File(dbDir, "newcerts");
- newCertsDir.mkdirs();
- RawLocalFileSystem fileSystem = null;
- try {
- fileSystem = new RawLocalFileSystem();
- FsPermission permissions = new FsPermission(FsAction.ALL, FsAction.NONE,
- FsAction.NONE);
- fileSystem.setPermission(new Path(dbDir.getAbsolutePath()),
- permissions);
- fileSystem.setPermission(new Path(dbDir.getAbsolutePath()), permissions);
- fileSystem.setPermission(new Path(newCertsDir.getAbsolutePath()),
- permissions);
- File indexFile = new File(dbDir, "index.txt");
- indexFile.createNewFile();
- SecurityUtils.writeCaConfigFile(secDirFile.getAbsolutePath().replace('\\', '/'));
-
- } catch (IOException e) {
- LOG.error("Unable to create SSL configuration directories/files", e);
- } finally {
- if (fileSystem != null) {
- try {
- fileSystem.close();
- } catch (IOException e) {
- LOG.warn("Unable to close fileSystem", e);
- }
- }
- }
- // need to create the password
- }
- keystorePass = getKeystorePassword(secDirFile, persistPassword);
- securityDir = secDirFile.getAbsolutePath();
- }
-
- private static String getKeystorePassword(File secDirFile,
- boolean persistPassword) {
- File passFile = new File(secDirFile, SliderKeys.CRT_PASS_FILE_NAME);
- String password = null;
- if (!passFile.exists()) {
- LOG.info("Generating keystore password");
- password = RandomStringUtils.randomAlphanumeric(
- Integer.valueOf(SliderKeys.PASS_LEN));
- if (persistPassword) {
- try {
- FileUtils.writeStringToFile(passFile, password);
- passFile.setWritable(true);
- passFile.setReadable(true);
- } catch (IOException e) {
- e.printStackTrace();
- throw new RuntimeException(
- "Error creating certificate password file");
- }
- }
- } else {
- LOG.info("Reading password from existing file");
- try {
- password = FileUtils.readFileToString(passFile);
- password = password.replaceAll("\\p{Cntrl}", "");
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- return password;
- }
-
- private static String getDefaultKeystoreLocation() {
- File workDir = null;
- try {
- workDir = new File(FileUtils.getTempDirectory().getAbsolutePath()
- + "/sec" + System.currentTimeMillis());
- if (!workDir.mkdirs()) {
- throw new IOException("Unable to create temporary security directory");
- }
- } catch (IOException e) {
- LOG.warn("Unable to create security directory");
- return null;
- }
-
- return new StringBuilder().append(workDir.getAbsolutePath())
- .append(File.separator)
- .append(SliderKeys.SECURITY_DIR)
- .append(File.separator)
- .append(SliderKeys.KEYSTORE_FILE_NAME).toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignCertResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignCertResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignCertResponse.java
deleted file mode 100644
index 8437d88..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignCertResponse.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.security;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-import javax.xml.bind.annotation.XmlType;
-
-/**
- *
- * Sign certificate response data model.
- *
- */
-@XmlRootElement
-@XmlAccessorType(XmlAccessType.FIELD)
-@XmlType(name = "", propOrder = {})
-public class SignCertResponse {
-
- public static final String ERROR_STATUS = "ERROR";
- public static final String OK_STATUS = "OK";
-
- @XmlElement
- private String result;
- @XmlElement
- private String signedCa;
- @XmlElement
- private String message;
-
- public String getResult() {
- return result;
- }
- public void setResult(String result) {
- this.result = result;
- }
- public String getSignedCa() {
- return signedCa;
- }
- public void setSignedCa(String signedCa) {
- this.signedCa = signedCa;
- }
-
- public String getMessage() {
- return message;
- }
- public void setMessage(String message) {
- this.message = message;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignMessage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignMessage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignMessage.java
deleted file mode 100644
index 4bccb87..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignMessage.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.security;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-import javax.xml.bind.annotation.XmlType;
-
-/**
- *
- * Sign certificate request data model.
- *
- */
-@XmlRootElement
-@XmlAccessorType(XmlAccessType.FIELD)
-@XmlType(name = "", propOrder = {})
-public class SignMessage {
-
- @XmlElement
- private String csr;
- @XmlElement
- private String passphrase;
- public String getCsr() {
- return csr;
- }
- public void setCsr(String csr) {
- this.csr = csr;
- }
- public String getPassphrase() {
- return passphrase;
- }
- public void setPassphrase(String passphrase) {
- this.passphrase = passphrase;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/StoresGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/StoresGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/StoresGenerator.java
deleted file mode 100644
index 226250f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/StoresGenerator.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.services.security;
-
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.MapOperations;
-import org.apache.slider.core.exceptions.SliderException;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- *
- */
-public class StoresGenerator {
-
- static CertificateManager certMgr = new CertificateManager();
- private static SecurityStoreGenerator[] GENERATORS = {
- new KeystoreGenerator(certMgr), new TruststoreGenerator(certMgr)
- };
-
- public static SecurityStore[] generateSecurityStores(String hostname,
- String containerId,
- String role,
- AggregateConf instanceDefinition,
- MapOperations compOps)
- throws SliderException, IOException {
- //discover which stores need generation based on the passwords configured
- List<SecurityStore> files = new ArrayList<SecurityStore>();
- for (SecurityStoreGenerator generator : GENERATORS) {
- if (generator.isStoreRequested(compOps)) {
- SecurityStore store = generator.generate(hostname,
- containerId,
- instanceDefinition,
- compOps,
- role);
- if (store != null) {
- files.add(store);
- }
- }
- }
-
- if (files.isEmpty()) {
- throw new SliderException("Security stores were requested but none were "
- + "generated. Check the AM logs and ensure "
- + "passwords are configured for the components "
- + "requiring the stores.");
- }
- return files.toArray(new SecurityStore[files.size()]);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/TruststoreGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/TruststoreGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/TruststoreGenerator.java
deleted file mode 100644
index d16dcbd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/TruststoreGenerator.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.services.security;
-
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.MapOperations;
-import org.apache.slider.core.exceptions.SliderException;
-
-import java.io.IOException;
-
-/**
- *
- */
-public class TruststoreGenerator extends AbstractSecurityStoreGenerator {
-
-
- public TruststoreGenerator(CertificateManager certificateMgr) {
- super(certificateMgr);
- }
-
- @Override
- public SecurityStore generate(String hostname, String containerId,
- AggregateConf instanceDefinition,
- MapOperations compOps, String role)
- throws SliderException, IOException {
- SecurityStore truststore = null;
- String password = getStorePassword(
- instanceDefinition.getAppConf().credentials, compOps, role);
- if (password != null) {
- truststore = certificateMgr.generateContainerTruststore(containerId,
- role, password);
- }
- return truststore;
- }
-
- @Override
- String getPassword(MapOperations compOps) {
- return compOps.get(
- compOps.get(SliderKeys.COMP_TRUSTSTORE_PASSWORD_PROPERTY_KEY));
- }
-
- @Override
- String getAlias(MapOperations compOps) {
- return compOps.getOption(SliderKeys.COMP_TRUSTSTORE_PASSWORD_ALIAS_KEY,
- SliderKeys.COMP_TRUSTSTORE_PASSWORD_ALIAS_DEFAULT);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto
index 4221b1d..d68da2b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto
@@ -175,10 +175,4 @@ service SliderClusterProtocolPB {
// ConfTree getLiveResources
rpc getLiveResources(EmptyPayloadProto)
returns(WrappedJsonProto);
-
- rpc getClientCertificateStore(GetCertificateStoreRequestProto)
- returns(GetCertificateStoreResponseProto);
-
-
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java
deleted file mode 100644
index 7a4a586..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java
+++ /dev/null
@@ -1,540 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.services.security;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.alias.CredentialProvider;
-import org.apache.hadoop.security.alias.CredentialProviderFactory;
-import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
-import org.apache.slider.Slider;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.SliderXmlConfKeys;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.MapOperations;
-import org.apache.slider.core.exceptions.SliderException;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.TrustManagerFactory;
-import javax.net.ssl.X509TrustManager;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.InetAddress;
-import java.security.KeyStore;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
-import java.security.Principal;
-import java.security.cert.Certificate;
-import java.security.cert.CertificateException;
-import java.security.cert.X509Certificate;
-import java.util.ArrayList;
-import java.util.Arrays;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- *
- */
-public class TestCertificateManager {
- @Rule
- public TemporaryFolder workDir = new TemporaryFolder();
- private File secDir;
- private CertificateManager certMan;
-
- @Before
- public void setup() throws Exception {
- certMan = new CertificateManager();
- MapOperations compOperations = new MapOperations();
- secDir = new File(workDir.getRoot(), SliderKeys.SECURITY_DIR);
- File keystoreFile = new File(secDir, SliderKeys.KEYSTORE_FILE_NAME);
- compOperations.put(SliderXmlConfKeys.KEY_KEYSTORE_LOCATION,
- keystoreFile.getAbsolutePath());
- certMan.initialize(compOperations, "cahost", null, null);
- }
-
- @Test
- public void testServerCertificateGenerated() throws Exception {
- File serverCrt = new File(secDir, SliderKeys.CRT_FILE_NAME);
- Assert.assertTrue("Server CRD does not exist:" + serverCrt,
- serverCrt.exists());
- }
-
- @Test
- public void testAMKeystoreGenerated() throws Exception {
- File keystoreFile = new File(secDir, SliderKeys.KEYSTORE_FILE_NAME);
- Assert.assertTrue("Keystore does not exist: " + keystoreFile,
- keystoreFile.exists());
- InputStream is = null;
- try {
-
- is = new FileInputStream(keystoreFile);
- KeyStore keystore = KeyStore.getInstance("pkcs12");
- String password = SecurityUtils.getKeystorePass();
- keystore.load(is, password.toCharArray());
-
- Certificate certificate = keystore.getCertificate(
- keystore.aliases().nextElement());
- Assert.assertNotNull(certificate);
-
- if (certificate instanceof X509Certificate) {
- X509Certificate x509cert = (X509Certificate) certificate;
-
- // Get subject
- Principal principal = x509cert.getSubjectDN();
- String subjectDn = principal.getName();
- Assert.assertEquals("wrong DN",
- "CN=cahost",
- subjectDn);
-
- // Get issuer
- principal = x509cert.getIssuerDN();
- String issuerDn = principal.getName();
- Assert.assertEquals("wrong Issuer DN",
- "CN=cahost",
- issuerDn);
- }
- } finally {
- if(null != is) {
- is.close();
- }
- }
- }
-
- @Test
- public void testContainerCertificateGeneration() throws Exception {
- certMan.generateContainerCertificate("testhost", "container1");
- Assert.assertTrue("container certificate not generated",
- new File(secDir, "container1.crt").exists());
- }
-
- @Test
- public void testContainerKeystoreGeneration() throws Exception {
- SecurityStore keystoreFile = certMan.generateContainerKeystore("testhost",
- "container1",
- "component1",
- "password");
- validateKeystore(keystoreFile.getFile(), "testhost", "cahost");
- }
-
- private void validateKeystore(File keystoreFile, String certHostname,
- String issuerHostname)
- throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException {
- Assert.assertTrue("container keystore not generated",
- keystoreFile.exists());
-
- InputStream is = null;
- try {
-
- is = new FileInputStream(keystoreFile);
- KeyStore keystore = KeyStore.getInstance("pkcs12");
- String password = "password";
- keystore.load(is, password.toCharArray());
-
- Certificate certificate = keystore.getCertificate(
- keystore.aliases().nextElement());
- Assert.assertNotNull(certificate);
-
- if (certificate instanceof X509Certificate) {
- X509Certificate x509cert = (X509Certificate) certificate;
-
- // Get subject
- Principal principal = x509cert.getSubjectDN();
- String subjectDn = principal.getName();
- Assert.assertEquals("wrong DN", "CN=" + certHostname + ", OU=container1",
- subjectDn);
-
- // Get issuer
- principal = x509cert.getIssuerDN();
- String issuerDn = principal.getName();
- Assert.assertEquals("wrong Issuer DN",
- "CN=" + issuerHostname,
- issuerDn);
- }
- } finally {
- if(null != is) {
- is.close();
- }
- }
- }
-
- @Test
- public void testContainerKeystoreGenerationViaStoresGenerator() throws Exception {
- AggregateConf instanceDefinition = new AggregateConf();
- MapOperations compOps = new MapOperations();
- instanceDefinition.getAppConf().components.put("component1", compOps);
- compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_PROPERTY_KEY,
- "app1.component1.password.property");
- compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true");
- instanceDefinition.getAppConf().global.put(
- "app1.component1.password.property", "password");
- instanceDefinition.resolve();
- SecurityStore[]
- files = StoresGenerator.generateSecurityStores("testhost",
- "container1",
- "component1",
- instanceDefinition,
- compOps);
- assertEquals("wrong number of stores", 1, files.length);
- validateKeystore(files[0].getFile(), "testhost", "cahost");
- }
-
- @Test
- public void testContainerKeystoreGenerationViaStoresGeneratorUsingGlobalProps() throws Exception {
- AggregateConf instanceDefinition = new AggregateConf();
- MapOperations compOps = new MapOperations();
- instanceDefinition.getAppConf().components.put("component1", compOps);
- compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_PROPERTY_KEY,
- "app1.component1.password.property");
- instanceDefinition.getAppConf().global.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true");
- compOps.put(
- "app1.component1.password.property", "password");
- instanceDefinition.resolve();
- SecurityStore[]
- files = StoresGenerator.generateSecurityStores("testhost",
- "container1",
- "component1",
- instanceDefinition,
- compOps);
- assertEquals("wrong number of stores", 1, files.length);
- validateKeystore(files[0].getFile(), "testhost", "cahost");
- }
-
- @Test
- public void testContainerKeystoreGenerationViaStoresGeneratorOverrideGlobalSetting() throws Exception {
- AggregateConf instanceDefinition = new AggregateConf();
- MapOperations compOps = setupComponentOptions(true, null,
- "app1.component1.password.property",
- null, null);
- instanceDefinition.getAppConf().components.put("component1", compOps);
- instanceDefinition.getAppConf().global.put(
- "app1.component1.password.property", "password");
- instanceDefinition.getAppConf().global.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "false");
- instanceDefinition.resolve();
- SecurityStore[]
- files = StoresGenerator.generateSecurityStores("testhost",
- "container1",
- "component1",
- instanceDefinition,
- compOps);
- assertEquals("wrong number of stores", 1, files.length);
- validateKeystore(files[0].getFile(), "testhost", "cahost");
- }
-
- @Test
- public void testContainerTrusttoreGeneration() throws Exception {
- SecurityStore keystoreFile =
- certMan.generateContainerKeystore("testhost",
- "container1",
- "component1",
- "keypass");
- Assert.assertTrue("container keystore not generated",
- keystoreFile.getFile().exists());
- SecurityStore truststoreFile =
- certMan.generateContainerTruststore("container1",
- "component1", "trustpass"
- );
- Assert.assertTrue("container truststore not generated",
- truststoreFile.getFile().exists());
-
- validateTruststore(keystoreFile.getFile(), truststoreFile.getFile());
- }
-
- @Test
- public void testContainerGenerationUsingStoresGeneratorNoTruststore() throws Exception {
- AggregateConf instanceDefinition = new AggregateConf();
- MapOperations compOps = new MapOperations();
- compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true");
- compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_KEY,
- "test.keystore.password");
-
- setupCredentials(instanceDefinition, "test.keystore.password", null);
-
- SecurityStore[]
- files = StoresGenerator.generateSecurityStores("testhost",
- "container1",
- "component1",
- instanceDefinition,
- compOps);
- assertEquals("wrong number of stores", 1, files.length);
- File keystoreFile = CertificateManager.getContainerKeystoreFilePath(
- "container1", "component1");
- Assert.assertTrue("container keystore not generated",
- keystoreFile.exists());
-
- Assert.assertTrue("keystore not in returned list",
- Arrays.asList(files).contains(new SecurityStore(keystoreFile,
- SecurityStore.StoreType.keystore)));
- File truststoreFile =
- CertificateManager.getContainerTruststoreFilePath("component1",
- "container1");
- Assert.assertFalse("container truststore generated",
- truststoreFile.exists());
- Assert.assertFalse("truststore in returned list",
- Arrays.asList(files).contains(new SecurityStore(truststoreFile,
- SecurityStore.StoreType.truststore)));
-
- }
-
- @Test
- public void testContainerGenerationUsingStoresGeneratorJustTruststoreWithDefaultAlias() throws Exception {
- AggregateConf instanceDefinition = new AggregateConf();
- MapOperations compOps = setupComponentOptions(true);
-
- setupCredentials(instanceDefinition, null,
- SliderKeys.COMP_TRUSTSTORE_PASSWORD_ALIAS_DEFAULT);
-
- SecurityStore[]
- files = StoresGenerator.generateSecurityStores("testhost",
- "container1",
- "component1",
- instanceDefinition,
- compOps);
- assertEquals("wrong number of stores", 1, files.length);
- File keystoreFile = CertificateManager.getContainerKeystoreFilePath(
- "container1", "component1");
- Assert.assertFalse("container keystore generated",
- keystoreFile.exists());
- Assert.assertFalse("keystore in returned list",
- Arrays.asList(files).contains(keystoreFile));
- File truststoreFile =
- CertificateManager.getContainerTruststoreFilePath("component1",
- "container1");
- Assert.assertTrue("container truststore not generated",
- truststoreFile.exists());
- Assert.assertTrue("truststore not in returned list",
- Arrays.asList(files).contains(new SecurityStore(truststoreFile,
- SecurityStore.StoreType.truststore)));
-
- }
-
- @Test
- public void testContainerTrusttoreGenerationUsingStoresGenerator() throws Exception {
- AggregateConf instanceDefinition = new AggregateConf();
- MapOperations compOps = setupComponentOptions(true,
- "test.keystore.password",
- null,
- "test.truststore.password",
- null);
-
- setupCredentials(instanceDefinition, "test.keystore.password",
- "test.truststore.password");
-
- SecurityStore[]
- files = StoresGenerator.generateSecurityStores("testhost",
- "container1",
- "component1",
- instanceDefinition,
- compOps);
- assertEquals("wrong number of stores", 2, files.length);
- File keystoreFile = CertificateManager.getContainerKeystoreFilePath(
- "container1", "component1");
- Assert.assertTrue("container keystore not generated",
- keystoreFile.exists());
- Assert.assertTrue("keystore not in returned list",
- Arrays.asList(files).contains(new SecurityStore(keystoreFile,
- SecurityStore.StoreType.keystore)));
- File truststoreFile =
- CertificateManager.getContainerTruststoreFilePath("component1",
- "container1");
- Assert.assertTrue("container truststore not generated",
- truststoreFile.exists());
- Assert.assertTrue("truststore not in returned list",
- Arrays.asList(files).contains(new SecurityStore(truststoreFile,
- SecurityStore.StoreType.truststore)));
-
- validateTruststore(keystoreFile, truststoreFile);
- }
-
- private void setupCredentials(AggregateConf instanceDefinition,
- String keyAlias, String trustAlias)
- throws Exception {
- Configuration conf = new Configuration();
- final Path jksPath = new Path(SecurityUtils.getSecurityDir(), "test.jks");
- final String ourUrl =
- JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
-
- File file = new File(SecurityUtils.getSecurityDir(), "test.jks");
- file.delete();
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl);
-
- instanceDefinition.getAppConf().credentials.put(ourUrl, new ArrayList<String>());
-
- CredentialProvider provider =
- CredentialProviderFactory.getProviders(conf).get(0);
-
- // create new aliases
- try {
-
- if (keyAlias != null) {
- char[] storepass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
- provider.createCredentialEntry(
- keyAlias, storepass);
- }
-
- if (trustAlias != null) {
- char[] trustpass = {'t', 'r', 'u', 's', 't', 'p', 'a', 's', 's'};
- provider.createCredentialEntry(
- trustAlias, trustpass);
- }
-
- // write out so that it can be found in checks
- provider.flush();
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- private MapOperations setupComponentOptions(boolean storesRequired) {
- return this.setupComponentOptions(storesRequired, null, null, null, null);
- }
-
- private MapOperations setupComponentOptions(boolean storesRequired,
- String keyAlias,
- String keyPwd,
- String trustAlias,
- String trustPwd) {
- MapOperations compOps = new MapOperations();
- compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY,
- Boolean.toString(storesRequired));
- if (keyAlias != null) {
- compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_KEY,
- "test.keystore.password");
- }
- if (trustAlias != null) {
- compOps.put(SliderKeys.COMP_TRUSTSTORE_PASSWORD_ALIAS_KEY,
- "test.truststore.password");
- }
- if (keyPwd != null) {
- compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_PROPERTY_KEY,
- keyPwd);
- }
- if (trustPwd != null) {
- compOps.put(SliderKeys.COMP_TRUSTSTORE_PASSWORD_PROPERTY_KEY,
- trustPwd);
- }
- return compOps;
- }
-
- @Test
- public void testContainerStoresGenerationKeystoreOnly() throws Exception {
- AggregateConf instanceDefinition = new AggregateConf();
- MapOperations compOps = new MapOperations();
- compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true");
-
- setupCredentials(instanceDefinition,
- SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_DEFAULT, null);
-
- SecurityStore[]
- files = StoresGenerator.generateSecurityStores("testhost",
- "container1",
- "component1",
- instanceDefinition,
- compOps);
- assertEquals("wrong number of stores", 1, files.length);
- File keystoreFile = CertificateManager.getContainerKeystoreFilePath(
- "container1", "component1");
- Assert.assertTrue("container keystore not generated",
- keystoreFile.exists());
- Assert.assertTrue("keystore not in returned list",
- Arrays.asList(files).contains(new SecurityStore(keystoreFile,
- SecurityStore.StoreType.keystore)));
- File truststoreFile =
- CertificateManager.getContainerTruststoreFilePath("component1",
- "container1");
- Assert.assertFalse("container truststore generated",
- truststoreFile.exists());
- Assert.assertFalse("truststore in returned list",
- Arrays.asList(files).contains(new SecurityStore(truststoreFile,
- SecurityStore.StoreType.truststore)));
-
- }
-
- @Test
- public void testContainerStoresGenerationMisconfiguration() throws Exception {
- AggregateConf instanceDefinition = new AggregateConf();
- MapOperations compOps = new MapOperations();
- compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true");
-
- setupCredentials(instanceDefinition, "cant.be.found", null);
-
- try {
- StoresGenerator.generateSecurityStores("testhost", "container1",
- "component1", instanceDefinition,
- compOps);
- Assert.fail("SliderException should have been generated");
- } catch (SliderException e) {
- // ignore - should be thrown
- }
- }
-
- private void validateTruststore(File keystoreFile, File truststoreFile)
- throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException {
- InputStream keyis = null;
- InputStream trustis = null;
- try {
-
- // create keystore
- keyis = new FileInputStream(keystoreFile);
- KeyStore keystore = KeyStore.getInstance("pkcs12");
- String password = "keypass";
- keystore.load(keyis, password.toCharArray());
-
- // obtain server cert
- Certificate certificate = keystore.getCertificate(
- keystore.aliases().nextElement());
- Assert.assertNotNull(certificate);
-
- // create trust store from generated trust store file
- trustis = new FileInputStream(truststoreFile);
- KeyStore truststore = KeyStore.getInstance("pkcs12");
- password = "trustpass";
- truststore.load(trustis, password.toCharArray());
-
- // validate keystore cert using trust store
- TrustManagerFactory
- trustManagerFactory =
- TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
- trustManagerFactory.init(truststore);
-
- for (TrustManager trustManager: trustManagerFactory.getTrustManagers()) {
- if (trustManager instanceof X509TrustManager) {
- X509TrustManager x509TrustManager = (X509TrustManager)trustManager;
- x509TrustManager.checkServerTrusted(
- new X509Certificate[] {(X509Certificate) certificate},
- "RSA_EXPORT");
- }
- }
-
- } finally {
- if(null != keyis) {
- keyis.close();
- }
- if(null != trustis) {
- trustis.close();
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cef1ce44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/security/TestMultiThreadedStoreGeneration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/security/TestMultiThreadedStoreGeneration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/security/TestMultiThreadedStoreGeneration.java
deleted file mode 100644
index 2e2ffce..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/security/TestMultiThreadedStoreGeneration.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.services.security;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.alias.CredentialProvider;
-import org.apache.hadoop.security.alias.CredentialProviderFactory;
-import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.SliderXmlConfKeys;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.MapOperations;
-import org.apache.slider.core.exceptions.SliderException;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.CountDownLatch;
-
-import static org.junit.Assert.assertTrue;
-
-/**
- *
- */
-public class TestMultiThreadedStoreGeneration {
-
- public static final int NUM_THREADS = 30;
- @Rule
- public TemporaryFolder workDir = new TemporaryFolder();;
-
- private void setupCredentials(AggregateConf instanceDefinition,
- String keyAlias, String trustAlias)
- throws Exception {
- Configuration conf = new Configuration();
- final Path jksPath = new Path(SecurityUtils.getSecurityDir(), "test.jks");
- final String ourUrl =
- JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
-
- File file = new File(SecurityUtils.getSecurityDir(), "test.jks");
- file.delete();
- conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl);
-
- instanceDefinition.getAppConf().credentials.put(ourUrl, new ArrayList<String>());
-
- CredentialProvider provider =
- CredentialProviderFactory.getProviders(conf).get(0);
-
- // create new aliases
- try {
-
- if (keyAlias != null) {
- char[] storepass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
- provider.createCredentialEntry(
- keyAlias, storepass);
- }
-
- if (trustAlias != null) {
- char[] trustpass = {'t', 'r', 'u', 's', 't', 'p', 'a', 's', 's'};
- provider.createCredentialEntry(
- trustAlias, trustpass);
- }
-
- // write out so that it can be found in checks
- provider.flush();
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
-
- @Test
- public void testMultiThreadedStoreGeneration() throws Exception {
-
- CertificateManager certMan = new CertificateManager();
- MapOperations compOperations = new MapOperations();
- File secDir = new File(workDir.getRoot(), SliderKeys.SECURITY_DIR);
- File keystoreFile = new File(secDir, SliderKeys.KEYSTORE_FILE_NAME);
- compOperations.put(SliderXmlConfKeys.KEY_KEYSTORE_LOCATION,
- keystoreFile.getAbsolutePath());
- certMan.initialize(compOperations, "cahost", null, null);
-
- final CountDownLatch latch = new CountDownLatch(1);
- final List<SecurityStore> stores = new ArrayList<>();
- List<Thread> threads = new ArrayList<>();
- final AggregateConf instanceDefinition = new AggregateConf();
-
- setupCredentials(instanceDefinition,
- SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_DEFAULT, null);
- final MapOperations compOps = new MapOperations();
- compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true");
-
- for (int i=0; i<NUM_THREADS; ++i) {
- final int finalI = i;
- Runnable runner = new Runnable() {
- public void run() {
- System.out.println ("----> In run");
- try {
- latch.await();
- SecurityStore[] stores1 = StoresGenerator.generateSecurityStores(
- "testhost",
- "container" + finalI,
- "component" + finalI,
- instanceDefinition,
- compOps);
- System.out.println ("----> stores1" + stores1);
- List<SecurityStore>
- securityStores =
- Arrays.asList(stores1);
- stores.addAll(securityStores);
- } catch (InterruptedException e) {
- e.printStackTrace();
- } catch (SliderException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- };
- Thread thread = new Thread(runner, "TestThread" + i);
- threads.add(thread);
- thread.start();
- }
- latch.countDown();
- for (Thread t : threads) {
- t.join();
- }
-
- for (int i=0; i < NUM_THREADS; i++) {
- assertTrue("keystore " + i + " not generated", stores.get(i).getFile().exists());
- }
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org