You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/06/21 18:33:39 UTC
[17/50] [abbrv] hadoop git commit: YARN-6335. Port slider's groovy
unit tests to yarn native services. Contributed by Billie Rinaldi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4afe1813/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/ContractTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/ContractTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/ContractTestUtils.java
new file mode 100644
index 0000000..fc51e31
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/ContractTestUtils.java
@@ -0,0 +1,901 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.utils;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.Assert;
+import org.junit.internal.AssumptionViolatedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.Properties;
+import java.util.UUID;
+
+/**
+ * Utilities used across test cases to make assertions about filesystems
+ * -assertions which fail with useful information.
+ * This is lifted from Hadoop common Test; that JAR isn't published, so
+ * we have to make do.
+ */
+public class ContractTestUtils extends Assert {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContractTestUtils.class);
+
+ public static final String IO_FILE_BUFFER_SIZE = "io.file.buffer.size";
+
+ // For scale testing, we can repeatedly write small chunk data to generate
+ // a large file.
+ public static final String IO_CHUNK_BUFFER_SIZE = "io.chunk.buffer.size";
+ public static final int DEFAULT_IO_CHUNK_BUFFER_SIZE = 128;
+ public static final String IO_CHUNK_MODULUS_SIZE = "io.chunk.modulus.size";
+ public static final int DEFAULT_IO_CHUNK_MODULUS_SIZE = 128;
+
+ /**
+ * Assert that a property in the property set matches the expected value
+ * @param props property set
+ * @param key property name
+ * @param expected expected value. If null, the property must not be in the set
+ */
+ public static void assertPropertyEquals(Properties props,
+ String key,
+ String expected) {
+ String val = props.getProperty(key);
+ if (expected == null) {
+ assertNull("Non null property " + key + " = " + val, val);
+ } else {
+ assertEquals("property " + key + " = " + val,
+ expected,
+ val);
+ }
+ }
+
+ /**
+ *
+ * Write a file and read it in, validating the result. Optional flags control
+ * whether file overwrite operations should be enabled, and whether the
+ * file should be deleted afterwards.
+ *
+ * If there is a mismatch between what was written and what was expected,
+ * a small range of bytes either side of the first error are logged to aid
+ * diagnosing what problem occurred -whether it was a previous file
+ * or a corrupting of the current file. This assumes that two
+ * sequential runs to the same path use datasets with different character
+ * moduli.
+ *
+ * @param fs filesystem
+ * @param path path to write to
+ * @param len length of data
+ * @param overwrite should the create option allow overwrites?
+ * @param delete should the file be deleted afterwards? -with a verification
+ * that it worked. Deletion is not attempted if an assertion has failed
+ * earlier -it is not in a <code>finally{}</code> block.
+ * @throws IOException IO problems
+ */
+ public static void writeAndRead(FileSystem fs,
+ Path path,
+ byte[] src,
+ int len,
+ int blocksize,
+ boolean overwrite,
+ boolean delete) throws IOException {
+ fs.mkdirs(path.getParent());
+
+ writeDataset(fs, path, src, len, blocksize, overwrite);
+
+ byte[] dest = readDataset(fs, path, len);
+
+ compareByteArrays(src, dest, len);
+
+ if (delete) {
+ rejectRootOperation(path);
+ boolean deleted = fs.delete(path, false);
+ assertTrue("Deleted", deleted);
+ assertPathDoesNotExist(fs, "Cleanup failed", path);
+ }
+ }
+
+ /**
+ * Write a file.
+ * Optional flags control
+ * whether file overwrite operations should be enabled
+ * @param fs filesystem
+ * @param path path to write to
+ * @param len length of data
+ * @param overwrite should the create option allow overwrites?
+ * @throws IOException IO problems
+ */
+ public static void writeDataset(FileSystem fs,
+ Path path,
+ byte[] src,
+ int len,
+ int buffersize,
+ boolean overwrite) throws IOException {
+ assertTrue(
+ "Not enough data in source array to write " + len + " bytes",
+ src.length >= len);
+ FSDataOutputStream out = fs.create(path,
+ overwrite,
+ fs.getConf()
+ .getInt(IO_FILE_BUFFER_SIZE,
+ 4096),
+ (short) 1,
+ buffersize);
+ out.write(src, 0, len);
+ out.close();
+ assertFileHasLength(fs, path, len);
+ }
+
+ /**
+ * Read the file and convert to a byte dataset.
+ * This implements readfully internally, so that it will read
+ * in the file without ever having to seek()
+ * @param fs filesystem
+ * @param path path to read from
+ * @param len length of data to read
+ * @return the bytes
+ * @throws IOException IO problems
+ */
+ public static byte[] readDataset(FileSystem fs, Path path, int len)
+ throws IOException {
+ FSDataInputStream in = fs.open(path);
+ byte[] dest = new byte[len];
+ int offset =0;
+ int nread = 0;
+ try {
+ while (nread < len) {
+ int nbytes = in.read(dest, offset + nread, len - nread);
+ if (nbytes < 0) {
+ throw new EOFException("End of file reached before reading fully.");
+ }
+ nread += nbytes;
+ }
+ } finally {
+ in.close();
+ }
+ return dest;
+ }
+
+ /**
+ * Read a file, verify its length and contents match the expected array
+ * @param fs filesystem
+ * @param path path to file
+ * @param original original dataset
+ * @throws IOException IO Problems
+ */
+ public static void verifyFileContents(FileSystem fs,
+ Path path,
+ byte[] original) throws IOException {
+ FileStatus stat = fs.getFileStatus(path);
+ String statText = stat.toString();
+ assertTrue("not a file " + statText, stat.isFile());
+ assertEquals("wrong length " + statText, original.length, stat.getLen());
+ byte[] bytes = readDataset(fs, path, original.length);
+ compareByteArrays(original,bytes,original.length);
+ }
+
+ /**
+ * Verify that the read at a specific offset in a stream
+ * matches that expected
+ * @param stm stream
+ * @param fileContents original file contents
+ * @param seekOff seek offset
+ * @param toRead number of bytes to read
+ * @throws IOException IO problems
+ */
+ public static void verifyRead(FSDataInputStream stm, byte[] fileContents,
+ int seekOff, int toRead) throws IOException {
+ byte[] out = new byte[toRead];
+ stm.seek(seekOff);
+ stm.readFully(out);
+ byte[] expected = Arrays.copyOfRange(fileContents, seekOff,
+ seekOff + toRead);
+ compareByteArrays(expected, out,toRead);
+ }
+
+ /**
+ * Assert that tthe array original[0..len] and received[] are equal.
+ * A failure triggers the logging of the bytes near where the first
+ * difference surfaces.
+ * @param original source data
+ * @param received actual
+ * @param len length of bytes to compare
+ */
+ public static void compareByteArrays(byte[] original,
+ byte[] received,
+ int len) {
+ assertEquals("Number of bytes read != number written",
+ len, received.length);
+ int errors = 0;
+ int first_error_byte = -1;
+ for (int i = 0; i < len; i++) {
+ if (original[i] != received[i]) {
+ if (errors == 0) {
+ first_error_byte = i;
+ }
+ errors++;
+ }
+ }
+
+ if (errors > 0) {
+ String message = String.format(" %d errors in file of length %d",
+ errors, len);
+ LOG.warn(message);
+ // the range either side of the first error to print
+ // this is a purely arbitrary number, to aid user debugging
+ final int overlap = 10;
+ for (int i = Math.max(0, first_error_byte - overlap);
+ i < Math.min(first_error_byte + overlap, len);
+ i++) {
+ byte actual = received[i];
+ byte expected = original[i];
+ String letter = toChar(actual);
+ String line = String.format("[%04d] %2x %s\n", i, actual, letter);
+ if (expected != actual) {
+ line = String.format("[%04d] %2x %s -expected %2x %s\n",
+ i,
+ actual,
+ letter,
+ expected,
+ toChar(expected));
+ }
+ LOG.warn(line);
+ }
+ fail(message);
+ }
+ }
+
+ /**
+ * Convert a byte to a character for printing. If the
+ * byte value is < 32 -and hence unprintable- the byte is
+ * returned as a two digit hex value
+ * @param b byte
+ * @return the printable character string
+ */
+ public static String toChar(byte b) {
+ if (b >= 0x20) {
+ return Character.toString((char) b);
+ } else {
+ return String.format("%02x", b);
+ }
+ }
+
+ /**
+ * Convert a buffer to a string, character by character
+ * @param buffer input bytes
+ * @return a string conversion
+ */
+ public static String toChar(byte[] buffer) {
+ StringBuilder builder = new StringBuilder(buffer.length);
+ for (byte b : buffer) {
+ builder.append(toChar(b));
+ }
+ return builder.toString();
+ }
+
+ public static byte[] toAsciiByteArray(String s) {
+ char[] chars = s.toCharArray();
+ int len = chars.length;
+ byte[] buffer = new byte[len];
+ for (int i = 0; i < len; i++) {
+ buffer[i] = (byte) (chars[i] & 0xff);
+ }
+ return buffer;
+ }
+
+ /**
+ * Cleanup at the end of a test run
+ * @param action action triggering the operation (for use in logging)
+ * @param fileSystem filesystem to work with. May be null
+ * @param cleanupPath path to delete as a string
+ */
+ public static void cleanup(String action,
+ FileSystem fileSystem,
+ String cleanupPath) {
+ if (fileSystem == null) {
+ return;
+ }
+ Path path = new Path(cleanupPath).makeQualified(fileSystem.getUri(),
+ fileSystem.getWorkingDirectory());
+ cleanup(action, fileSystem, path);
+ }
+
+ /**
+ * Cleanup at the end of a test run
+ * @param action action triggering the operation (for use in logging)
+ * @param fileSystem filesystem to work with. May be null
+ * @param path path to delete
+ */
+ public static void cleanup(String action, FileSystem fileSystem, Path path) {
+ noteAction(action);
+ try {
+ rm(fileSystem, path, true, false);
+ } catch (Exception e) {
+ LOG.error("Error deleting in "+ action + " - " + path + ": " + e, e);
+ }
+ }
+
+ /**
+ * Delete a directory. There's a safety check for operations against the
+ * root directory -these are intercepted and rejected with an IOException
+ * unless the allowRootDelete flag is true
+ * @param fileSystem filesystem to work with. May be null
+ * @param path path to delete
+ * @param recursive flag to enable recursive delete
+ * @param allowRootDelete can the root directory be deleted?
+ * @throws IOException on any problem.
+ */
+ public static boolean rm(FileSystem fileSystem,
+ Path path,
+ boolean recursive,
+ boolean allowRootDelete) throws
+ IOException {
+ if (fileSystem != null) {
+ rejectRootOperation(path, allowRootDelete);
+ if (fileSystem.exists(path)) {
+ return fileSystem.delete(path, recursive);
+ }
+ }
+ return false;
+
+ }
+
+ /**
+ * Block any operation on the root path. This is a safety check
+ * @param path path in the filesystem
+ * @param allowRootOperation can the root directory be manipulated?
+ * @throws IOException if the operation was rejected
+ */
+ public static void rejectRootOperation(Path path,
+ boolean allowRootOperation) throws IOException {
+ if (path.isRoot() && !allowRootOperation) {
+ throw new IOException("Root directory operation rejected: " + path);
+ }
+ }
+
+ /**
+ * Block any operation on the root path. This is a safety check
+ * @param path path in the filesystem
+ * @throws IOException if the operation was rejected
+ */
+ public static void rejectRootOperation(Path path) throws IOException {
+ rejectRootOperation(path, false);
+ }
+
+
+ public static void noteAction(String action) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("============== "+ action +" =============");
+ }
+ }
+
+ /**
+ * downgrade a failure to a message and a warning, then an
+ * exception for the Junit test runner to mark as failed
+ * @param message text message
+ * @param failure what failed
+ * @throws AssumptionViolatedException always
+ */
+ public static void downgrade(String message, Throwable failure) {
+ LOG.warn("Downgrading test " + message, failure);
+ AssumptionViolatedException ave =
+ new AssumptionViolatedException(failure, null);
+ throw ave;
+ }
+
+ /**
+ * report an overridden test as unsupported
+ * @param message message to use in the text
+ * @throws AssumptionViolatedException always
+ */
+ public static void unsupported(String message) {
+ skip(message);
+ }
+
+ /**
+ * report a test has been skipped for some reason
+ * @param message message to use in the text
+ * @throws AssumptionViolatedException always
+ */
+ public static void skip(String message) {
+ LOG.info("Skipping: {}", message);
+ throw new AssumptionViolatedException(message);
+ }
+
+ /**
+ * Fail with an exception that was received
+ * @param text text to use in the exception
+ * @param thrown a (possibly null) throwable to init the cause with
+ * @throws AssertionError with the text and throwable -always
+ */
+ public static void fail(String text, Throwable thrown) {
+ AssertionError e = new AssertionError(text);
+ e.initCause(thrown);
+ throw e;
+ }
+
+ /**
+ * Make an assertion about the length of a file
+ * @param fs filesystem
+ * @param path path of the file
+ * @param expected expected length
+ * @throws IOException on File IO problems
+ */
+ public static void assertFileHasLength(FileSystem fs, Path path,
+ int expected) throws IOException {
+ FileStatus status = fs.getFileStatus(path);
+ assertEquals(
+ "Wrong file length of file " + path + " status: " + status,
+ expected,
+ status.getLen());
+ }
+
+ /**
+ * Assert that a path refers to a directory
+ * @param fs filesystem
+ * @param path path of the directory
+ * @throws IOException on File IO problems
+ */
+ public static void assertIsDirectory(FileSystem fs,
+ Path path) throws IOException {
+ FileStatus fileStatus = fs.getFileStatus(path);
+ assertIsDirectory(fileStatus);
+ }
+
+ /**
+ * Assert that a path refers to a directory
+ * @param fileStatus stats to check
+ */
+ public static void assertIsDirectory(FileStatus fileStatus) {
+ assertTrue("Should be a directory -but isn't: " + fileStatus,
+ fileStatus.isDirectory());
+ }
+
+ /**
+ * Write the text to a file, returning the converted byte array
+ * for use in validating the round trip
+ * @param fs filesystem
+ * @param path path of file
+ * @param text text to write
+ * @param overwrite should the operation overwrite any existing file?
+ * @return the read bytes
+ * @throws IOException on IO problems
+ */
+ public static byte[] writeTextFile(FileSystem fs,
+ Path path,
+ String text,
+ boolean overwrite) throws IOException {
+ byte[] bytes = new byte[0];
+ if (text != null) {
+ bytes = toAsciiByteArray(text);
+ }
+ createFile(fs, path, overwrite, bytes);
+ return bytes;
+ }
+
+ /**
+ * Create a file
+ * @param fs filesystem
+ * @param path path to write
+ * @param overwrite overwrite flag
+ * @param data source dataset. Can be null
+ * @throws IOException on any problem
+ */
+ public static void createFile(FileSystem fs,
+ Path path,
+ boolean overwrite,
+ byte[] data) throws IOException {
+ FSDataOutputStream stream = fs.create(path, overwrite);
+ if (data != null && data.length > 0) {
+ stream.write(data);
+ }
+ stream.close();
+ }
+
+ /**
+ * Touch a file
+ * @param fs filesystem
+ * @param path path
+ * @throws IOException IO problems
+ */
+ public static void touch(FileSystem fs,
+ Path path) throws IOException {
+ createFile(fs, path, true, null);
+ }
+
+ /**
+ * Delete a file/dir and assert that delete() returned true
+ * <i>and</i> that the path no longer exists. This variant rejects
+ * all operations on root directories
+ * @param fs filesystem
+ * @param file path to delete
+ * @param recursive flag to enable recursive delete
+ * @throws IOException IO problems
+ */
+ public static void assertDeleted(FileSystem fs,
+ Path file,
+ boolean recursive) throws IOException {
+ assertDeleted(fs, file, recursive, false);
+ }
+
+ /**
+ * Delete a file/dir and assert that delete() returned true
+ * <i>and</i> that the path no longer exists. This variant rejects
+ * all operations on root directories
+ * @param fs filesystem
+ * @param file path to delete
+ * @param recursive flag to enable recursive delete
+ * @param allowRootOperations can the root dir be deleted?
+ * @throws IOException IO problems
+ */
+ public static void assertDeleted(FileSystem fs,
+ Path file,
+ boolean recursive,
+ boolean allowRootOperations) throws IOException {
+ rejectRootOperation(file, allowRootOperations);
+ assertPathExists(fs, "about to be deleted file", file);
+ boolean deleted = fs.delete(file, recursive);
+ String dir = ls(fs, file.getParent());
+ assertTrue("Delete failed on " + file + ": " + dir, deleted);
+ assertPathDoesNotExist(fs, "Deleted file", file);
+ }
+
+ /**
+ * Read in "length" bytes, convert to an ascii string
+ * @param fs filesystem
+ * @param path path to read
+ * @param length #of bytes to read.
+ * @return the bytes read and converted to a string
+ * @throws IOException IO problems
+ */
+ public static String readBytesToString(FileSystem fs,
+ Path path,
+ int length) throws IOException {
+ FSDataInputStream in = fs.open(path);
+ try {
+ byte[] buf = new byte[length];
+ in.readFully(0, buf);
+ return toChar(buf);
+ } finally {
+ in.close();
+ }
+ }
+
+ /**
+ * Take an array of filestats and convert to a string (prefixed w/ a [01] counter
+ * @param stats array of stats
+ * @param separator separator after every entry
+ * @return a stringified set
+ */
+ public static String fileStatsToString(FileStatus[] stats, String separator) {
+ StringBuilder buf = new StringBuilder(stats.length * 128);
+ for (int i = 0; i < stats.length; i++) {
+ buf.append(String.format("[%02d] %s", i, stats[i])).append(separator);
+ }
+ return buf.toString();
+ }
+
+ /**
+ * List a directory
+ * @param fileSystem FS
+ * @param path path
+ * @return a directory listing or failure message
+ * @throws IOException
+ */
+ public static String ls(FileSystem fileSystem, Path path) throws IOException {
+ if (path == null) {
+ //surfaces when someone calls getParent() on something at the top of the path
+ return "/";
+ }
+ FileStatus[] stats;
+ String pathtext = "ls " + path;
+ try {
+ stats = fileSystem.listStatus(path);
+ } catch (FileNotFoundException e) {
+ return pathtext + " -file not found";
+ } catch (IOException e) {
+ return pathtext + " -failed: " + e;
+ }
+ return dumpStats(pathtext, stats);
+ }
+
+ public static String dumpStats(String pathname, FileStatus[] stats) {
+ return pathname + fileStatsToString(stats, "\n");
+ }
+
+ /**
+ * Assert that a file exists and whose {@link FileStatus} entry
+ * declares that this is a file and not a symlink or directory.
+ * @param fileSystem filesystem to resolve path against
+ * @param filename name of the file
+ * @throws IOException IO problems during file operations
+ */
+ public static void assertIsFile(FileSystem fileSystem, Path filename) throws
+ IOException {
+ assertPathExists(fileSystem, "Expected file", filename);
+ FileStatus status = fileSystem.getFileStatus(filename);
+ assertIsFile(filename, status);
+ }
+
+ /**
+ * Assert that a file exists and whose {@link FileStatus} entry
+ * declares that this is a file and not a symlink or directory.
+ * @param filename name of the file
+ * @param status file status
+ */
+ public static void assertIsFile(Path filename, FileStatus status) {
+ String fileInfo = filename + " " + status;
+ assertFalse("File claims to be a directory " + fileInfo,
+ status.isDirectory());
+ assertFalse("File claims to be a symlink " + fileInfo,
+ status.isSymlink());
+ }
+
+ /**
+ * Create a dataset for use in the tests; all data is in the range
+ * base to (base+modulo-1) inclusive
+ * @param len length of data
+ * @param base base of the data
+ * @param modulo the modulo
+ * @return the newly generated dataset
+ */
+ public static byte[] dataset(int len, int base, int modulo) {
+ byte[] dataset = new byte[len];
+ for (int i = 0; i < len; i++) {
+ dataset[i] = (byte) (base + (i % modulo));
+ }
+ return dataset;
+ }
+
+ /**
+ * Assert that a path exists -but make no assertions as to the
+ * type of that entry
+ *
+ * @param fileSystem filesystem to examine
+ * @param message message to include in the assertion failure message
+ * @param path path in the filesystem
+ * @throws FileNotFoundException raised if the path is missing
+ * @throws IOException IO problems
+ */
+ public static void assertPathExists(FileSystem fileSystem, String message,
+ Path path) throws IOException {
+ if (!fileSystem.exists(path)) {
+ //failure, report it
+ String listing = ls(fileSystem, path.getParent());
+ throw new FileNotFoundException(message + ": not found " + path
+ + " in \"" + path.getParent() + "\" :\n" + listing);
+ }
+ }
+
+ /**
+ * Assert that a path does not exist
+ *
+ * @param fileSystem filesystem to examine
+ * @param message message to include in the assertion failure message
+ * @param path path in the filesystem
+ * @throws IOException IO problems
+ */
+ public static void assertPathDoesNotExist(FileSystem fileSystem,
+ String message,
+ Path path) throws IOException {
+ try {
+ FileStatus status = fileSystem.getFileStatus(path);
+ fail(message + ": unexpectedly found " + path + " as " + status);
+ } catch (FileNotFoundException expected) {
+ //this is expected
+
+ }
+ }
+
+ /**
+ * Assert that a FileSystem.listStatus on a dir finds the subdir/child entry
+ * @param fs filesystem
+ * @param dir directory to scan
+ * @param subdir full path to look for
+ * @throws IOException IO probles
+ */
+ public static void assertListStatusFinds(FileSystem fs,
+ Path dir,
+ Path subdir) throws IOException {
+ FileStatus[] stats = fs.listStatus(dir);
+ boolean found = false;
+ StringBuilder builder = new StringBuilder();
+ for (FileStatus stat : stats) {
+ builder.append(stat.toString()).append('\n');
+ if (stat.getPath().equals(subdir)) {
+ found = true;
+ }
+ }
+ assertTrue("Path " + subdir
+ + " not found in directory " + dir + ":" + builder,
+ found);
+ }
+
+ /**
+ * Test for the host being an OSX machine
+ * @return true if the JVM thinks that is running on OSX
+ */
+ public static boolean isOSX() {
+ return System.getProperty("os.name").contains("OS X");
+ }
+
+ /**
+ * compare content of file operations using a double byte array
+ * @param concat concatenated files
+ * @param bytes bytes
+ */
+ public static void validateFileContent(byte[] concat, byte[][] bytes) {
+ int idx = 0;
+ boolean mismatch = false;
+
+ for (byte[] bb : bytes) {
+ for (byte b : bb) {
+ if (b != concat[idx++]) {
+ mismatch = true;
+ break;
+ }
+ }
+ if (mismatch)
+ break;
+ }
+ assertFalse("File content of file is not as expected at offset " + idx,
+ mismatch);
+ }
+
+ /**
+ * Receives test data from the given input file and checks the size of the
+ * data as well as the pattern inside the received data.
+ *
+ * @param fs FileSystem
+ * @param path Input file to be checked
+ * @param expectedSize the expected size of the data to be read from the
+ * input file in bytes
+ * @param bufferLen Pattern length
+ * @param modulus Pattern modulus
+ * @throws IOException
+ * thrown if an error occurs while reading the data
+ */
+ public static void verifyReceivedData(FileSystem fs, Path path,
+ final long expectedSize,
+ final int bufferLen,
+ final int modulus) throws IOException {
+ final byte[] testBuffer = new byte[bufferLen];
+
+ long totalBytesRead = 0;
+ int nextExpectedNumber = 0;
+ final InputStream inputStream = fs.open(path);
+ try {
+ while (true) {
+ final int bytesRead = inputStream.read(testBuffer);
+ if (bytesRead < 0) {
+ break;
+ }
+
+ totalBytesRead += bytesRead;
+
+ for (int i = 0; i < bytesRead; ++i) {
+ if (testBuffer[i] != nextExpectedNumber) {
+ throw new IOException("Read number " + testBuffer[i]
+ + " but expected " + nextExpectedNumber);
+ }
+
+ ++nextExpectedNumber;
+
+ if (nextExpectedNumber == modulus) {
+ nextExpectedNumber = 0;
+ }
+ }
+ }
+
+ if (totalBytesRead != expectedSize) {
+ throw new IOException("Expected to read " + expectedSize +
+ " bytes but only received " + totalBytesRead);
+ }
+ } finally {
+ inputStream.close();
+ }
+ }
+
+ /**
+ * Generates test data of the given size according to some specific pattern
+ * and writes it to the provided output file.
+ *
+ * @param fs FileSystem
+ * @param path Test file to be generated
+ * @param size The size of the test data to be generated in bytes
+ * @param bufferLen Pattern length
+ * @param modulus Pattern modulus
+ * @throws IOException
+ * thrown if an error occurs while writing the data
+ */
+ public static long generateTestFile(FileSystem fs, Path path,
+ final long size,
+ final int bufferLen,
+ final int modulus) throws IOException {
+ final byte[] testBuffer = new byte[bufferLen];
+ for (int i = 0; i < testBuffer.length; ++i) {
+ testBuffer[i] = (byte) (i % modulus);
+ }
+
+ final OutputStream outputStream = fs.create(path, false);
+ long bytesWritten = 0;
+ try {
+ while (bytesWritten < size) {
+ final long diff = size - bytesWritten;
+ if (diff < testBuffer.length) {
+ outputStream.write(testBuffer, 0, (int) diff);
+ bytesWritten += diff;
+ } else {
+ outputStream.write(testBuffer);
+ bytesWritten += testBuffer.length;
+ }
+ }
+
+ return bytesWritten;
+ } finally {
+ outputStream.close();
+ }
+ }
+
+ /**
+ * Creates and reads a file with the given size. The test file is generated
+ * according to a specific pattern so it can be easily verified even if it's
+ * a multi-GB one.
+ * During the read phase the incoming data stream is also checked against
+ * this pattern.
+ *
+ * @param fs FileSystem
+ * @param parent Test file parent dir path
+ * @throws IOException
+ * thrown if an I/O error occurs while writing or reading the test file
+ */
+ public static void createAndVerifyFile(FileSystem fs, Path parent, final long fileSize)
+ throws IOException {
+ int testBufferSize = fs.getConf()
+ .getInt(IO_CHUNK_BUFFER_SIZE, DEFAULT_IO_CHUNK_BUFFER_SIZE);
+ int modulus = fs.getConf()
+ .getInt(IO_CHUNK_MODULUS_SIZE, DEFAULT_IO_CHUNK_MODULUS_SIZE);
+
+ final String objectName = UUID.randomUUID().toString();
+ final Path objectPath = new Path(parent, objectName);
+
+ // Write test file in a specific pattern
+ assertEquals(fileSize,
+ generateTestFile(fs, objectPath, fileSize, testBufferSize, modulus));
+ assertPathExists(fs, "not created successful", objectPath);
+
+ // Now read the same file back and verify its content
+ try {
+ verifyReceivedData(fs, objectPath, fileSize, testBufferSize, modulus);
+ } finally {
+ // Delete test file
+ fs.delete(objectPath, false);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4afe1813/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/KeysForTests.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/KeysForTests.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/KeysForTests.java
new file mode 100644
index 0000000..cf96407
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/KeysForTests.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.utils;
+
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXMLConfKeysForTesting;
+
+/**
+ * Keys shared across tests.
+ */
+public interface KeysForTests extends SliderKeys, SliderXMLConfKeysForTesting {
+ /**
+ * Username for all clusters, ZK, etc.
+ */
+ String USERNAME = "bigdataborat";
+
+ int WAIT_TIME = 120;
+ String WAIT_TIME_ARG = Integer.toString(WAIT_TIME);
+
+ String SLIDER_TEST_XML = "slider-test.xml";
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4afe1813/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/MicroZKCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/MicroZKCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/MicroZKCluster.java
new file mode 100644
index 0000000..be452f1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/MicroZKCluster.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.utils;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.impl.zk.RegistryOperationsService;
+import org.apache.hadoop.registry.server.services.MicroZookeeperService;
+import org.apache.slider.common.tools.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+ * Test ZK cluster.
+ */
+public class MicroZKCluster implements Closeable {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MicroZKCluster.class);
+
+ public static final String HOSTS = "127.0.0.1";
+ private MicroZookeeperService zkService;
+ private String zkBindingString;
+ private final Configuration conf;
+ private RegistryOperations registryOperations;
+
+ MicroZKCluster() {
+ this(SliderUtils.createConfiguration());
+ }
+
+ MicroZKCluster(Configuration conf) {
+ this.conf = conf;
+ }
+
+ String getZkBindingString() {
+ return zkBindingString;
+ }
+
+ void createCluster(String name) {
+ zkService = new MicroZookeeperService(name);
+
+ zkService.init(conf);
+ zkService.start();
+ zkBindingString = zkService.getConnectionString();
+ LOG.info("Created {}", this);
+ registryOperations = new RegistryOperationsService(
+ "registry",
+ zkService);
+ registryOperations.init(conf);
+ registryOperations.start();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (registryOperations != null) {
+ registryOperations.stop();
+ }
+ if (zkService != null) {
+ zkService.stop();
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "Micro ZK cluster as " + zkBindingString;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4afe1813/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/Outcome.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/Outcome.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/Outcome.java
new file mode 100644
index 0000000..52875d3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/Outcome.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.utils;
+
+/**
+ * Outcome for probes.
+ */
+public final class Outcome {
+
+ private final String name;
+
+ private Outcome(String name) {
+ this.name = name;
+ }
+
+ public static final Outcome SUCCESS = new Outcome(
+ "Success");
+ public static final Outcome RETRY = new Outcome("Retry");
+ public static final Outcome FAIL = new Outcome("Fail");
+
+ /**
+ * Build from a bool, where false is mapped to retry.
+ * @param b boolean
+ * @return an outcome
+ */
+ static Outcome fromBool(boolean b) {
+ return b ? SUCCESS : RETRY;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4afe1813/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestBase.java
new file mode 100644
index 0000000..f7da585
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestBase.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.utils;
+
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.slider.common.SliderXMLConfKeysForTesting;
+import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.rules.TestName;
+
+import java.io.File;
+
+
+/**
+ * Base class for unit tests as well as ones starting mini clusters
+ * -the foundational code and methods.
+ *
+ */
+public abstract class SliderTestBase extends SliderTestUtils {
+
+ /**
+ * Singleton metric registry.
+ */
+ public static final MetricsAndMonitoring METRICS = new MetricsAndMonitoring();
+ public static final int WEB_STARTUP_TIME = 30000;
+
+ @Rule
+ public TestName methodName = new TestName();
+
+ @BeforeClass
+ public static void nameThread() {
+ Thread.currentThread().setName("JUnit");
+ }
+
+ @Before
+ public void setup() throws Exception {
+ setSliderClientClassName(DEFAULT_SLIDER_CLIENT);
+ FileUtil.fullyDelete(new File(SliderXMLConfKeysForTesting
+ .TEST_SECURITY_DIR));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4afe1813/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestUtils.java
new file mode 100644
index 0000000..fc29b5e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/SliderTestUtils.java
@@ -0,0 +1,1065 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.utils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonParser;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.service.ServiceStateException;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.Container;
+import org.apache.slider.client.SliderClient;
+import org.apache.slider.common.params.Arguments;
+import org.apache.slider.common.tools.Duration;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.main.LauncherExitCodes;
+import org.apache.slider.core.main.ServiceLaunchException;
+import org.apache.slider.core.main.ServiceLauncher;
+import org.apache.slider.core.persist.JsonSerDeser;
+import org.apache.slider.core.registry.docstore.PublishedConfigSet;
+import org.apache.slider.core.registry.docstore.PublishedConfiguration;
+import org.apache.slider.server.services.workflow.ForkedProcessService;
+import org.codehaus.jackson.map.PropertyNamingStrategy;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.slider.common.params.Arguments.ARG_OPTION;
+
+/**
+ * Static utils for tests in this package and in other test projects.
+ *
+ * It is designed to work with mini clusters as well as remote ones
+ *
+ * This class is not final and may be extended for test cases.
+ *
+ * Some of these methods are derived from the SwiftUtils and SwiftTestUtils
+ * classes -replicated here so that they are available in Hadoop-2.0 code
+ */
+public class SliderTestUtils extends Assert {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SliderTestUtils.class);
+ public static final String DEFAULT_SLIDER_CLIENT = SliderClient.class
+ .getName();
+ private static String sliderClientClassName = DEFAULT_SLIDER_CLIENT;
+
+ public static final Map<String, String> EMPTY_MAP = Collections.emptyMap();
+ public static final Map<String, Integer> EMPTY_INT_MAP = Collections
+ .emptyMap();
+ public static final List<String> EMPTY_LIST = Collections.emptyList();
+
+ public static final ObjectReader OBJECT_READER;
+ public static final ObjectWriter OBJECT_WRITER;
+
+ public static final JsonSerDeser<Application> JSON_SER_DESER =
+ new JsonSerDeser<>(Application.class,
+ PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
+
+ static {
+ ObjectMapper mapper = new ObjectMapper();
+ OBJECT_READER = mapper.readerFor(Object.class);
+ OBJECT_WRITER = mapper.writer();
+ }
+
+ /**
+ * Action that returns an object.
+ */
+ public interface Action {
+ Object invoke() throws Exception;
+ }
+
+ /**
+ * Probe that returns an Outcome.
+ */
+ public interface Probe {
+ Outcome invoke(Map args) throws Exception;
+ }
+
+ public static void setSliderClientClassName(String sliderClientClassName) {
+ sliderClientClassName = sliderClientClassName;
+ }
+
+ public static void describe(String s) {
+ LOG.info("");
+ LOG.info("===============================");
+ LOG.info(s);
+ LOG.info("===============================");
+ LOG.info("");
+ }
+
+ /**
+ * Convert a JSON string to something readable.
+ * @param json
+ * @return a string for printing
+ */
+ public static String prettyPrintJson(String json) {
+ Gson gson = new GsonBuilder().setPrettyPrinting().create();
+ return gson.toJson(new JsonParser().parse(json));
+ }
+
+ /**
+ * Convert an object to something readable.
+ * @param src
+ * @return a string for printing
+ */
+ public static String prettyPrintAsJson(Object src)
+ throws JsonProcessingException, UnsupportedEncodingException {
+ return new String(OBJECT_WRITER.writeValueAsBytes(src), "UTF8");
+ }
+
+ /**
+ * Skip the test with a message.
+ * @param message message logged and thrown
+ */
+ public static void skip(String message) {
+ LOG.warn("Skipping test: {}", message);
+ Assume.assumeTrue(message, false);
+ }
+
+ /**
+ * Skip the test with a message if condition holds.
+ * @param condition predicate
+ * @param message message logged and thrown
+ */
+ public static void assume(boolean condition, String message) {
+ if (!condition) {
+ skip(message);
+ }
+ }
+
+ /**
+ * Skip a test if not running on Windows.
+ */
+ public static void assumeWindows() {
+ assume(Shell.WINDOWS, "not windows");
+ }
+
+ /**
+ * Skip a test if running on Windows.
+ */
+ public static void assumeNotWindows() {
+ assume(!Shell.WINDOWS, "windows");
+ }
+
+ /**
+ * Skip a test on windows.
+ */
+ public static void skipOnWindows() {
+ assumeNotWindows();
+ }
+
+ /**
+ * Equality size for a list.
+ * @param left
+ * @param right
+ */
+ public static void assertListEquals(List left, List right) {
+ String lval = collectionToString(left);
+ String rval = collectionToString(right);
+ String text = "comparing " + lval + " to " + rval;
+ assertEquals(text, left.size(), right.size());
+ for (int i = 0; i < left.size(); i++) {
+ assertEquals(text, left.get(i), right.get(i));
+ }
+ }
+
+ /**
+ * Assert a list has a given length.
+ * @param list list
+ * @param size size to have
+ */
+ public static void assertListLength(List list, int size) {
+ String lval = collectionToString(list);
+ assertEquals(lval, size, list.size());
+ }
+
+ /**
+ * Stringify a collection with [ ] at either end.
+ * @param collection collection
+ * @return string value
+ */
+ public static String collectionToString(List collection) {
+ return "[" + SliderUtils.join(collection, ", ", false) + "]";
+ }
+
+ /**
+ * Assume that a string option is set and not equal to "".
+ * @param conf configuration file
+ * @param key key to look for
+ */
+ public static void assumeStringOptionSet(Configuration conf, String key) {
+ if (SliderUtils.isUnset(conf.getTrimmed(key))) {
+ skip("Configuration key " + key + " not set");
+ }
+ }
+
+ /**
+ * assert that a string option is set and not equal to "".
+ * @param conf configuration file
+ * @param key key to look for
+ */
+ public static void assertStringOptionSet(Configuration conf, String key) {
+ getRequiredConfOption(conf, key);
+ }
+
+ /**
+ * Assume that a boolean option is set and true.
+ * Unset or false triggers a test skip
+ * @param conf configuration file
+ * @param key key to look for
+ */
+ public static void assumeBoolOptionTrue(Configuration conf, String key) {
+ assumeBoolOption(conf, key, false);
+ }
+
+ /**
+ * Assume that a boolean option is true.
+ * False triggers a test skip
+ * @param conf configuration file
+ * @param key key to look for
+ * @param defval default value if the property is not defined
+ */
+ public static void assumeBoolOption(
+ Configuration conf, String key, boolean defval) {
+ assume(conf.getBoolean(key, defval),
+ "Configuration key " + key + " is false");
+ }
+
+ /**
+ * Get a required config option (trimmed, incidentally).
+ * Test will fail if not set
+ * @param conf configuration
+ * @param key key
+ * @return the string
+ */
+ public static String getRequiredConfOption(Configuration conf, String key) {
+ String val = conf.getTrimmed(key);
+ if (SliderUtils.isUnset(val)) {
+ fail("Missing configuration option " + key);
+ }
+ return val;
+ }
+
+ /**
+ * Fails a test because required behavior has not been implemented.
+ */
+ public static void failNotImplemented() {
+ fail("Not implemented");
+ }
+
+ /**
+ * Assert that any needed libraries being present. On Unix none are needed;
+ * on windows they must be present
+ */
+ public static void assertNativeLibrariesPresent() {
+ String errorText = SliderUtils.checkForRequiredNativeLibraries();
+ if (SliderUtils.isSet(errorText)) {
+ fail(errorText);
+ }
+ }
+
+ protected static String[] toArray(List<Object> args) {
+ String[] converted = new String[args.size()];
+ for (int i = 0; i < args.size(); i++) {
+ Object elt = args.get(i);
+ assertNotNull(args.get(i));
+ converted[i] = elt.toString();
+ }
+ return converted;
+ }
+
+ public static void waitWhileClusterLive(SliderClient client, int timeout)
+ throws IOException, YarnException {
+ Duration duration = new Duration(timeout);
+ duration.start();
+ while (client.actionExists(client.getDeployedClusterName(), true) ==
+ LauncherExitCodes.EXIT_SUCCESS && !duration.getLimitExceeded()) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ }
+ }
+ if (duration.getLimitExceeded()) {
+ fail("Cluster " + client.getDeployedClusterName() + " still live after " +
+ timeout + " ms");
+ }
+ }
+
+ public static void waitUntilClusterLive(SliderClient client, int timeout)
+ throws IOException, YarnException {
+ Duration duration = new Duration(timeout);
+ duration.start();
+ while (LauncherExitCodes.EXIT_SUCCESS != client.actionExists(
+ client.getDeployedClusterName(), true) &&
+ !duration.getLimitExceeded()) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ }
+ }
+ if (duration.getLimitExceeded()) {
+ fail("Cluster " + client.getDeployedClusterName() + " not live after " +
+ timeout + " ms");
+ }
+ }
+
+ public static void dumpClusterDescription(
+ String text,
+ Application status) throws IOException {
+ describe(text);
+ LOG.info(JSON_SER_DESER.toJson(status));
+ }
+
+ /**
+ * Assert that a service operation succeeded.
+ * @param service service
+ */
+ public static void assertSucceeded(ServiceLauncher service) {
+ assertEquals(0, service.getServiceExitCode());
+ }
+
+ public static void assertContainersLive(Application application,
+ String component, int expected) {
+ LOG.info("Asserting component {} expected count {}", component, expected);
+ int actual = extractLiveContainerCount(application, component);
+ if (expected != actual) {
+ LOG.warn("{} actual={}, expected {} in \n{}\n", component, actual,
+ expected, application);
+ }
+ assertEquals(expected, actual);
+ }
+
+ /**
+ * Robust extraction of live container count.
+ * @param application status
+ * @param component component to resolve
+ * @return the number of containers live.
+ */
+ public static int extractLiveContainerCount(
+ Application application,
+ String component) {
+ int actual = 0;
+ if (application.getContainers() != null) {
+ for (Container container : application.getContainers()) {
+ if (container.getComponentName().equals(component)) {
+ actual++;
+ }
+ }
+ }
+ return actual;
+ }
+
+ /**
+ * Exec a set of commands, wait a few seconds for it to finish.
+ * @param status code
+ * @param commands
+ * @return the process
+ */
+ public static ForkedProcessService exec(int status, List<String> commands)
+ throws IOException, TimeoutException {
+ ForkedProcessService process = exec(commands);
+
+ Integer exitCode = process.getExitCode();
+ assertNotNull(exitCode);
+ assertEquals(status, exitCode.intValue());
+ return process;
+ }
+
+ /**
+ * Exec a set of commands, wait a few seconds for it to finish.
+ * @param commands
+ * @return
+ */
+ public static ForkedProcessService exec(List<String> commands)
+ throws IOException, TimeoutException {
+ ForkedProcessService process;
+ process = new ForkedProcessService(
+ commands.get(0),
+ EMPTY_MAP,
+ commands);
+ process.init(new Configuration());
+ process.start();
+ int timeoutMillis = 5000;
+ if (!process.waitForServiceToStop(timeoutMillis)) {
+ throw new TimeoutException(
+ "Process did not stop in " + timeoutMillis + "mS");
+ }
+ return process;
+ }
+
+ /**
+ * Determine whether an application exists. Run the commands and if the
+ * operation fails with a FileNotFoundException, then
+ * this method returns false.
+ * <p>
+ * Run something harmless like a -version command, something
+ * which must return 0
+ *
+ * @param commands
+ * @return true if the command sequence succeeded
+ * false if they failed with no file
+ * @throws Exception on any other failure cause
+ */
+ public static boolean doesAppExist(List<String> commands)
+ throws IOException, TimeoutException {
+ try {
+ exec(0, commands);
+ return true;
+ } catch (ServiceStateException e) {
+ if (!(e.getCause() instanceof FileNotFoundException)) {
+ throw e;
+ }
+ return false;
+ }
+ }
+
+ /**
+ * Locate an executable on the path.
+ * @param exe executable name. If it is an absolute path which
+ * exists then it will returned direct
+ * @return the path to an exe or null for no match
+ */
+ public static File locateExecutable(String exe) {
+ File exeNameAsPath = new File(exe).getAbsoluteFile();
+ if (exeNameAsPath.exists()) {
+ return exeNameAsPath;
+ }
+
+ File exepath = null;
+ String path = extractPath();
+ String[] dirs = path.split(System.getProperty("path.separator"));
+ for (String dirname : dirs) {
+ File dir = new File(dirname);
+
+ File possible = new File(dir, exe);
+ if (possible.exists()) {
+ exepath = possible;
+ }
+ }
+ return exepath;
+ }
+
+ /**
+ * Lookup the PATH env var.
+ * @return the path or null
+ */
+ public static String extractPath() {
+ return extractEnvVar("PATH");
+ }
+
+ /**
+ * Find an environment variable. Uses case independent checking for
+ * the benefit of windows.
+ * Will fail if the var is not found.
+ * @param var path variable <i>in upper case</i>
+ * @return the env var
+ */
+ public static String extractEnvVar(String var) {
+ String realkey = "";
+
+ for (String it : System.getenv().keySet()) {
+ if (it.toUpperCase(Locale.ENGLISH).equals(var)) {
+ realkey = it;
+ }
+ }
+
+ if (SliderUtils.isUnset(realkey)) {
+ fail("No environment variable " + var + " found");
+ }
+ String val = System.getenv(realkey);
+
+ LOG.info("{} = {}", realkey, val);
+ return val;
+ }
+
+ /**
+ * Create a temp JSON file. After coming up with the name, the file
+ * is deleted
+ * @return the filename
+ */
+ public static File createTempJsonFile() throws IOException {
+ return tmpFile(".json");
+ }
+
+ /**
+ * Create a temp file with the specific name. It's deleted after creation,
+ * to avoid "file exists exceptions"
+ * @param suffix suffix, e.g. ".txt"
+ * @return a path to a file which may be created
+ */
+ public static File tmpFile(String suffix) throws IOException {
+ File reportFile = File.createTempFile(
+ "temp",
+ suffix,
+ new File("target"));
+ reportFile.delete();
+ return reportFile;
+ }
+
+ /**
+ * Execute a closure, assert it fails with a given exit code and text.
+ * @param exitCode exit code
+ * @param text text (can be "")
+ * @param action action
+ * @return
+ */
+ public void assertFailsWithException(int exitCode,
+ String text,
+ Action action) throws Exception {
+ try {
+ action.invoke();
+ fail("Operation was expected to fail —but it succeeded");
+ } catch (ServiceLaunchException e) {
+ assertExceptionDetails(e, exitCode, text);
+ }
+ }
+
+ /**
+ * Execute a closure, assert it fails with a given exit code and text.
+ * @param text text (can be "")
+ * @param action action
+ * @return
+ */
+ public void assertFailsWithExceptionClass(Class clazz,
+ String text,
+ Action action) throws Exception {
+ try {
+ action.invoke();
+ fail("Operation was expected to fail —but it succeeded");
+ } catch (Exception e) {
+ assertExceptionDetails(e, clazz, text);
+ }
+ }
+
+ public static void assertExceptionDetails(
+ ServiceLaunchException ex,
+ int exitCode) {
+ assertExceptionDetails(ex, exitCode, null);
+ }
+
+ /**
+ * Make an assertion about the exit code of an exception.
+ * @param ex exception
+ * @param exitCode exit code
+ * @param text error text to look for in the exception
+ */
+ public static void assertExceptionDetails(
+ ServiceLaunchException ex,
+ int exitCode,
+ String text) {
+ if (exitCode != ex.getExitCode()) {
+ String message = String.format("Wrong exit code, expected %d but" +
+ " got %d in %s", exitCode, ex.getExitCode(), ex);
+ LOG.warn(message, ex);
+ throw new AssertionError(message, ex);
+ }
+ if (SliderUtils.isSet(text)) {
+ if (!(ex.toString().contains(text))) {
+ String message = String.format("String match for \"%s\"failed in %s",
+ text, ex);
+ LOG.warn(message, ex);
+ throw new AssertionError(message, ex);
+ }
+ }
+ }
+
+ /**
+ * Make an assertion about the class of an exception.
+ * @param ex exception
+ * @param clazz exit code
+ * @param text error text to look for in the exception
+ */
+ static void assertExceptionDetails(
+ Exception ex,
+ Class clazz,
+ String text) throws Exception {
+ if (ex.getClass() != clazz) {
+ throw ex;
+ }
+ if (SliderUtils.isSet(text) && !(ex.toString().contains(text))) {
+ throw ex;
+ }
+ }
+
+ /**
+ * Launch the slider client with the specific args; no validation
+ * of return code takes place.
+ * @param conf configuration
+ * @param args arg list
+ * @return the launcher
+ */
+ protected static ServiceLauncher<SliderClient> execSliderCommand(
+ Configuration conf,
+ List args) throws Throwable {
+ ServiceLauncher<SliderClient> serviceLauncher =
+ new ServiceLauncher<>(sliderClientClassName);
+
+ LOG.debug("slider {}", SliderUtils.join(args, " ", false));
+ serviceLauncher.launchService(conf,
+ toArray(args),
+ false);
+ return serviceLauncher;
+ }
+
+ /**
+ * Launch a slider command to a given exit code.
+ * Most failures will trigger exceptions; this is for the exit code of the
+ * runService() call.
+ * @param exitCode desired exit code
+ * @param conf configuration
+ * @param args arg list
+ * @return the launcher
+ */
+ protected static ServiceLauncher<SliderClient> execSliderCommand(
+ int exitCode,
+ Configuration conf,
+ List args) throws Throwable {
+ ServiceLauncher<SliderClient> serviceLauncher = execSliderCommand(conf,
+ args);
+ assertEquals(exitCode, serviceLauncher.getServiceExitCode());
+ return serviceLauncher;
+ }
+
+ public static ServiceLauncher launch(Class serviceClass,
+ Configuration conf,
+ List<Object> args) throws
+ Throwable {
+ ServiceLauncher serviceLauncher =
+ new ServiceLauncher(serviceClass.getName());
+
+ String joinedArgs = SliderUtils.join(args, " ", false);
+ LOG.debug("slider {}", joinedArgs);
+
+ serviceLauncher.launchService(conf,
+ toArray(args),
+ false);
+ return serviceLauncher;
+ }
+
+ public static Throwable launchExpectingException(Class serviceClass,
+ Configuration conf,
+ String expectedText,
+ List args)
+ throws Throwable {
+ try {
+ ServiceLauncher launch = launch(serviceClass, conf, args);
+ throw new AssertionError("Expected an exception with text containing " +
+ expectedText + " -but the service completed with exit code " +
+ launch.getServiceExitCode());
+ } catch (AssertionError error) {
+ throw error;
+ } catch (Throwable thrown) {
+ if (SliderUtils.isSet(expectedText) && !thrown.toString().contains(
+ expectedText)) {
+ //not the right exception -rethrow
+ LOG.warn("Caught Exception did not contain expected text" +
+ "\"" + expectedText + "\"");
+ throw thrown;
+ }
+ return thrown;
+ }
+ }
+
+
+ public static ServiceLauncher<SliderClient> launchClientAgainstRM(
+ String address,
+ List<String> args,
+ Configuration conf) throws Throwable {
+ assertNotNull(address);
+ LOG.info("Connecting to rm at {}", address);
+ if (!args.contains(Arguments.ARG_MANAGER)) {
+ args.add(Arguments.ARG_MANAGER);
+ args.add(address);
+ }
+ ServiceLauncher<SliderClient> launcher = execSliderCommand(conf, args);
+ return launcher;
+ }
+
+ /**
+ * Add a configuration parameter as a cluster configuration option.
+ * @param extraArgs extra arguments
+ * @param conf config
+ * @param option option
+ */
+ public static void addClusterConfigOption(
+ List<String> extraArgs,
+ YarnConfiguration conf,
+ String option) {
+
+ conf.getTrimmed(option);
+ extraArgs.add(ARG_OPTION);
+ extraArgs.add(option);
+ extraArgs.add(getRequiredConfOption(conf, option));
+ }
+
+ /**
+ * Assert that a path refers to a directory.
+ * @param fs filesystem
+ * @param path path of the directory
+ * @throws IOException on File IO problems
+ */
+ public static void assertIsDirectory(FileSystem fs,
+ Path path) throws IOException {
+ FileStatus fileStatus = fs.getFileStatus(path);
+ assertIsDirectory(fileStatus);
+ }
+
+ /**
+ * Assert that a path refers to a directory.
+ * @param fileStatus stats to check
+ */
+ public static void assertIsDirectory(FileStatus fileStatus) {
+ assertTrue("Should be a dir -but isn't: " + fileStatus,
+ fileStatus.isDirectory());
+ }
+
+ /**
+ * Assert that a path exists -but make no assertions as to the
+ * type of that entry.
+ *
+ * @param fileSystem filesystem to examine
+ * @param message message to include in the assertion failure message
+ * @param path path in the filesystem
+ * @throws IOException IO problems
+ */
+ public static void assertPathExists(
+ FileSystem fileSystem,
+ String message,
+ Path path) throws IOException {
+ if (!fileSystem.exists(path)) {
+ //failure, report it
+ fail(
+ message + ": not found \"" + path + "\" in " + path.getParent() +
+ "-" +
+ ls(fileSystem, path.getParent()));
+ }
+ }
+
+ /**
+ * Assert that a path does not exist.
+ *
+ * @param fileSystem filesystem to examine
+ * @param message message to include in the assertion failure message
+ * @param path path in the filesystem
+ * @throws IOException IO problems
+ */
+ public static void assertPathDoesNotExist(
+ FileSystem fileSystem,
+ String message,
+ Path path) throws IOException {
+ try {
+ FileStatus status = fileSystem.getFileStatus(path);
+ // a status back implies there is a file here
+ fail(message + ": unexpectedly found " + path + " as " + status);
+ } catch (FileNotFoundException expected) {
+ //this is expected
+
+ }
+ }
+
+ /**
+ * Assert that a FileSystem.listStatus on a dir finds the subdir/child entry.
+ * @param fs filesystem
+ * @param dir directory to scan
+ * @param subdir full path to look for
+ * @throws IOException IO probles
+ */
+ public static void assertListStatusFinds(FileSystem fs,
+ Path dir,
+ Path subdir) throws IOException {
+ FileStatus[] stats = fs.listStatus(dir);
+ boolean found = false;
+ StringBuilder builder = new StringBuilder();
+ for (FileStatus stat : stats) {
+ builder.append(stat.toString()).append('\n');
+ if (stat.getPath().equals(subdir)) {
+ found = true;
+ }
+ }
+ assertTrue("Path " + subdir
+ + " not found in directory " + dir + ":" + builder,
+ found);
+ }
+
+ /**
+ * List a a path to string.
+ * @param fileSystem filesystem
+ * @param path directory
+ * @return a listing of the filestatuses of elements in the directory, one
+ * to a line, precedeed by the full path of the directory
+ * @throws IOException connectivity problems
+ */
+ public static String ls(FileSystem fileSystem, Path path)
+ throws IOException {
+ if (path == null) {
+ //surfaces when someone calls getParent() on something at the top of
+ // the path
+ return "/";
+ }
+ FileStatus[] stats;
+ String pathtext = "ls " + path;
+ try {
+ stats = fileSystem.listStatus(path);
+ } catch (FileNotFoundException e) {
+ return pathtext + " -file not found";
+ } catch (IOException e) {
+ return pathtext + " -failed: " + e;
+ }
+ return pathtext + fileStatsToString(stats, "\n");
+ }
+
+ /**
+ * Take an array of filestats and convert to a string (prefixed w/ a [01]
+ * counter).
+ * @param stats array of stats
+ * @param separator separator after every entry
+ * @return a stringified set
+ */
+ public static String fileStatsToString(FileStatus[] stats, String separator) {
+ StringBuilder buf = new StringBuilder(stats.length * 128);
+ for (int i = 0; i < stats.length; i++) {
+ buf.append(String.format("[%02d] %s", i, stats[i])).append(separator);
+ }
+ return buf.toString();
+ }
+
+ public static void waitWhileClusterLive(SliderClient sliderClient)
+ throws IOException, YarnException {
+ waitWhileClusterLive(sliderClient, 30000);
+ }
+
+ public static void dumpRegistryInstances(
+ Map<String, ServiceRecord> instances) {
+ describe("service registry slider instances");
+ for (Entry<String, ServiceRecord> it : instances.entrySet()) {
+ LOG.info(" {} : {}", it.getKey(), it.getValue());
+ }
+ describe("end list service registry slider instances");
+ }
+
+
+ public static void dumpRegistryInstanceIDs(List<String> instanceIds) {
+ describe("service registry instance IDs");
+ dumpCollection(instanceIds);
+ }
+
+ public static void dumpRegistryServiceTypes(Collection<String> entries) {
+ describe("service registry types");
+ dumpCollection(entries);
+ }
+
+ public static <V> void dumpCollection(Collection<V> entries) {
+ LOG.info("number of entries: {}", entries.size());
+ for (V it : entries) {
+ LOG.info(it.toString());
+ }
+ }
+
+ public static void dumpArray(Object[] entries) {
+ LOG.info("number of entries: {}", entries.length);
+ for (Object it : entries) {
+ LOG.info(it.toString());
+ }
+ }
+
+ public static <K, V> void dumpMap(Map<K, V> map) {
+ for (Entry<K, V> it : map.entrySet()) {
+ LOG.info("\"{}\": \"{}\"", it.getKey().toString(), it.getValue()
+ .toString());
+ }
+ }
+
+ /**
+ * Get a time option in seconds if set, otherwise the default value (also
+ * in seconds).
+ * This operation picks up the time value as a system property if set -that
+ * value overrides anything in the test file
+ * @param conf
+ * @param key
+ * @param defValMillis
+ * @return
+ */
+ public static int getTimeOptionMillis(
+ Configuration conf,
+ String key,
+ int defValMillis) {
+ int val = conf.getInt(key, 0);
+ val = Integer.getInteger(key, val);
+ int time = 1000 * val;
+ if (time == 0) {
+ time = defValMillis;
+ }
+ return time;
+ }
+
+ public void dumpConfigurationSet(PublishedConfigSet confSet) {
+ for (String key : confSet.keys()) {
+ PublishedConfiguration config = confSet.get(key);
+ LOG.info("{} -- {}", key, config.description);
+ }
+ }
+
+ /**
+ * Convert a file to a URI suitable for use in an argument.
+ * @param file file
+ * @return a URI string valid on all platforms
+ */
+ public String toURIArg(File file) {
+ return file.getAbsoluteFile().toURI().toString();
+ }
+
+ /**
+ * Assert a file exists; fails with a listing of the parent dir.
+ * @param text text for front of message
+ * @param file file to look for
+ * @throws FileNotFoundException
+ */
+ public void assertFileExists(String text, File file)
+ throws FileNotFoundException {
+ if (!file.exists()) {
+ File parent = file.getParentFile();
+ String[] files = parent.list();
+ StringBuilder builder = new StringBuilder();
+ builder.append(parent.getAbsolutePath());
+ builder.append(":\n");
+ for (String name : files) {
+ builder.append(" ");
+ builder.append(name);
+ builder.append("\n");
+ }
+ throw new FileNotFoundException(text + ": " + file + " not found in " +
+ builder);
+ }
+ }
+
+ /**
+ * Repeat a probe until it succeeds, if it does not execute a failure
+ * closure then raise an exception with the supplied message.
+ * @param probe probe
+ * @param timeout time in millis before giving up
+ * @param sleepDur sleep between failing attempts
+ * @param args map of arguments to the probe
+ * @param failIfUnsuccessful if the probe fails after all the attempts
+ * —should it raise an exception
+ * @param failureMessage message to include in exception raised
+ * @param failureHandler closure to invoke prior to the failure being raised
+ */
+ protected void repeatUntilSuccess(
+ String action,
+ Probe probe,
+ int timeout,
+ int sleepDur,
+ Map args,
+ boolean failIfUnsuccessful,
+ String failureMessage,
+ Action failureHandler) throws Exception {
+ LOG.debug("Probe {} timelimit {}", action, timeout);
+ if (timeout < 1000) {
+ fail("Timeout " + timeout + " too low: milliseconds are expected, not " +
+ "seconds");
+ }
+ int attemptCount = 1;
+ boolean succeeded = false;
+ boolean completed = false;
+ Duration duration = new Duration(timeout);
+ duration.start();
+ while (!completed) {
+ Outcome outcome = probe.invoke(args);
+ if (outcome.equals(Outcome.SUCCESS)) {
+ // success
+ LOG.debug("Success after {} attempt(s)", attemptCount);
+ succeeded = true;
+ completed = true;
+ } else if (outcome.equals(Outcome.RETRY)) {
+ // failed but retry possible
+ attemptCount++;
+ completed = duration.getLimitExceeded();
+ if (!completed) {
+ LOG.debug("Attempt {} failed", attemptCount);
+ try {
+ Thread.sleep(sleepDur);
+ } catch (InterruptedException e) {
+ }
+ }
+ } else if (outcome.equals(Outcome.FAIL)) {
+ // fast fail
+ LOG.debug("Fast fail of probe");
+ completed = true;
+ }
+ }
+ if (!succeeded) {
+ if (duration.getLimitExceeded()) {
+ LOG.info("probe timed out after {} and {} attempts", timeout,
+ attemptCount);
+ }
+ if (failureHandler != null) {
+ failureHandler.invoke();
+ }
+ if (failIfUnsuccessful) {
+ fail(failureMessage);
+ }
+ }
+ }
+
+ /**
+ * Get a value from a map; raise an assertion if it is not there.
+ * @param map map to look up
+ * @param key key
+ * @return the string value
+ */
+ public <K, V> String requiredMapValue(Map<K, V> map, String key) {
+ assertNotNull(map.get(key));
+ return map.get(key).toString();
+ }
+
+ public static void assertStringContains(String expected, String text) {
+ assertNotNull("null text", text);
+ if (!text.contains(expected)) {
+ String message = String.format("did not find %s in \"%s\"", expected,
+ text);
+ LOG.error(message);
+ fail(message);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4afe1813/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestAssertions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestAssertions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestAssertions.java
new file mode 100644
index 0000000..9806ac3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestAssertions.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.utils;
+
+import org.apache.slider.api.resource.Application;
+import org.junit.Test;
+
+import java.util.Collections;
+
+/**
+ * Test for some of the command test base operations.
+ */
+public class TestAssertions {
+
+ public static final String CLUSTER_JSON = "json/cluster.json";
+
+ @Test
+ public void testNoInstances() throws Throwable {
+ Application application = new Application();
+ application.setContainers(null);
+ SliderTestUtils.assertContainersLive(application, "example", 0);
+ }
+
+ @Test
+ public void testEmptyInstances() throws Throwable {
+ Application application = new Application();
+ application.setContainers(Collections.emptyList());
+ SliderTestUtils.assertContainersLive(application, "example", 0);
+ }
+
+// TODO test metrics retrieval
+// @Test
+// public void testLiveInstances() throws Throwable {
+// InputStream stream = getClass().getClassLoader().getResourceAsStream(
+// CLUSTER_JSON);
+// assertNotNull("could not load " + CLUSTER_JSON, stream);
+// ClusterDescription liveCD = ClusterDescription.fromStream(stream);
+// assertNotNull(liveCD);
+// SliderTestUtils.assertContainersLive(liveCD, "SLEEP_LONG", 4);
+// assertEquals((Integer) 1, liveCD.statistics.get("SLEEP_LONG").get(
+// StatusKeys.STATISTICS_CONTAINERS_ANTI_AFFINE_PENDING));
+// }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4afe1813/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java
new file mode 100644
index 0000000..5493198
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.utils;
+
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
+import org.apache.commons.compress.utils.IOUtils;
+import org.junit.Assert;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+/**
+ * Various utility methods
+ * Byte comparison methods are from
+ * <code>org.apache.hadoop.fs.contract.ContractTestUtils</code>
+ */
+public class TestUtility {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestUtility.class);
+
+ public static void addDir(File dirObj, ZipArchiveOutputStream zipFile, String prefix) throws IOException {
+ for (File file : dirObj.listFiles()) {
+ if (file.isDirectory()) {
+ addDir(file, zipFile, prefix + file.getName() + File.separator);
+ } else {
+ log.info("Adding to zip - " + prefix + file.getName());
+ zipFile.putArchiveEntry(new ZipArchiveEntry(prefix + file.getName()));
+ IOUtils.copy(new FileInputStream(file), zipFile);
+ zipFile.closeArchiveEntry();
+ }
+ }
+ }
+
+ public static void zipDir(String zipFile, String dir) throws IOException {
+ File dirObj = new File(dir);
+ ZipArchiveOutputStream out = new ZipArchiveOutputStream(new FileOutputStream(zipFile));
+ log.info("Creating : {}", zipFile);
+ try {
+ addDir(dirObj, out, "");
+ } finally {
+ out.close();
+ }
+ }
+
+ public static String createAppPackage(
+ TemporaryFolder folder, String subDir, String pkgName, String srcPath) throws IOException {
+ String zipFileName;
+ File pkgPath = folder.newFolder(subDir);
+ File zipFile = new File(pkgPath, pkgName).getAbsoluteFile();
+ zipFileName = zipFile.getAbsolutePath();
+ TestUtility.zipDir(zipFileName, srcPath);
+ log.info("Created temporary zip file at {}", zipFileName);
+ return zipFileName;
+ }
+
+
+ /**
+ * Assert that tthe array original[0..len] and received[] are equal.
+ * A failure triggers the logging of the bytes near where the first
+ * difference surfaces.
+ * @param original source data
+ * @param received actual
+ * @param len length of bytes to compare
+ */
+ public static void compareByteArrays(byte[] original,
+ byte[] received,
+ int len) {
+ Assert.assertEquals("Number of bytes read != number written",
+ len, received.length);
+ int errors = 0;
+ int first_error_byte = -1;
+ for (int i = 0; i < len; i++) {
+ if (original[i] != received[i]) {
+ if (errors == 0) {
+ first_error_byte = i;
+ }
+ errors++;
+ }
+ }
+
+ if (errors > 0) {
+ String message = String.format(" %d errors in file of length %d",
+ errors, len);
+ log.warn(message);
+ // the range either side of the first error to print
+ // this is a purely arbitrary number, to aid user debugging
+ final int overlap = 10;
+ for (int i = Math.max(0, first_error_byte - overlap);
+ i < Math.min(first_error_byte + overlap, len);
+ i++) {
+ byte actual = received[i];
+ byte expected = original[i];
+ String letter = toChar(actual);
+ String line = String.format("[%04d] %2x %s\n", i, actual, letter);
+ if (expected != actual) {
+ line = String.format("[%04d] %2x %s -expected %2x %s\n",
+ i,
+ actual,
+ letter,
+ expected,
+ toChar(expected));
+ }
+ log.warn(line);
+ }
+ Assert.fail(message);
+ }
+ }
+ /**
+ * Convert a byte to a character for printing. If the
+ * byte value is < 32 -and hence unprintable- the byte is
+ * returned as a two digit hex value
+ * @param b byte
+ * @return the printable character string
+ */
+ public static String toChar(byte b) {
+ if (b >= 0x20) {
+ return Character.toString((char) b);
+ } else {
+ return String.format("%02x", b);
+ }
+ }
+
+ /**
+ * Convert a buffer to a string, character by character
+ * @param buffer input bytes
+ * @return a string conversion
+ */
+ public static String toChar(byte[] buffer) {
+ StringBuilder builder = new StringBuilder(buffer.length);
+ for (byte b : buffer) {
+ builder.append(toChar(b));
+ }
+ return builder.toString();
+ }
+
+ public static byte[] toAsciiByteArray(String s) {
+ char[] chars = s.toCharArray();
+ int len = chars.length;
+ byte[] buffer = new byte[len];
+ for (int i = 0; i < len; i++) {
+ buffer[i] = (byte) (chars[i] & 0xff);
+ }
+ return buffer;
+ }
+
+ /**
+ * Create a dataset for use in the tests; all data is in the range
+ * base to (base+modulo-1) inclusive
+ * @param len length of data
+ * @param base base of the data
+ * @param modulo the modulo
+ * @return the newly generated dataset
+ */
+ public static byte[] dataset(int len, int base, int modulo) {
+ byte[] dataset = new byte[len];
+ for (int i = 0; i < len; i++) {
+ dataset[i] = (byte) (base + (i % modulo));
+ }
+ return dataset;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org