You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/12/17 20:41:39 UTC
[1/3] incubator-slider git commit: SLIDER-1031 AgentPingSocketIT
failing on windows
Repository: incubator-slider
Updated Branches:
refs/heads/develop 0a9b30681 -> 8ee057a05
SLIDER-1031 AgentPingSocketIT failing on windows
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/520f6446
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/520f6446
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/520f6446
Branch: refs/heads/develop
Commit: 520f644612c3069430beacb282c85d8647a17f32
Parents: 0a9b306
Author: Steve Loughran <st...@apache.org>
Authored: Thu Dec 17 13:45:23 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Dec 17 13:45:23 2015 +0000
----------------------------------------------------------------------
.../slider/common/SliderXMLConfKeysForTesting.java | 6 ++++++
.../slider/funtest/framework/CommandTestBase.groovy | 7 ++++++-
.../funtest/commands/CommandEnvironmentIT.groovy | 2 +-
.../funtest/lifecycle/AgentClientInstallIT.groovy | 2 +-
.../funtest/lifecycle/AgentPingSocketIT.groovy | 15 +++++++++++----
5 files changed, 25 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/520f6446/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java b/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
index 1e3cf64..61c828e 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderXMLConfKeysForTesting.java
@@ -74,4 +74,10 @@ public interface SliderXMLConfKeysForTesting {
*/
String KEY_TEST_AM_KEYTAB = "slider.test.am.keytab.local";
+ /**
+ * Is the test cluster windows? Default is: same as the local system.
+ * {@value}
+ */
+ String KEY_TEST_WINDOWS_CLUSTER = "slider.test.windows.cluster";
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/520f6446/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 46fefff..a887271 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -101,6 +101,7 @@ abstract class CommandTestBase extends SliderTestUtils {
public static final int REGISTRY_STARTUP_TIMEOUT = 90000
public static final String E_LAUNCH_FAIL = 'Application did not start'
+ public static final WINDOWS_CLUSTER
/*
Static initializer for test configurations. If this code throws exceptions
(which it may) the class will not be instantiable.
@@ -126,7 +127,8 @@ abstract class CommandTestBase extends SliderTestUtils {
TEST_AM_KEYTAB = SLIDER_CONFIG.getTrimmed(
KEY_TEST_AM_KEYTAB)
- TILDE = Shell.WINDOWS? "~" : "\\~"
+ TILDE = Shell.WINDOWS? "~" : "\\~"
+ WINDOWS_CLUSTER = SLIDER_CONFIG.getBoolean(KEY_TEST_WINDOWS_CLUSTER, Shell.WINDOWS)
}
@Rule
@@ -1596,4 +1598,7 @@ abstract class CommandTestBase extends SliderTestUtils {
}
}
+ void assumeTestClusterNotWindows() {
+ assume(!WINDOWS_CLUSTER, "Test cluster is windows")
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/520f6446/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandEnvironmentIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandEnvironmentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandEnvironmentIT.groovy
index 8a2d4bd..6082a8f 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandEnvironmentIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandEnvironmentIT.groovy
@@ -90,7 +90,7 @@ public class CommandEnvironmentIT extends CommandTestBase {
if (python) {
SliderShell.scriptFile = SLIDER_SCRIPT_PYTHON;
} else {
- assumeNotWindows()
+ assumeTestClusterNotWindows()
SliderShell.scriptFile = SLIDER_SCRIPT;
}
SliderShell shell = new SliderShell([
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/520f6446/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClientInstallIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClientInstallIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClientInstallIT.groovy
index 6d1f759..17317e9 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClientInstallIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClientInstallIT.groovy
@@ -37,7 +37,7 @@ public class AgentClientInstallIT extends AgentCommandTestBase
@Test
public void testAgentClientInstall() throws Throwable {
- assumeNotWindows()
+ assumeTestClusterNotWindows()
describe "Install command logger client"
File zipFileName = new File(TEST_APP_PKG_DIR, TEST_APP_PKG_FILE).canonicalFile
File tmpFile = File.createTempFile("del", "");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/520f6446/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentPingSocketIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentPingSocketIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentPingSocketIT.groovy
index b10e792..4aaba90 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentPingSocketIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentPingSocketIT.groovy
@@ -21,6 +21,7 @@ package org.apache.slider.funtest.lifecycle
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import groovy.json.*
+import org.apache.hadoop.io.IOUtils
import org.apache.hadoop.net.NetUtils
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.slider.common.SliderExitCodes
@@ -58,9 +59,9 @@ public class AgentPingSocketIT extends AgentCommandTestBase
}
@Test
- public void testAgentRegistry() throws Throwable {
+ public void testAgentPingSocket() throws Throwable {
describe("Create a cluster using metainfo, resources, and appConfig that calls nc to listen on a port")
- assumeNotWindows()
+ assumeTestClusterNotWindows()
buildClusterPath(CLUSTER)
File launchReportFile = createTempJsonFile();
@@ -102,7 +103,10 @@ public class AgentPingSocketIT extends AgentCommandTestBase
describe(outfile.absolutePath)
- def result = new JsonSlurper().parseText(outfile.text)
+ def text = outfile.text
+ log.info("Registry data\n$text")
+
+ def result = new JsonSlurper().parseText(text)
Map jsonResult = (Map) result
List host_ports = (List)jsonResult.get("host_port")
Map host_port = (Map)host_ports[0]
@@ -111,13 +115,16 @@ public class AgentPingSocketIT extends AgentCommandTestBase
def host = tokens[0]
def port = tokens[1].toInteger()
+ def socket = null
try {
- def socket = new Socket();
+ socket = new Socket();
def addr = new InetSocketAddress(host, port)
socket.connect(addr, 2000)
socket.close()
} catch (IOException e) {
throw NetUtils.wrapException(host, port, "localhost", 0, e)
+ } finally {
+ IOUtils.closeSocket(socket)
}
//stop
[3/3] incubator-slider git commit: SLIDER-1033
AMClientCertStoreRetrievalIT.testRetrieveCertificateStores failing
Posted by st...@apache.org.
SLIDER-1033 AMClientCertStoreRetrievalIT.testRetrieveCertificateStores failing
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8ee057a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8ee057a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8ee057a0
Branch: refs/heads/develop
Commit: 8ee057a0547cefb41287204e3af74649bed54a34
Parents: d497731
Author: Steve Loughran <st...@apache.org>
Authored: Thu Dec 17 19:41:24 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Dec 17 19:41:24 2015 +0000
----------------------------------------------------------------------
.../AMClientCertStoreRetrievalIT.groovy | 42 ++++++++++----------
1 file changed, 21 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ee057a0/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMClientCertStoreRetrievalIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMClientCertStoreRetrievalIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMClientCertStoreRetrievalIT.groovy
index 05adebd..d3ff43f 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMClientCertStoreRetrievalIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMClientCertStoreRetrievalIT.groovy
@@ -32,6 +32,7 @@ import org.apache.slider.common.params.SliderActions
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
+import org.apache.slider.test.ContractTestUtils
import org.junit.After
import org.junit.Test
@@ -87,7 +88,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
launchReportFile)
logShell(shell)
- def appId = ensureYarnApplicationIsUp(launchReportFile)
+ ensureYarnApplicationIsUp(launchReportFile)
expectContainerRequestedCountReached(APPLICATION_NAME, COMMAND_LOGGER, 1,
CONTAINER_LAUNCH_TIMEOUT)
@@ -100,50 +101,50 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
assert loggerStats["containers.requested"] == 1
assert loggerStats["containers.live"] == 1
- File myTempDir = Files.createTempDir();
- String filename = myTempDir.canonicalPath + File.separator + "test.keystore"
- String password = "welcome";
+ final File myTempDir = Files.createTempDir();
+ final String keystoreName = myTempDir.canonicalPath + File.separator + "test.keystore"
+ final String password = "welcome";
// ensure file doesn't exist
- def keystoreFile = new File(filename)
+ def keystoreFile = new File(keystoreName)
keystoreFile.delete();
shell = slider(EXIT_SUCCESS,
[
ACTION_CLIENT,
ARG_GETCERTSTORE,
- ARG_KEYSTORE, filename,
+ ARG_KEYSTORE, keystoreName,
ARG_NAME, APPLICATION_NAME,
ARG_PASSWORD, password
])
assert keystoreFile.exists()
- KeyStore keystore = loadKeystoreFromFile(filename, password.toCharArray())
+ KeyStore keystore = loadKeystoreFromFile(keystoreName, password.toCharArray())
validateKeystore(keystore)
- filename = myTempDir.canonicalPath + File.separator + "test.truststore"
+ final def truststoreName = myTempDir.canonicalPath + File.separator + "test.truststore"
// ensure file doesn't exist
- keystoreFile.delete();
+ final def trustStoreFile = new File(truststoreName)
+ trustStoreFile.delete();
shell = slider(EXIT_SUCCESS,
[
ACTION_CLIENT,
ARG_GETCERTSTORE,
- ARG_TRUSTSTORE, filename,
+ ARG_TRUSTSTORE, truststoreName,
ARG_NAME, APPLICATION_NAME,
ARG_PASSWORD, password
])
- assert keystoreFile.exists()
+ assert trustStoreFile.exists()
- KeyStore truststore = loadKeystoreFromFile(filename, password.toCharArray())
+ KeyStore truststore = loadKeystoreFromFile(truststoreName, password.toCharArray())
validateTruststore(keystore, truststore);
// test retrieving using credential provider to provide password
- filename = myTempDir.canonicalPath + File.separator + "test.keystore"
String alias = "alias.for.password"
String providerString = "jceks://hdfs/user/" +
UserGroupInformation.getCurrentUser().getShortUserName() + "/test-" +
@@ -159,7 +160,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
CredentialProviderFactory.getProviders(conf).get(0)
provider.createCredentialEntry(alias, password.toCharArray())
provider.flush()
- assert clusterFS.exists(providerPath), "jks $providerString not created"
+ ContractTestUtils.assertPathExists(clusterFS, "jks $providerString not created", providerPath)
log.info("Created credential provider $providerString for test")
// ensure file doesn't exist
@@ -169,7 +170,7 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
[
ACTION_CLIENT,
ARG_GETCERTSTORE,
- ARG_KEYSTORE, filename,
+ ARG_KEYSTORE, keystoreName,
ARG_NAME, APPLICATION_NAME,
ARG_ALIAS, alias,
ARG_PROVIDER, providerString
@@ -177,27 +178,26 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
assert keystoreFile.exists()
- keystore = loadKeystoreFromFile(filename, password.toCharArray())
+ keystore = loadKeystoreFromFile(keystoreName, password.toCharArray())
validateKeystore(keystore)
- filename = myTempDir.canonicalPath + File.separator + "test.truststore"
// ensure file doesn't exist
- keystoreFile.delete();
+ trustStoreFile.delete();
shell = slider(EXIT_SUCCESS,
[
ACTION_CLIENT,
ARG_GETCERTSTORE,
- ARG_TRUSTSTORE, filename,
+ ARG_TRUSTSTORE, truststoreName,
ARG_NAME, APPLICATION_NAME,
ARG_ALIAS, alias,
ARG_PROVIDER, providerString
])
- assert keystoreFile.exists()
+ assert trustStoreFile.exists()
- truststore = loadKeystoreFromFile(filename, password.toCharArray())
+ truststore = loadKeystoreFromFile(truststoreName, password.toCharArray())
validateTruststore(keystore, truststore);
[2/3] incubator-slider git commit: SLIDER-1032
ApplicationWithAddonPackagesIT failing (against windows)
Posted by st...@apache.org.
SLIDER-1032 ApplicationWithAddonPackagesIT failing (against windows)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d497731e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d497731e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d497731e
Branch: refs/heads/develop
Commit: d497731e4fa33640838d75bbe922e2974f240dad
Parents: 520f644
Author: Steve Loughran <st...@apache.org>
Authored: Thu Dec 17 19:17:21 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Dec 17 19:17:21 2015 +0000
----------------------------------------------------------------------
.../apache/slider/test/ContractTestUtils.java | 901 +++++++++++++++++++
.../framework/AgentCommandTestBase.groovy | 70 +-
...nentConfigsInAppConfigShowUpOnAgentIT.groovy | 2 +-
.../ApplicationWithAddonPackagesIT.groovy | 67 +-
4 files changed, 990 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d497731e/slider-core/src/test/java/org/apache/slider/test/ContractTestUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/test/ContractTestUtils.java b/slider-core/src/test/java/org/apache/slider/test/ContractTestUtils.java
new file mode 100644
index 0000000..7eaaefe
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/test/ContractTestUtils.java
@@ -0,0 +1,901 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.test;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.Assert;
+import org.junit.internal.AssumptionViolatedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.Properties;
+import java.util.UUID;
+
+/**
+ * Utilities used across test cases to make assertions about filesystems
+ * -assertions which fail with useful information.
+ * This is lifted from Hadoop common Test; that JAR isn't published, so
+ * we have to make do.
+ */
+public class ContractTestUtils extends Assert {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContractTestUtils.class);
+
+ public static final String IO_FILE_BUFFER_SIZE = "io.file.buffer.size";
+
+ // For scale testing, we can repeatedly write small chunk data to generate
+ // a large file.
+ public static final String IO_CHUNK_BUFFER_SIZE = "io.chunk.buffer.size";
+ public static final int DEFAULT_IO_CHUNK_BUFFER_SIZE = 128;
+ public static final String IO_CHUNK_MODULUS_SIZE = "io.chunk.modulus.size";
+ public static final int DEFAULT_IO_CHUNK_MODULUS_SIZE = 128;
+
+ /**
+ * Assert that a property in the property set matches the expected value
+ * @param props property set
+ * @param key property name
+ * @param expected expected value. If null, the property must not be in the set
+ */
+ public static void assertPropertyEquals(Properties props,
+ String key,
+ String expected) {
+ String val = props.getProperty(key);
+ if (expected == null) {
+ assertNull("Non null property " + key + " = " + val, val);
+ } else {
+ assertEquals("property " + key + " = " + val,
+ expected,
+ val);
+ }
+ }
+
+ /**
+ *
+ * Write a file and read it in, validating the result. Optional flags control
+ * whether file overwrite operations should be enabled, and whether the
+ * file should be deleted afterwards.
+ *
+ * If there is a mismatch between what was written and what was expected,
+ * a small range of bytes either side of the first error are logged to aid
+ * diagnosing what problem occurred -whether it was a previous file
+ * or a corrupting of the current file. This assumes that two
+ * sequential runs to the same path use datasets with different character
+ * moduli.
+ *
+ * @param fs filesystem
+ * @param path path to write to
+ * @param len length of data
+ * @param overwrite should the create option allow overwrites?
+ * @param delete should the file be deleted afterwards? -with a verification
+ * that it worked. Deletion is not attempted if an assertion has failed
+ * earlier -it is not in a <code>finally{}</code> block.
+ * @throws IOException IO problems
+ */
+ public static void writeAndRead(FileSystem fs,
+ Path path,
+ byte[] src,
+ int len,
+ int blocksize,
+ boolean overwrite,
+ boolean delete) throws IOException {
+ fs.mkdirs(path.getParent());
+
+ writeDataset(fs, path, src, len, blocksize, overwrite);
+
+ byte[] dest = readDataset(fs, path, len);
+
+ compareByteArrays(src, dest, len);
+
+ if (delete) {
+ rejectRootOperation(path);
+ boolean deleted = fs.delete(path, false);
+ assertTrue("Deleted", deleted);
+ assertPathDoesNotExist(fs, "Cleanup failed", path);
+ }
+ }
+
+ /**
+ * Write a file.
+ * Optional flags control
+ * whether file overwrite operations should be enabled
+ * @param fs filesystem
+ * @param path path to write to
+ * @param len length of data
+ * @param overwrite should the create option allow overwrites?
+ * @throws IOException IO problems
+ */
+ public static void writeDataset(FileSystem fs,
+ Path path,
+ byte[] src,
+ int len,
+ int buffersize,
+ boolean overwrite) throws IOException {
+ assertTrue(
+ "Not enough data in source array to write " + len + " bytes",
+ src.length >= len);
+ FSDataOutputStream out = fs.create(path,
+ overwrite,
+ fs.getConf()
+ .getInt(IO_FILE_BUFFER_SIZE,
+ 4096),
+ (short) 1,
+ buffersize);
+ out.write(src, 0, len);
+ out.close();
+ assertFileHasLength(fs, path, len);
+ }
+
+ /**
+ * Read the file and convert to a byte dataset.
+ * This implements readfully internally, so that it will read
+ * in the file without ever having to seek()
+ * @param fs filesystem
+ * @param path path to read from
+ * @param len length of data to read
+ * @return the bytes
+ * @throws IOException IO problems
+ */
+ public static byte[] readDataset(FileSystem fs, Path path, int len)
+ throws IOException {
+ FSDataInputStream in = fs.open(path);
+ byte[] dest = new byte[len];
+ int offset =0;
+ int nread = 0;
+ try {
+ while (nread < len) {
+ int nbytes = in.read(dest, offset + nread, len - nread);
+ if (nbytes < 0) {
+ throw new EOFException("End of file reached before reading fully.");
+ }
+ nread += nbytes;
+ }
+ } finally {
+ in.close();
+ }
+ return dest;
+ }
+
+ /**
+ * Read a file, verify its length and contents match the expected array
+ * @param fs filesystem
+ * @param path path to file
+ * @param original original dataset
+ * @throws IOException IO Problems
+ */
+ public static void verifyFileContents(FileSystem fs,
+ Path path,
+ byte[] original) throws IOException {
+ FileStatus stat = fs.getFileStatus(path);
+ String statText = stat.toString();
+ assertTrue("not a file " + statText, stat.isFile());
+ assertEquals("wrong length " + statText, original.length, stat.getLen());
+ byte[] bytes = readDataset(fs, path, original.length);
+ compareByteArrays(original,bytes,original.length);
+ }
+
+ /**
+ * Verify that the read at a specific offset in a stream
+ * matches that expected
+ * @param stm stream
+ * @param fileContents original file contents
+ * @param seekOff seek offset
+ * @param toRead number of bytes to read
+ * @throws IOException IO problems
+ */
+ public static void verifyRead(FSDataInputStream stm, byte[] fileContents,
+ int seekOff, int toRead) throws IOException {
+ byte[] out = new byte[toRead];
+ stm.seek(seekOff);
+ stm.readFully(out);
+ byte[] expected = Arrays.copyOfRange(fileContents, seekOff,
+ seekOff + toRead);
+ compareByteArrays(expected, out,toRead);
+ }
+
+ /**
+ * Assert that tthe array original[0..len] and received[] are equal.
+ * A failure triggers the logging of the bytes near where the first
+ * difference surfaces.
+ * @param original source data
+ * @param received actual
+ * @param len length of bytes to compare
+ */
+ public static void compareByteArrays(byte[] original,
+ byte[] received,
+ int len) {
+ assertEquals("Number of bytes read != number written",
+ len, received.length);
+ int errors = 0;
+ int first_error_byte = -1;
+ for (int i = 0; i < len; i++) {
+ if (original[i] != received[i]) {
+ if (errors == 0) {
+ first_error_byte = i;
+ }
+ errors++;
+ }
+ }
+
+ if (errors > 0) {
+ String message = String.format(" %d errors in file of length %d",
+ errors, len);
+ LOG.warn(message);
+ // the range either side of the first error to print
+ // this is a purely arbitrary number, to aid user debugging
+ final int overlap = 10;
+ for (int i = Math.max(0, first_error_byte - overlap);
+ i < Math.min(first_error_byte + overlap, len);
+ i++) {
+ byte actual = received[i];
+ byte expected = original[i];
+ String letter = toChar(actual);
+ String line = String.format("[%04d] %2x %s\n", i, actual, letter);
+ if (expected != actual) {
+ line = String.format("[%04d] %2x %s -expected %2x %s\n",
+ i,
+ actual,
+ letter,
+ expected,
+ toChar(expected));
+ }
+ LOG.warn(line);
+ }
+ fail(message);
+ }
+ }
+
+ /**
+ * Convert a byte to a character for printing. If the
+ * byte value is < 32 -and hence unprintable- the byte is
+ * returned as a two digit hex value
+ * @param b byte
+ * @return the printable character string
+ */
+ public static String toChar(byte b) {
+ if (b >= 0x20) {
+ return Character.toString((char) b);
+ } else {
+ return String.format("%02x", b);
+ }
+ }
+
+ /**
+ * Convert a buffer to a string, character by character
+ * @param buffer input bytes
+ * @return a string conversion
+ */
+ public static String toChar(byte[] buffer) {
+ StringBuilder builder = new StringBuilder(buffer.length);
+ for (byte b : buffer) {
+ builder.append(toChar(b));
+ }
+ return builder.toString();
+ }
+
+ public static byte[] toAsciiByteArray(String s) {
+ char[] chars = s.toCharArray();
+ int len = chars.length;
+ byte[] buffer = new byte[len];
+ for (int i = 0; i < len; i++) {
+ buffer[i] = (byte) (chars[i] & 0xff);
+ }
+ return buffer;
+ }
+
+ /**
+ * Cleanup at the end of a test run
+ * @param action action triggering the operation (for use in logging)
+ * @param fileSystem filesystem to work with. May be null
+ * @param cleanupPath path to delete as a string
+ */
+ public static void cleanup(String action,
+ FileSystem fileSystem,
+ String cleanupPath) {
+ if (fileSystem == null) {
+ return;
+ }
+ Path path = new Path(cleanupPath).makeQualified(fileSystem.getUri(),
+ fileSystem.getWorkingDirectory());
+ cleanup(action, fileSystem, path);
+ }
+
+ /**
+ * Cleanup at the end of a test run
+ * @param action action triggering the operation (for use in logging)
+ * @param fileSystem filesystem to work with. May be null
+ * @param path path to delete
+ */
+ public static void cleanup(String action, FileSystem fileSystem, Path path) {
+ noteAction(action);
+ try {
+ rm(fileSystem, path, true, false);
+ } catch (Exception e) {
+ LOG.error("Error deleting in "+ action + " - " + path + ": " + e, e);
+ }
+ }
+
+ /**
+ * Delete a directory. There's a safety check for operations against the
+ * root directory -these are intercepted and rejected with an IOException
+ * unless the allowRootDelete flag is true
+ * @param fileSystem filesystem to work with. May be null
+ * @param path path to delete
+ * @param recursive flag to enable recursive delete
+ * @param allowRootDelete can the root directory be deleted?
+ * @throws IOException on any problem.
+ */
+ public static boolean rm(FileSystem fileSystem,
+ Path path,
+ boolean recursive,
+ boolean allowRootDelete) throws
+ IOException {
+ if (fileSystem != null) {
+ rejectRootOperation(path, allowRootDelete);
+ if (fileSystem.exists(path)) {
+ return fileSystem.delete(path, recursive);
+ }
+ }
+ return false;
+
+ }
+
+ /**
+ * Block any operation on the root path. This is a safety check
+ * @param path path in the filesystem
+ * @param allowRootOperation can the root directory be manipulated?
+ * @throws IOException if the operation was rejected
+ */
+ public static void rejectRootOperation(Path path,
+ boolean allowRootOperation) throws IOException {
+ if (path.isRoot() && !allowRootOperation) {
+ throw new IOException("Root directory operation rejected: " + path);
+ }
+ }
+
+ /**
+ * Block any operation on the root path. This is a safety check
+ * @param path path in the filesystem
+ * @throws IOException if the operation was rejected
+ */
+ public static void rejectRootOperation(Path path) throws IOException {
+ rejectRootOperation(path, false);
+ }
+
+
+ public static void noteAction(String action) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("============== "+ action +" =============");
+ }
+ }
+
+ /**
+ * downgrade a failure to a message and a warning, then an
+ * exception for the Junit test runner to mark as failed
+ * @param message text message
+ * @param failure what failed
+ * @throws AssumptionViolatedException always
+ */
+ public static void downgrade(String message, Throwable failure) {
+ LOG.warn("Downgrading test " + message, failure);
+ AssumptionViolatedException ave =
+ new AssumptionViolatedException(failure, null);
+ throw ave;
+ }
+
+ /**
+ * report an overridden test as unsupported
+ * @param message message to use in the text
+ * @throws AssumptionViolatedException always
+ */
+ public static void unsupported(String message) {
+ skip(message);
+ }
+
+ /**
+ * report a test has been skipped for some reason
+ * @param message message to use in the text
+ * @throws AssumptionViolatedException always
+ */
+ public static void skip(String message) {
+ LOG.info("Skipping: {}", message);
+ throw new AssumptionViolatedException(message);
+ }
+
+ /**
+ * Fail with an exception that was received
+ * @param text text to use in the exception
+ * @param thrown a (possibly null) throwable to init the cause with
+ * @throws AssertionError with the text and throwable -always
+ */
+ public static void fail(String text, Throwable thrown) {
+ AssertionError e = new AssertionError(text);
+ e.initCause(thrown);
+ throw e;
+ }
+
+ /**
+ * Make an assertion about the length of a file
+ * @param fs filesystem
+ * @param path path of the file
+ * @param expected expected length
+ * @throws IOException on File IO problems
+ */
+ public static void assertFileHasLength(FileSystem fs, Path path,
+ int expected) throws IOException {
+ FileStatus status = fs.getFileStatus(path);
+ assertEquals(
+ "Wrong file length of file " + path + " status: " + status,
+ expected,
+ status.getLen());
+ }
+
+ /**
+ * Assert that a path refers to a directory
+ * @param fs filesystem
+ * @param path path of the directory
+ * @throws IOException on File IO problems
+ */
+ public static void assertIsDirectory(FileSystem fs,
+ Path path) throws IOException {
+ FileStatus fileStatus = fs.getFileStatus(path);
+ assertIsDirectory(fileStatus);
+ }
+
+ /**
+ * Assert that a path refers to a directory
+ * @param fileStatus stats to check
+ */
+ public static void assertIsDirectory(FileStatus fileStatus) {
+ assertTrue("Should be a directory -but isn't: " + fileStatus,
+ fileStatus.isDirectory());
+ }
+
+ /**
+ * Write the text to a file, returning the converted byte array
+ * for use in validating the round trip
+ * @param fs filesystem
+ * @param path path of file
+ * @param text text to write
+ * @param overwrite should the operation overwrite any existing file?
+ * @return the read bytes
+ * @throws IOException on IO problems
+ */
+ public static byte[] writeTextFile(FileSystem fs,
+ Path path,
+ String text,
+ boolean overwrite) throws IOException {
+ byte[] bytes = new byte[0];
+ if (text != null) {
+ bytes = toAsciiByteArray(text);
+ }
+ createFile(fs, path, overwrite, bytes);
+ return bytes;
+ }
+
+ /**
+ * Create a file
+ * @param fs filesystem
+ * @param path path to write
+ * @param overwrite overwrite flag
+ * @param data source dataset. Can be null
+ * @throws IOException on any problem
+ */
+ public static void createFile(FileSystem fs,
+ Path path,
+ boolean overwrite,
+ byte[] data) throws IOException {
+ FSDataOutputStream stream = fs.create(path, overwrite);
+ if (data != null && data.length > 0) {
+ stream.write(data);
+ }
+ stream.close();
+ }
+
+ /**
+ * Touch a file
+ * @param fs filesystem
+ * @param path path
+ * @throws IOException IO problems
+ */
+ public static void touch(FileSystem fs,
+ Path path) throws IOException {
+ createFile(fs, path, true, null);
+ }
+
+ /**
+ * Delete a file/dir and assert that delete() returned true
+ * <i>and</i> that the path no longer exists. This variant rejects
+ * all operations on root directories
+ * @param fs filesystem
+ * @param file path to delete
+ * @param recursive flag to enable recursive delete
+ * @throws IOException IO problems
+ */
+ public static void assertDeleted(FileSystem fs,
+ Path file,
+ boolean recursive) throws IOException {
+ assertDeleted(fs, file, recursive, false);
+ }
+
+ /**
+ * Delete a file/dir and assert that delete() returned true
+ * <i>and</i> that the path no longer exists. This variant rejects
+ * all operations on root directories
+ * @param fs filesystem
+ * @param file path to delete
+ * @param recursive flag to enable recursive delete
+ * @param allowRootOperations can the root dir be deleted?
+ * @throws IOException IO problems
+ */
+ public static void assertDeleted(FileSystem fs,
+ Path file,
+ boolean recursive,
+ boolean allowRootOperations) throws IOException {
+ rejectRootOperation(file, allowRootOperations);
+ assertPathExists(fs, "about to be deleted file", file);
+ boolean deleted = fs.delete(file, recursive);
+ String dir = ls(fs, file.getParent());
+ assertTrue("Delete failed on " + file + ": " + dir, deleted);
+ assertPathDoesNotExist(fs, "Deleted file", file);
+ }
+
+ /**
+ * Read in "length" bytes, convert to an ascii string
+ * @param fs filesystem
+ * @param path path to read
+ * @param length #of bytes to read.
+ * @return the bytes read and converted to a string
+ * @throws IOException IO problems
+ */
+ public static String readBytesToString(FileSystem fs,
+ Path path,
+ int length) throws IOException {
+ FSDataInputStream in = fs.open(path);
+ try {
+ byte[] buf = new byte[length];
+ in.readFully(0, buf);
+ return toChar(buf);
+ } finally {
+ in.close();
+ }
+ }
+
+ /**
+ * Take an array of filestats and convert to a string (prefixed w/ a [01] counter
+ * @param stats array of stats
+ * @param separator separator after every entry
+ * @return a stringified set
+ */
+ public static String fileStatsToString(FileStatus[] stats, String separator) {
+ StringBuilder buf = new StringBuilder(stats.length * 128);
+ for (int i = 0; i < stats.length; i++) {
+ buf.append(String.format("[%02d] %s", i, stats[i])).append(separator);
+ }
+ return buf.toString();
+ }
+
+ /**
+ * List a directory
+ * @param fileSystem FS
+ * @param path path
+ * @return a directory listing or failure message
+ * @throws IOException
+ */
+ public static String ls(FileSystem fileSystem, Path path) throws IOException {
+ if (path == null) {
+ //surfaces when someone calls getParent() on something at the top of the path
+ return "/";
+ }
+ FileStatus[] stats;
+ String pathtext = "ls " + path;
+ try {
+ stats = fileSystem.listStatus(path);
+ } catch (FileNotFoundException e) {
+ return pathtext + " -file not found";
+ } catch (IOException e) {
+ return pathtext + " -failed: " + e;
+ }
+ return dumpStats(pathtext, stats);
+ }
+
+ public static String dumpStats(String pathname, FileStatus[] stats) {
+ return pathname + fileStatsToString(stats, "\n");
+ }
+
+ /**
+ * Assert that a file exists and whose {@link FileStatus} entry
+ * declares that this is a file and not a symlink or directory.
+ * @param fileSystem filesystem to resolve path against
+ * @param filename name of the file
+ * @throws IOException IO problems during file operations
+ */
+ public static void assertIsFile(FileSystem fileSystem, Path filename) throws
+ IOException {
+ assertPathExists(fileSystem, "Expected file", filename);
+ FileStatus status = fileSystem.getFileStatus(filename);
+ assertIsFile(filename, status);
+ }
+
+ /**
+ * Assert that a file exists and whose {@link FileStatus} entry
+ * declares that this is a file and not a symlink or directory.
+ * @param filename name of the file
+ * @param status file status
+ */
+ public static void assertIsFile(Path filename, FileStatus status) {
+ String fileInfo = filename + " " + status;
+ assertFalse("File claims to be a directory " + fileInfo,
+ status.isDirectory());
+ assertFalse("File claims to be a symlink " + fileInfo,
+ status.isSymlink());
+ }
+
+ /**
+ * Create a dataset for use in the tests; all data is in the range
+ * base to (base+modulo-1) inclusive
+ * @param len length of data
+ * @param base base of the data
+ * @param modulo the modulo
+ * @return the newly generated dataset
+ */
+ public static byte[] dataset(int len, int base, int modulo) {
+ byte[] dataset = new byte[len];
+ for (int i = 0; i < len; i++) {
+ dataset[i] = (byte) (base + (i % modulo));
+ }
+ return dataset;
+ }
+
+ /**
+ * Assert that a path exists -but make no assertions as to the
+ * type of that entry
+ *
+ * @param fileSystem filesystem to examine
+ * @param message message to include in the assertion failure message
+ * @param path path in the filesystem
+ * @throws FileNotFoundException raised if the path is missing
+ * @throws IOException IO problems
+ */
+ public static void assertPathExists(FileSystem fileSystem, String message,
+ Path path) throws IOException {
+ if (!fileSystem.exists(path)) {
+ //failure, report it
+ String listing = ls(fileSystem, path.getParent());
+ throw new FileNotFoundException(message + ": not found " + path
+ + " in \"" + path.getParent() + "\" :\n" + listing);
+ }
+ }
+
+ /**
+ * Assert that a path does not exist
+ *
+ * @param fileSystem filesystem to examine
+ * @param message message to include in the assertion failure message
+ * @param path path in the filesystem
+ * @throws IOException IO problems
+ */
+ public static void assertPathDoesNotExist(FileSystem fileSystem,
+ String message,
+ Path path) throws IOException {
+ try {
+ FileStatus status = fileSystem.getFileStatus(path);
+ fail(message + ": unexpectedly found " + path + " as " + status);
+ } catch (FileNotFoundException expected) {
+ //this is expected
+
+ }
+ }
+
+ /**
+ * Assert that a FileSystem.listStatus on a dir finds the subdir/child entry
+ * @param fs filesystem
+ * @param dir directory to scan
+ * @param subdir full path to look for
+ * @throws IOException IO probles
+ */
+ public static void assertListStatusFinds(FileSystem fs,
+ Path dir,
+ Path subdir) throws IOException {
+ FileStatus[] stats = fs.listStatus(dir);
+ boolean found = false;
+ StringBuilder builder = new StringBuilder();
+ for (FileStatus stat : stats) {
+ builder.append(stat.toString()).append('\n');
+ if (stat.getPath().equals(subdir)) {
+ found = true;
+ }
+ }
+ assertTrue("Path " + subdir
+ + " not found in directory " + dir + ":" + builder,
+ found);
+ }
+
+ /**
+ * Test for the host being an OSX machine
+ * @return true if the JVM thinks that is running on OSX
+ */
+ public static boolean isOSX() {
+ return System.getProperty("os.name").contains("OS X");
+ }
+
+ /**
+ * compare content of file operations using a double byte array
+ * @param concat concatenated files
+ * @param bytes bytes
+ */
+ public static void validateFileContent(byte[] concat, byte[][] bytes) {
+ int idx = 0;
+ boolean mismatch = false;
+
+ for (byte[] bb : bytes) {
+ for (byte b : bb) {
+ if (b != concat[idx++]) {
+ mismatch = true;
+ break;
+ }
+ }
+ if (mismatch)
+ break;
+ }
+ assertFalse("File content of file is not as expected at offset " + idx,
+ mismatch);
+ }
+
+ /**
+ * Receives test data from the given input file and checks the size of the
+ * data as well as the pattern inside the received data.
+ *
+ * @param fs FileSystem
+ * @param path Input file to be checked
+ * @param expectedSize the expected size of the data to be read from the
+ * input file in bytes
+ * @param bufferLen Pattern length
+ * @param modulus Pattern modulus
+ * @throws IOException
+ * thrown if an error occurs while reading the data
+ */
+ public static void verifyReceivedData(FileSystem fs, Path path,
+ final long expectedSize,
+ final int bufferLen,
+ final int modulus) throws IOException {
+ final byte[] testBuffer = new byte[bufferLen];
+
+ long totalBytesRead = 0;
+ int nextExpectedNumber = 0;
+ final InputStream inputStream = fs.open(path);
+ try {
+ while (true) {
+ final int bytesRead = inputStream.read(testBuffer);
+ if (bytesRead < 0) {
+ break;
+ }
+
+ totalBytesRead += bytesRead;
+
+ for (int i = 0; i < bytesRead; ++i) {
+ if (testBuffer[i] != nextExpectedNumber) {
+ throw new IOException("Read number " + testBuffer[i]
+ + " but expected " + nextExpectedNumber);
+ }
+
+ ++nextExpectedNumber;
+
+ if (nextExpectedNumber == modulus) {
+ nextExpectedNumber = 0;
+ }
+ }
+ }
+
+ if (totalBytesRead != expectedSize) {
+ throw new IOException("Expected to read " + expectedSize +
+ " bytes but only received " + totalBytesRead);
+ }
+ } finally {
+ inputStream.close();
+ }
+ }
+
+ /**
+ * Generates test data of the given size according to some specific pattern
+ * and writes it to the provided output file.
+ *
+ * @param fs FileSystem
+ * @param path Test file to be generated
+ * @param size The size of the test data to be generated in bytes
+ * @param bufferLen Pattern length
+ * @param modulus Pattern modulus
+ * @throws IOException
+ * thrown if an error occurs while writing the data
+ */
+ public static long generateTestFile(FileSystem fs, Path path,
+ final long size,
+ final int bufferLen,
+ final int modulus) throws IOException {
+ final byte[] testBuffer = new byte[bufferLen];
+ for (int i = 0; i < testBuffer.length; ++i) {
+ testBuffer[i] = (byte) (i % modulus);
+ }
+
+ final OutputStream outputStream = fs.create(path, false);
+ long bytesWritten = 0;
+ try {
+ while (bytesWritten < size) {
+ final long diff = size - bytesWritten;
+ if (diff < testBuffer.length) {
+ outputStream.write(testBuffer, 0, (int) diff);
+ bytesWritten += diff;
+ } else {
+ outputStream.write(testBuffer);
+ bytesWritten += testBuffer.length;
+ }
+ }
+
+ return bytesWritten;
+ } finally {
+ outputStream.close();
+ }
+ }
+
+ /**
+ * Creates and reads a file with the given size. The test file is generated
+ * according to a specific pattern so it can be easily verified even if it's
+ * a multi-GB one.
+ * During the read phase the incoming data stream is also checked against
+ * this pattern.
+ *
+ * @param fs FileSystem
+ * @param parent Test file parent dir path
+ * @throws IOException
+ * thrown if an I/O error occurs while writing or reading the test file
+ */
+ public static void createAndVerifyFile(FileSystem fs, Path parent, final long fileSize)
+ throws IOException {
+ int testBufferSize = fs.getConf()
+ .getInt(IO_CHUNK_BUFFER_SIZE, DEFAULT_IO_CHUNK_BUFFER_SIZE);
+ int modulus = fs.getConf()
+ .getInt(IO_CHUNK_MODULUS_SIZE, DEFAULT_IO_CHUNK_MODULUS_SIZE);
+
+ final String objectName = UUID.randomUUID().toString();
+ final Path objectPath = new Path(parent, objectName);
+
+ // Write test file in a specific pattern
+ assertEquals(fileSize,
+ generateTestFile(fs, objectPath, fileSize, testBufferSize, modulus));
+ assertPathExists(fs, "not created successful", objectPath);
+
+ // Now read the same file back and verify its content
+ try {
+ verifyReceivedData(fs, objectPath, fileSize, testBufferSize, modulus);
+ } finally {
+ // Delete test file
+ fs.delete(objectPath, false);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d497731e/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
index 12890b7..ec4b03a 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
@@ -19,13 +19,15 @@
package org.apache.slider.funtest.framework
import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.FileSystem
import org.apache.hadoop.fs.Path
import org.apache.hadoop.security.UserGroupInformation
import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.hadoop.fs.FileSystem;
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.test.ContractTestUtils
+import org.apache.slider.test.Outcome
import org.apache.tools.zip.ZipEntry
import org.apache.tools.zip.ZipOutputStream
import org.junit.Before
@@ -151,30 +153,52 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
}
}
}
-
- protected static void verifyFileExist(String filePath){
- try{
- Path pt = new Path(filePath);
- def uploader = new FileUploader(SLIDER_CONFIG,
- UserGroupInformation.currentUser)
- FileSystem fs = uploader.getFileSystem();
- assert fs.exists(pt);
- }catch(IOException e){
- log.error("IOException during verifying file exist " + e.toString());
+
+ /**
+ * Probe callback for the the app root web page up.
+ * Raising an SSL exception is considered a sign of liveness.
+ * @param args map where 'applicationId' must be set
+ * @return the outcome
+ */
+ protected static Outcome doesFileExist(
+ Map<String, String> args) {
+ return fileExists(args['filename']) ? Outcome.Success : Outcome.Retry
+ }
+
+ /**
+ * Await for a file to exist
+ * @param filename file to look for
+ * @param timeout launch timeout
+ */
+ void awaitFileExists(String filename, int timeout) {
+ repeatUntilSuccess(
+ "await File existence",
+ this.&doesFileExist,
+ timeout,
+ PROBE_SLEEP_TIME,
+ [
+ filename: filename
+ ],
+ true,
+ "file $filename not found") {
+ verifyFileExists(filename, "After timeout of $timeout ms")
}
}
-
- protected static void cleanupHdfsFile(String filePath){
- try{
- Path pt = new Path(filePath);
- def uploader = new FileUploader(SLIDER_CONFIG,
- UserGroupInformation.currentUser)
- FileSystem fs = uploader.getFileSystem();
- if( fs.exists(pt)){
- fs.delete(pt, false);
- }
- }catch(IOException e){
- log.error("IOException during deleting file: " + e.toString());
+
+ protected static boolean fileExists(String filePath) {
+ assert filePath
+ clusterFS.exists(new Path(filePath));
+ }
+
+ protected static void verifyFileExists(String filePath, String message = "") {
+ ContractTestUtils.assertPathExists(clusterFS, message, new Path(filePath))
+ }
+
+ protected static void cleanupHdfsFile(String filePath) {
+ Path pt = new Path(filePath);
+ FileSystem fs = clusterFS
+ if (fs.exists(pt)) {
+ fs.delete(pt, false);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d497731e/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ComponentConfigsInAppConfigShowUpOnAgentIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ComponentConfigsInAppConfigShowUpOnAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ComponentConfigsInAppConfigShowUpOnAgentIT.groovy
index 91797f9..0d0eab2 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ComponentConfigsInAppConfigShowUpOnAgentIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ComponentConfigsInAppConfigShowUpOnAgentIT.groovy
@@ -78,7 +78,7 @@ public class ComponentConfigsInAppConfigShowUpOnAgentIT extends AgentCommandTest
list(0, [ARG_STATE, "running"])
status(0, CLUSTER)
Thread.sleep(10000)
- verifyFileExist(TARGET_FILE)
+ verifyFileExists(TARGET_FILE)
}
public void setupApplicationPackage() {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d497731e/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy
index b32275e..66572ba 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy
@@ -18,11 +18,14 @@ package org.apache.slider.funtest.coprocessors
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.Path
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.funtest.framework.SliderShell
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.CommandTestBase
+import org.apache.slider.test.ContractTestUtils
+import org.apache.slider.test.Outcome
import org.junit.After
import org.junit.Before
import org.junit.Test
@@ -46,12 +49,15 @@ public class ApplicationWithAddonPackagesIT extends AgentCommandTestBase{
static String ADD_ON_PACKAGE_NO_COMPONENT_PKG_NAME = "add-on-package-apply-on-no-component"
static String ADD_ON_PACKAGE_NO_COMPONENT_PKG_FILE = "target/package-tmp/add-on-package-apply-on-no-component.zip"
static String TARGET_FILE = "/tmp/test_slider.txt"
+ public static final int FILE_EXISTS_TIMEOUT = 20000
protected String APP_RESOURCE = getAppResource()
protected String APP_TEMPLATE = getAppTemplate()
@Before
public void prepareCluster() {
setupCluster(CLUSTER)
+ cleanupHdfsFile(TARGET_FILE)
+ clusterFS.mkdirs(new Path(TARGET_FILE).getParent())
}
@After
@@ -86,30 +92,35 @@ public class ApplicationWithAddonPackagesIT extends AgentCommandTestBase{
list(0, [ARG_LIVE])
list(0, [ARG_STATE, "running"])
status(0, CLUSTER)
- Thread.sleep(10000)
- verifyFileExist(TARGET_FILE)
+ awaitTargetFileExists()
}
-
+
+ protected void awaitTargetFileExists() {
+ awaitFileExists(TARGET_FILE, FILE_EXISTS_TIMEOUT)
+ }
+
@Test
public void testCreateApplicationWithOneAddonPackagesForNoComponents() throws Throwable {
describe("Create a cluster with an addon package that apply to no components")
SliderUtils.zipFolder(new File(ADD_ON_PACKAGE_NO_COMPONENT), new File(ADD_ON_PACKAGE_NO_COMPONENT_PKG_FILE))
def clusterpath = buildClusterPath(CLUSTER)
File launchReportFile = createTempJsonFile();
- cleanupHdfsFile(TARGET_FILE)
-
+
//default waiting time too long, temporarily lower it
int temp_holder = CommandTestBase.THAW_WAIT_TIME;
CommandTestBase.THAW_WAIT_TIME = 100;
- SliderShell shell = createTemplatedSliderApplication(CLUSTER,
+ SliderShell shell
+ try {
+ shell = createTemplatedSliderApplication(CLUSTER,
APP_TEMPLATE,
APP_RESOURCE2,
[Arguments.ARG_ADDON, ADD_ON_PACKAGE_NO_COMPONENT_PKG_NAME, ADD_ON_PACKAGE_NO_COMPONENT_PKG_FILE],
launchReportFile)
- CommandTestBase.THAW_WAIT_TIME = temp_holder;
+ } finally {
+ CommandTestBase.THAW_WAIT_TIME = temp_holder;
+ }
logShell(shell)
-
Thread.sleep(10000)
//the Slider AM will fail while checking no components in metainfo.json of addon pkg
// SLIDER-897 - Disabling this flaky assert. Have to re-write the test to
@@ -124,8 +135,7 @@ public class ApplicationWithAddonPackagesIT extends AgentCommandTestBase{
SliderUtils.zipFolder(new File(ADD_ON_PACKAGE_MULTI_COMPONENT), new File(ADD_ON_PACKAGE_MULTI_COMPONENT_PKG_FILE))
def clusterpath = buildClusterPath(CLUSTER)
File launchReportFile = createTempJsonFile();
- cleanupHdfsFile(TARGET_FILE)
-
+
SliderShell shell = createTemplatedSliderApplication(CLUSTER,
APP_TEMPLATE,
APP_RESOURCE2,
@@ -144,8 +154,8 @@ public class ApplicationWithAddonPackagesIT extends AgentCommandTestBase{
list(0, [ARG_LIVE])
list(0, [ARG_STATE, "running"])
status(0, CLUSTER)
- Thread.sleep(10000)
- verifyFileExist(TARGET_FILE)
+ awaitTargetFileExists()
+
}
@Test
@@ -154,8 +164,7 @@ public class ApplicationWithAddonPackagesIT extends AgentCommandTestBase{
SliderUtils.zipFolder(new File(ADD_ON_PACKAGE_ALL_COMPONENT), new File(ADD_ON_PACKAGE_ALL_COMPONENT_PKG_FILE))
def clusterpath = buildClusterPath(CLUSTER)
File launchReportFile = createTempJsonFile();
- cleanupHdfsFile(TARGET_FILE)
-
+
SliderShell shell = createTemplatedSliderApplication(CLUSTER,
APP_TEMPLATE,
APP_RESOURCE2,
@@ -163,8 +172,7 @@ public class ApplicationWithAddonPackagesIT extends AgentCommandTestBase{
launchReportFile)
logShell(shell)
- Thread.sleep(10000);
- def appId = ensureYarnApplicationIsUp(launchReportFile)
+ ensureYarnApplicationIsUp(launchReportFile)
exists(0, CLUSTER)
list(0, [CLUSTER])
@@ -174,21 +182,17 @@ public class ApplicationWithAddonPackagesIT extends AgentCommandTestBase{
list(0, [ARG_LIVE])
list(0, [ARG_STATE, "running"])
status(0, CLUSTER)
-
- Thread.sleep(10000)
- verifyFileExist(TARGET_FILE)
+ awaitTargetFileExists()
}
-
-
+
@Test
- public void testCreateApplicationWithMultipeAddonPackages() throws Throwable {
+ public void testCreateApplicationWithMultipleAddonPackages() throws Throwable {
describe("Create a cluster with multiple addon packages")
SliderUtils.zipFolder(new File(ADD_ON_PACKAGE_ALL_COMPONENT), new File(ADD_ON_PACKAGE_ALL_COMPONENT_PKG_FILE))
SliderUtils.zipFolder(new File(ADD_ON_PACKAGE_ONE_COMPONENT), new File(ADD_ON_PACKAGE_ONE_COMPONENT_PKG_FILE))
def clusterpath = buildClusterPath(CLUSTER)
File launchReportFile = createTempJsonFile();
- cleanupHdfsFile(TARGET_FILE)
-
+
SliderShell shell = createTemplatedSliderApplication(CLUSTER,
APP_TEMPLATE,
APP_RESOURCE2,
@@ -208,8 +212,19 @@ public class ApplicationWithAddonPackagesIT extends AgentCommandTestBase{
list(0, [ARG_LIVE])
list(0, [ARG_STATE, "running"])
status(0, CLUSTER)
- Thread.sleep(10000)
- verifyFileExist(TARGET_FILE)
+
+ awaitTargetFileExists()
}
+
+ /**
+ * Here to check file permissions and isolate failures due to that alone
+ * @throws Throwable
+ */
+ @Test
+ public void testCreateTargetFile() throws Throwable {
+ ContractTestUtils.touch(clusterFS, new Path(TARGET_FILE))
+ awaitTargetFileExists()
+ }
+
}