You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cm...@apache.org on 2014/03/06 09:21:17 UTC
svn commit: r1574796 - in
/hadoop/common/trunk/hadoop-common-project/hadoop-common/src:
main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java
test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java
Author: cmccabe
Date: Thu Mar 6 08:21:16 2014
New Revision: 1574796
URL: http://svn.apache.org/r1574796
Log:
HDFS-6061. Allow dfs.datanode.shared.file.descriptor.path to contain multiple entries and fall back when needed (cmccabe)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java?rev=1574796&r1=1574795&r2=1574796&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/SharedFileDescriptorFactory.java Thu Mar 6 08:21:16 2014
@@ -22,11 +22,11 @@ import java.io.IOException;
import java.io.FileDescriptor;
import org.apache.commons.lang.SystemUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import com.google.common.base.Preconditions;
-
/**
* A factory for creating shared file descriptors inside a given directory.
* Typically, the directory will be /dev/shm or /tmp.
@@ -45,6 +45,7 @@ import com.google.common.base.Preconditi
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class SharedFileDescriptorFactory {
+ public static final Log LOG = LogFactory.getLog(SharedFileDescriptorFactory.class);
private final String prefix;
private final String path;
@@ -59,17 +60,57 @@ public class SharedFileDescriptorFactory
}
/**
+ * Create a new SharedFileDescriptorFactory.
+ *
+ * @param prefix The prefix to prepend to all the file names created
+ * by this factory.
+ * @param paths An array of paths to use. We will try each path in
+ * succession, and return a factory using the first
+ * usable path.
+ * @return The factory.
+ * @throws IOException If a factory could not be created for any reason.
+ */
+ public static SharedFileDescriptorFactory create(String prefix,
+ String paths[]) throws IOException {
+ String loadingFailureReason = getLoadingFailureReason();
+ if (loadingFailureReason != null) {
+ throw new IOException(loadingFailureReason);
+ }
+ if (paths.length == 0) {
+ throw new IOException("no SharedFileDescriptorFactory paths were " +
+ "configured.");
+ }
+ StringBuilder errors = new StringBuilder();
+ String strPrefix = "";
+ for (String path : paths) {
+ try {
+ FileInputStream fis =
+ new FileInputStream(createDescriptor0(prefix + "test", path, 1));
+ fis.close();
+ deleteStaleTemporaryFiles0(prefix, path);
+ return new SharedFileDescriptorFactory(prefix, path);
+ } catch (IOException e) {
+ errors.append(strPrefix).append("Error creating file descriptor in ").
+ append(path).append(": ").append(e.getMessage());
+ strPrefix = ", ";
+ }
+ }
+ throw new IOException(errors.toString());
+ }
+
+ /**
* Create a SharedFileDescriptorFactory.
*
* @param prefix Prefix to add to all file names we use.
* @param path Path to use.
*/
- public SharedFileDescriptorFactory(String prefix, String path)
- throws IOException {
- Preconditions.checkState(getLoadingFailureReason() == null);
+ private SharedFileDescriptorFactory(String prefix, String path) {
this.prefix = prefix;
this.path = path;
- deleteStaleTemporaryFiles0(prefix, path);
+ }
+
+ public String getPath() {
+ return path;
}
/**
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java?rev=1574796&r1=1574795&r2=1574796&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestSharedFileDescriptorFactory.java Thu Mar 6 08:21:16 2014
@@ -20,9 +20,11 @@ package org.apache.hadoop.io.nativeio;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
+import java.io.IOException;
import org.junit.Assert;
import org.junit.Assume;
+import org.junit.Before;
import org.junit.Test;
import org.apache.commons.lang.SystemUtils;
import org.apache.commons.logging.Log;
@@ -36,14 +38,19 @@ public class TestSharedFileDescriptorFac
private static final File TEST_BASE =
new File(System.getProperty("test.build.data", "/tmp"));
+ @Before
+ public void setup() throws Exception {
+ Assume.assumeTrue(null ==
+ SharedFileDescriptorFactory.getLoadingFailureReason());
+ }
+
@Test(timeout=10000)
public void testReadAndWrite() throws Exception {
- Assume.assumeTrue(NativeIO.isAvailable());
- Assume.assumeTrue(SystemUtils.IS_OS_UNIX);
File path = new File(TEST_BASE, "testReadAndWrite");
path.mkdirs();
SharedFileDescriptorFactory factory =
- new SharedFileDescriptorFactory("woot_", path.getAbsolutePath());
+ SharedFileDescriptorFactory.create("woot_",
+ new String[] { path.getAbsolutePath() });
FileInputStream inStream =
factory.createDescriptor("testReadAndWrite", 4096);
FileOutputStream outStream = new FileOutputStream(inStream.getFD());
@@ -73,11 +80,34 @@ public class TestSharedFileDescriptorFac
Path.SEPARATOR + "woot2_remainder2";
createTempFile(remainder1);
createTempFile(remainder2);
- new SharedFileDescriptorFactory("woot2_", path.getAbsolutePath());
+ SharedFileDescriptorFactory.create("woot2_",
+ new String[] { path.getAbsolutePath() });
// creating the SharedFileDescriptorFactory should have removed
// the remainders
Assert.assertFalse(new File(remainder1).exists());
Assert.assertFalse(new File(remainder2).exists());
FileUtil.fullyDelete(path);
}
+
+ @Test(timeout=60000)
+ public void testDirectoryFallbacks() throws Exception {
+ File nonExistentPath = new File(TEST_BASE, "nonexistent");
+ File permissionDeniedPath = new File("/");
+ File goodPath = new File(TEST_BASE, "testDirectoryFallbacks");
+ goodPath.mkdirs();
+ try {
+ SharedFileDescriptorFactory.create("shm_",
+ new String[] { nonExistentPath.getAbsolutePath(),
+ permissionDeniedPath.getAbsolutePath() });
+ Assert.fail();
+ } catch (IOException e) {
+ }
+ SharedFileDescriptorFactory factory =
+ SharedFileDescriptorFactory.create("shm_",
+ new String[] { nonExistentPath.getAbsolutePath(),
+ permissionDeniedPath.getAbsolutePath(),
+ goodPath.getAbsolutePath() } );
+ Assert.assertEquals(goodPath.getAbsolutePath(), factory.getPath());
+ FileUtil.fullyDelete(goodPath);
+ }
}