You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2017/08/14 17:30:02 UTC
hadoop git commit: HDFS-12289. [READ] HDFS-12091 breaks the tests for
provided block reads
Repository: hadoop
Updated Branches:
refs/heads/HDFS-9806 5c2a0a1c5 -> 950330a31
HDFS-12289. [READ] HDFS-12091 breaks the tests for provided block reads
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/950330a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/950330a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/950330a3
Branch: refs/heads/HDFS-9806
Commit: 950330a3188af3bc22c76025a2a6ac42142acc81
Parents: 5c2a0a1
Author: Virajith Jalaparti <vi...@apache.org>
Authored: Mon Aug 14 10:29:47 2017 -0700
Committer: Virajith Jalaparti <vi...@apache.org>
Committed: Mon Aug 14 10:29:47 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 30 +++++++++++++++++++-
.../TestNameNodeProvidedImplementation.java | 4 ++-
2 files changed, 32 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/950330a3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index f9908fe..7088b90 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -146,6 +146,9 @@ public class MiniDFSCluster implements AutoCloseable {
GenericTestUtils.SYSPROP_TEST_DATA_DIR;
/** Configuration option to set the data dir: {@value} */
public static final String HDFS_MINIDFS_BASEDIR = "hdfs.minidfs.basedir";
+ /** Configuration option to set the provided data dir: {@value} */
+ public static final String HDFS_MINIDFS_BASEDIR_PROVIDED =
+ "hdfs.minidfs.basedir.provided";
public static final String DFS_NAMENODE_SAFEMODE_EXTENSION_TESTING_KEY
= DFS_NAMENODE_SAFEMODE_EXTENSION_KEY + ".testing";
@@ -1383,7 +1386,12 @@ public class MiniDFSCluster implements AutoCloseable {
if ((storageTypes != null) && (j >= storageTypes.length)) {
break;
}
- File dir = getInstanceStorageDir(dnIndex, j);
+ File dir;
+ if (storageTypes != null && storageTypes[j] == StorageType.PROVIDED) {
+ dir = getProvidedStorageDir(dnIndex, j);
+ } else {
+ dir = getInstanceStorageDir(dnIndex, j);
+ }
dir.mkdirs();
if (!dir.isDirectory()) {
throw new IOException("Mkdirs failed to create directory for DataNode " + dir);
@@ -2833,6 +2841,26 @@ public class MiniDFSCluster implements AutoCloseable {
}
/**
+ * Get a storage directory for PROVIDED storages.
+ * The PROVIDED directory to return can be set by using the configuration
+ * parameter {@link #HDFS_MINIDFS_BASEDIR_PROVIDED}. If this parameter is
+ * not set, this function behaves exactly the same as
+ * {@link #getInstanceStorageDir(int, int)}. Currently, the two parameters
+ * are ignored as only one PROVIDED storage is supported in HDFS-9806.
+ *
+ * @param dnIndex datanode index (starts from 0)
+ * @param dirIndex directory index
+ * @return Storage directory
+ */
+ public File getProvidedStorageDir(int dnIndex, int dirIndex) {
+ String base = conf.get(HDFS_MINIDFS_BASEDIR_PROVIDED, null);
+ if (base == null) {
+ return getInstanceStorageDir(dnIndex, dirIndex);
+ }
+ return new File(base);
+ }
+
+ /**
* Get a storage directory for a datanode.
* <ol>
* <li><base directory>/data/data<2*dnIndex + 1></li>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/950330a3/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java b/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java
index 60b306f..3f937c4 100644
--- a/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java
+++ b/hadoop-tools/hadoop-fs2img/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeProvidedImplementation.java
@@ -74,7 +74,7 @@ public class TestNameNodeProvidedImplementation {
final Random r = new Random();
final File fBASE = new File(MiniDFSCluster.getBaseDirectory());
final Path BASE = new Path(fBASE.toURI().toString());
- final Path NAMEPATH = new Path(BASE, "providedDir");;
+ final Path NAMEPATH = new Path(BASE, "providedDir");
final Path NNDIRPATH = new Path(BASE, "nnDir");
final Path BLOCKFILE = new Path(NNDIRPATH, "blocks.csv");
final String SINGLEUSER = "usr1";
@@ -116,6 +116,8 @@ public class TestNameNodeProvidedImplementation {
BLOCKFILE.toString());
conf.set(DFSConfigKeys.DFS_PROVIDED_BLOCK_MAP_DELIMITER, ",");
+ conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR_PROVIDED,
+ new File(NAMEPATH.toUri()).toString());
File imageDir = new File(NAMEPATH.toUri());
if (!imageDir.exists()) {
LOG.info("Creating directory: " + imageDir);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org