You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2014/10/01 03:06:18 UTC
[04/13] git commit: HDFS-7167. NPE while running Mover if the given
path is for a file. Contributed by Jing Zhao.
HDFS-7167. NPE while running Mover if the given path is for a file. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cdf1af0e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cdf1af0e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cdf1af0e
Branch: refs/heads/HDFS-6581
Commit: cdf1af0e5a21361924f3f7c6cea5170767d2b6bc
Parents: ea32a66
Author: Jing Zhao <ji...@apache.org>
Authored: Tue Sep 30 11:13:35 2014 -0700
Committer: Jing Zhao <ji...@apache.org>
Committed: Tue Sep 30 11:13:35 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 10 +++--
.../apache/hadoop/hdfs/server/mover/Mover.java | 22 ++++-------
.../hdfs/server/mover/TestStorageMover.java | 40 ++++++++++++++++++--
3 files changed, 50 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdf1af0e/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1a97659..40891bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -772,6 +772,11 @@ Release 2.6.0 - UNRELEASED
HDFS-7157. Using Time.now() for recording start/end time of reconfiguration
tasks (Lei Xu via cmccabe)
+ HDFS-6664. HDFS permissions guide documentation states incorrect default
+ group mapping class. (Ray Chiang via aw)
+
+ HDFS-4227. Document dfs.namenode.resource.* (Daisuke Kobayashi via aw)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
@@ -1003,10 +1008,7 @@ Release 2.6.0 - UNRELEASED
HDFS-7140. Add a tool to list all the existing block storage policies.
(jing9)
- HDFS-6664. HDFS permissions guide documentation states incorrect default
- group mapping class. (Ray Chiang via aw)
-
- HDFS-4227. Document dfs.namenode.resource.* (Daisuke Kobayashi via aw)
+ HDFS-7167. NPE while running Mover if the given path is for a file. (jing9)
Release 2.5.1 - 2014-09-05
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdf1af0e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
index 4db0df6..04133bd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
@@ -252,14 +252,9 @@ public class Mover {
*/
private boolean processNamespace() {
getSnapshottableDirs();
- boolean hasRemaining = true;
- try {
- for (Path target : targetPaths) {
- hasRemaining = processDirRecursively("", dfs.getFileInfo(target
- .toUri().getPath()));
- }
- } catch (IOException e) {
- LOG.warn("Failed to get root directory status. Ignore and continue.", e);
+ boolean hasRemaining = false;
+ for (Path target : targetPaths) {
+ hasRemaining |= processPath(target.toUri().getPath());
}
// wait for pending move to finish and retry the failed migration
hasRemaining |= Dispatcher.waitForMoveCompletion(storages.targets.values());
@@ -270,7 +265,7 @@ public class Mover {
* @return whether there is still remaing migration work for the next
* round
*/
- private boolean processChildrenList(String fullPath) {
+ private boolean processPath(String fullPath) {
boolean hasRemaining = false;
for (byte[] lastReturnedName = HdfsFileStatus.EMPTY_NAME;;) {
final DirectoryListing children;
@@ -285,7 +280,7 @@ public class Mover {
return hasRemaining;
}
for (HdfsFileStatus child : children.getPartialListing()) {
- hasRemaining |= processDirRecursively(fullPath, child);
+ hasRemaining |= processRecursively(fullPath, child);
}
if (children.hasMore()) {
lastReturnedName = children.getLastName();
@@ -296,8 +291,7 @@ public class Mover {
}
/** @return whether the migration requires next round */
- private boolean processDirRecursively(String parent,
- HdfsFileStatus status) {
+ private boolean processRecursively(String parent, HdfsFileStatus status) {
String fullPath = status.getFullName(parent);
boolean hasRemaining = false;
if (status.isDir()) {
@@ -305,11 +299,11 @@ public class Mover {
fullPath = fullPath + Path.SEPARATOR;
}
- hasRemaining = processChildrenList(fullPath);
+ hasRemaining = processPath(fullPath);
// process snapshots if this is a snapshottable directory
if (snapshottableDirs.contains(fullPath)) {
final String dirSnapshot = fullPath + HdfsConstants.DOT_SNAPSHOT_DIR;
- hasRemaining |= processChildrenList(dirSnapshot);
+ hasRemaining |= processPath(dirSnapshot);
}
} else if (!status.isSymlink()) { // file
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdf1af0e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
index 0001e3b..a6edd80 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hdfs.server.mover;
-import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
@@ -27,12 +26,10 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
-import com.google.common.base.Joiner;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.ReconfigurationException;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
@@ -46,7 +43,6 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -515,6 +511,42 @@ public class TestStorageMover {
}
/**
+ * Run Mover with arguments specifying files and directories
+ */
+ @Test
+ public void testMoveSpecificPaths() throws Exception {
+ LOG.info("testMoveSpecificPaths");
+ final Path foo = new Path("/foo");
+ final Path barFile = new Path(foo, "bar");
+ final Path foo2 = new Path("/foo2");
+ final Path bar2File = new Path(foo2, "bar2");
+ Map<Path, BlockStoragePolicy> policyMap = Maps.newHashMap();
+ policyMap.put(foo, COLD);
+ policyMap.put(foo2, WARM);
+ NamespaceScheme nsScheme = new NamespaceScheme(Arrays.asList(foo, foo2),
+ Arrays.asList(barFile, bar2File), BLOCK_SIZE, null, policyMap);
+ ClusterScheme clusterScheme = new ClusterScheme(DEFAULT_CONF,
+ NUM_DATANODES, REPL, genStorageTypes(NUM_DATANODES), null);
+ MigrationTest test = new MigrationTest(clusterScheme, nsScheme);
+ test.setupCluster();
+
+ try {
+ test.prepareNamespace();
+ test.setStoragePolicy();
+
+ Map<URI, List<Path>> map = Mover.Cli.getNameNodePathsToMove(test.conf,
+ "-p", "/foo/bar", "/foo2");
+ int result = Mover.run(map, test.conf);
+ Assert.assertEquals(ExitStatus.SUCCESS.getExitCode(), result);
+
+ Thread.sleep(5000);
+ test.verify(true);
+ } finally {
+ test.shutdownCluster();
+ }
+ }
+
+ /**
* Move an open file into archival storage
*/
@Test