You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2014/12/11 03:26:13 UTC
[1/2] hadoop git commit: HDFS-7475. Make
TestLazyPersistFiles#testLazyPersistBlocksAreSaved deterministic.
(Contributed by Xiaoyu Yao)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 87b3fc8ca -> d39809016
refs/heads/trunk 92916ae48 -> 9a44db48b
HDFS-7475. Make TestLazyPersistFiles#testLazyPersistBlocksAreSaved deterministic. (Contributed by Xiaoyu Yao)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9a44db48
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9a44db48
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9a44db48
Branch: refs/heads/trunk
Commit: 9a44db48b4bfc097284f68a0576c058a0fd167bf
Parents: 92916ae
Author: arp <ar...@apache.org>
Authored: Wed Dec 10 18:24:22 2014 -0800
Committer: arp <ar...@apache.org>
Committed: Wed Dec 10 18:24:22 2014 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../fsdataset/impl/LazyPersistTestCase.java | 44 ++++++++++++++++++++
.../fsdataset/impl/TestLazyPersistFiles.java | 32 +-------------
3 files changed, 48 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a44db48/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1db358f..7b4e0c5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -567,6 +567,9 @@ Release 2.7.0 - UNRELEASED
HDFS-5578. [JDK8] Fix Javadoc errors caused by incorrect or illegal tags
in doc comments. (Andrew Purtell via wheat9)
+ HDFS-7475. Make TestLazyPersistFiles#testLazyPersistBlocksAreSaved
+ deterministic. (Xiaoyu Yao via Arpit Agarwal)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a44db48/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
index c762849..2de5bb7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
@@ -50,6 +50,8 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
+import java.util.HashSet;
+import java.util.Set;
import java.util.UUID;
import static org.apache.hadoop.fs.CreateFlag.CREATE;
@@ -131,6 +133,48 @@ public abstract class LazyPersistTestCase {
return locatedBlocks;
}
+ /**
+ * Make sure at least one non-transient volume has a saved copy of the replica.
+ * An infinite loop is used to ensure the async lazy persist tasks are completely
+ * done before verification. Caller of ensureLazyPersistBlocksAreSaved expects
+ * either a successful pass or timeout failure.
+ */
+ protected final void ensureLazyPersistBlocksAreSaved(
+ LocatedBlocks locatedBlocks) throws IOException, InterruptedException {
+ final String bpid = cluster.getNamesystem().getBlockPoolId();
+ List<? extends FsVolumeSpi> volumes =
+ cluster.getDataNodes().get(0).getFSDataset().getVolumes();
+ final Set<Long> persistedBlockIds = new HashSet<Long>();
+
+ while (persistedBlockIds.size() < locatedBlocks.getLocatedBlocks().size()) {
+ // Take 1 second sleep before each verification iteration
+ Thread.sleep(1000);
+
+ for (LocatedBlock lb : locatedBlocks.getLocatedBlocks()) {
+ for (FsVolumeSpi v : volumes) {
+ if (v.isTransientStorage()) {
+ continue;
+ }
+
+ FsVolumeImpl volume = (FsVolumeImpl) v;
+ File lazyPersistDir = volume.getBlockPoolSlice(bpid).getLazypersistDir();
+
+ long blockId = lb.getBlock().getBlockId();
+ File targetDir =
+ DatanodeUtil.idToBlockDir(lazyPersistDir, blockId);
+ File blockFile = new File(targetDir, lb.getBlock().getBlockName());
+ if (blockFile.exists()) {
+ // Found a persisted copy for this block and added to the Set
+ persistedBlockIds.add(blockId);
+ }
+ }
+ }
+ }
+
+ // We should have found a persisted copy for each located block.
+ assertThat(persistedBlockIds.size(), is(locatedBlocks.getLocatedBlocks().size()));
+ }
+
protected final void makeRandomTestFile(Path path, long length,
boolean isLazyPersist, long seed) throws IOException {
DFSTestUtil.createFile(fs, path, isLazyPersist, BUFFER_LENGTH, length,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a44db48/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
index 771609c..49d3c6d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
@@ -304,37 +304,7 @@ public class TestLazyPersistFiles extends LazyPersistTestCase {
// Make sure that there is a saved copy of the replica on persistent
// storage.
- final String bpid = cluster.getNamesystem().getBlockPoolId();
- List<? extends FsVolumeSpi> volumes =
- cluster.getDataNodes().get(0).getFSDataset().getVolumes();
-
- final Set<Long> persistedBlockIds = new HashSet<Long>();
-
- // Make sure at least one non-transient volume has a saved copy of
- // the replica.
- for (FsVolumeSpi v : volumes) {
- if (v.isTransientStorage()) {
- continue;
- }
-
- FsVolumeImpl volume = (FsVolumeImpl) v;
- File lazyPersistDir = volume.getBlockPoolSlice(bpid).getLazypersistDir();
-
- for (LocatedBlock lb : locatedBlocks.getLocatedBlocks()) {
- File targetDir = DatanodeUtil.idToBlockDir(lazyPersistDir, lb.getBlock().getBlockId());
- File blockFile = new File(targetDir, lb.getBlock().getBlockName());
- if (blockFile.exists()) {
- // Found a persisted copy for this block!
- boolean added = persistedBlockIds.add(lb.getBlock().getBlockId());
- assertThat(added, is(true));
- } else {
- LOG.error(blockFile + " not found");
- }
- }
- }
-
- // We should have found a persisted copy for each located block.
- assertThat(persistedBlockIds.size(), is(locatedBlocks.getLocatedBlocks().size()));
+ ensureLazyPersistBlocksAreSaved(locatedBlocks);
}
/**
[2/2] hadoop git commit: HDFS-7475. Make
TestLazyPersistFiles#testLazyPersistBlocksAreSaved deterministic.
(Contributed by Xiaoyu Yao)
Posted by ar...@apache.org.
HDFS-7475. Make TestLazyPersistFiles#testLazyPersistBlocksAreSaved deterministic. (Contributed by Xiaoyu Yao)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3980901
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3980901
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3980901
Branch: refs/heads/branch-2
Commit: d39809016df7620ef6e1679e75c222fd34524353
Parents: 87b3fc8
Author: arp <ar...@apache.org>
Authored: Wed Dec 10 18:24:22 2014 -0800
Committer: arp <ar...@apache.org>
Committed: Wed Dec 10 18:24:50 2014 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../fsdataset/impl/LazyPersistTestCase.java | 44 ++++++++++++++++++++
.../fsdataset/impl/TestLazyPersistFiles.java | 32 +-------------
3 files changed, 48 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3980901/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index eb0ff86..68271f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -310,6 +310,9 @@ Release 2.7.0 - UNRELEASED
HDFS-5578. [JDK8] Fix Javadoc errors caused by incorrect or illegal tags
in doc comments. (Andrew Purtell via wheat9)
+ HDFS-7475. Make TestLazyPersistFiles#testLazyPersistBlocksAreSaved
+ deterministic. (Xiaoyu Yao via Arpit Agarwal)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3980901/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
index c762849..2de5bb7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/LazyPersistTestCase.java
@@ -50,6 +50,8 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
+import java.util.HashSet;
+import java.util.Set;
import java.util.UUID;
import static org.apache.hadoop.fs.CreateFlag.CREATE;
@@ -131,6 +133,48 @@ public abstract class LazyPersistTestCase {
return locatedBlocks;
}
+ /**
+ * Make sure at least one non-transient volume has a saved copy of the replica.
+ * An infinite loop is used to ensure the async lazy persist tasks are completely
+ * done before verification. Caller of ensureLazyPersistBlocksAreSaved expects
+ * either a successful pass or timeout failure.
+ */
+ protected final void ensureLazyPersistBlocksAreSaved(
+ LocatedBlocks locatedBlocks) throws IOException, InterruptedException {
+ final String bpid = cluster.getNamesystem().getBlockPoolId();
+ List<? extends FsVolumeSpi> volumes =
+ cluster.getDataNodes().get(0).getFSDataset().getVolumes();
+ final Set<Long> persistedBlockIds = new HashSet<Long>();
+
+ while (persistedBlockIds.size() < locatedBlocks.getLocatedBlocks().size()) {
+ // Take 1 second sleep before each verification iteration
+ Thread.sleep(1000);
+
+ for (LocatedBlock lb : locatedBlocks.getLocatedBlocks()) {
+ for (FsVolumeSpi v : volumes) {
+ if (v.isTransientStorage()) {
+ continue;
+ }
+
+ FsVolumeImpl volume = (FsVolumeImpl) v;
+ File lazyPersistDir = volume.getBlockPoolSlice(bpid).getLazypersistDir();
+
+ long blockId = lb.getBlock().getBlockId();
+ File targetDir =
+ DatanodeUtil.idToBlockDir(lazyPersistDir, blockId);
+ File blockFile = new File(targetDir, lb.getBlock().getBlockName());
+ if (blockFile.exists()) {
+ // Found a persisted copy for this block and added to the Set
+ persistedBlockIds.add(blockId);
+ }
+ }
+ }
+ }
+
+ // We should have found a persisted copy for each located block.
+ assertThat(persistedBlockIds.size(), is(locatedBlocks.getLocatedBlocks().size()));
+ }
+
protected final void makeRandomTestFile(Path path, long length,
boolean isLazyPersist, long seed) throws IOException {
DFSTestUtil.createFile(fs, path, isLazyPersist, BUFFER_LENGTH, length,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3980901/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
index 771609c..49d3c6d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
@@ -304,37 +304,7 @@ public class TestLazyPersistFiles extends LazyPersistTestCase {
// Make sure that there is a saved copy of the replica on persistent
// storage.
- final String bpid = cluster.getNamesystem().getBlockPoolId();
- List<? extends FsVolumeSpi> volumes =
- cluster.getDataNodes().get(0).getFSDataset().getVolumes();
-
- final Set<Long> persistedBlockIds = new HashSet<Long>();
-
- // Make sure at least one non-transient volume has a saved copy of
- // the replica.
- for (FsVolumeSpi v : volumes) {
- if (v.isTransientStorage()) {
- continue;
- }
-
- FsVolumeImpl volume = (FsVolumeImpl) v;
- File lazyPersistDir = volume.getBlockPoolSlice(bpid).getLazypersistDir();
-
- for (LocatedBlock lb : locatedBlocks.getLocatedBlocks()) {
- File targetDir = DatanodeUtil.idToBlockDir(lazyPersistDir, lb.getBlock().getBlockId());
- File blockFile = new File(targetDir, lb.getBlock().getBlockName());
- if (blockFile.exists()) {
- // Found a persisted copy for this block!
- boolean added = persistedBlockIds.add(lb.getBlock().getBlockId());
- assertThat(added, is(true));
- } else {
- LOG.error(blockFile + " not found");
- }
- }
- }
-
- // We should have found a persisted copy for each located block.
- assertThat(persistedBlockIds.size(), is(locatedBlocks.getLocatedBlocks().size()));
+ ensureLazyPersistBlocksAreSaved(locatedBlocks);
}
/**