You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xk...@apache.org on 2018/05/04 19:27:49 UTC
[21/50] [abbrv] hadoop git commit: HDFS-13503. Fix TestFsck test
failures on Windows. Contributed by Xiao Liang.
HDFS-13503. Fix TestFsck test failures on Windows. Contributed by Xiao Liang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e2cfb2d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e2cfb2d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e2cfb2d
Branch: refs/heads/HDFS-12943
Commit: 9e2cfb2d3f1a18984d07c81f9c46626dd842402a
Parents: 4e1382a
Author: Inigo Goiri <in...@apache.org>
Authored: Tue May 1 08:12:46 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Tue May 1 08:12:46 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 3 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 123 ++++++++++++-------
2 files changed, 81 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e2cfb2d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index acb720e..c2e2a68 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -2924,7 +2924,8 @@ public class MiniDFSCluster implements AutoCloseable {
* @return Storage directory
*/
public File getStorageDir(int dnIndex, int dirIndex) {
- return new File(getBaseDirectory(), getStorageDirPath(dnIndex, dirIndex));
+ return new File(determineDfsBaseDir(),
+ getStorageDirPath(dnIndex, dirIndex));
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e2cfb2d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index f80fd70..1a392da 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.namenode;
+import static org.apache.hadoop.hdfs.MiniDFSCluster.HDFS_MINIDFS_BASEDIR;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -209,7 +210,9 @@ public class TestFsck {
conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY,
precision);
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
+ .numDataNodes(4).build();
fs = cluster.getFileSystem();
final String fileName = "/srcdat";
util.createFiles(fs, fileName);
@@ -297,7 +300,9 @@ public class TestFsck {
setNumFiles(20).build();
FileSystem fs = null;
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
+ .numDataNodes(4).build();
fs = cluster.getFileSystem();
util.createFiles(fs, "/srcdat");
util.waitReplication(fs, "/srcdat", (short)3);
@@ -315,7 +320,9 @@ public class TestFsck {
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
// Create a cluster with the current user, write some files
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
+ .numDataNodes(4).build();
final MiniDFSCluster c2 = cluster;
final String dir = "/dfsck";
final Path dirpath = new Path(dir);
@@ -361,8 +368,9 @@ public class TestFsck {
DFSTestUtil util = new DFSTestUtil("TestFsck", 5, 3,
(5 * dfsBlockSize) + (dfsBlockSize - 1), 5 * dfsBlockSize);
FileSystem fs = null;
- cluster = new MiniDFSCluster.Builder(conf).
- numDataNodes(numDatanodes).build();
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
+ .numDataNodes(numDatanodes).build();
String topDir = "/srcdat";
fs = cluster.getFileSystem();
cluster.waitActive();
@@ -568,7 +576,9 @@ public class TestFsck {
FileSystem fs = null;
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1);
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
+ .numDataNodes(4).build();
String topDir = "/srcdat";
fs = cluster.getFileSystem();
cluster.waitActive();
@@ -632,7 +642,9 @@ public class TestFsck {
setNumFiles(4).build();
FileSystem fs = null;
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
+ .numDataNodes(4).build();
String topDir = "/srcdat";
String randomString = "HADOOP ";
fs = cluster.getFileSystem();
@@ -685,7 +697,8 @@ public class TestFsck {
final int numAllUnits = dataBlocks + ecPolicy.getNumParityUnits();
int blockSize = 2 * cellSize;
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).numDataNodes(
numAllUnits + 1).build();
String topDir = "/myDir";
cluster.waitActive();
@@ -776,7 +789,9 @@ public class TestFsck {
String outStr = null;
short factor = 1;
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
+ .numDataNodes(1).build();
cluster.waitActive();
fs = cluster.getFileSystem();
Path file1 = new Path("/testCorruptBlock");
@@ -847,7 +862,9 @@ public class TestFsck {
Random random = new Random();
String outStr = null;
short factor = 1;
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
+ .numDataNodes(2).build();
cluster.waitActive();
fs = cluster.getFileSystem();
Path file1 = new Path("/testUnderMinReplicatedBlock");
@@ -919,9 +936,9 @@ public class TestFsck {
conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
DistributedFileSystem dfs;
- cluster =
- new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts)
- .racks(racks).build();
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
+ .numDataNodes(numDn).hosts(hosts).racks(racks).build();
cluster.waitClusterUp();
dfs = cluster.getFileSystem();
@@ -1068,7 +1085,8 @@ public class TestFsck {
@Test
public void testFsckError() throws Exception {
// bring up a one-node cluster
- cluster = new MiniDFSCluster.Builder(conf).build();
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).build();
String fileName = "/test.txt";
Path filePath = new Path(fileName);
FileSystem fs = cluster.getFileSystem();
@@ -1100,7 +1118,8 @@ public class TestFsck {
conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1);
FileSystem fs = null;
- cluster = new MiniDFSCluster.Builder(conf).build();
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).build();
cluster.waitActive();
fs = cluster.getFileSystem();
DFSTestUtil util = new DFSTestUtil.Builder().
@@ -1163,7 +1182,8 @@ public class TestFsck {
@Test
public void testToCheckTheFsckCommandOnIllegalArguments() throws Exception {
// bring up a one-node cluster
- cluster = new MiniDFSCluster.Builder(conf).build();
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).build();
String fileName = "/test.txt";
Path filePath = new Path(fileName);
FileSystem fs = cluster.getFileSystem();
@@ -1207,8 +1227,9 @@ public class TestFsck {
DistributedFileSystem dfs = null;
// Startup a minicluster
- cluster =
- new MiniDFSCluster.Builder(conf).numDataNodes(numReplicas).build();
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
+ .numDataNodes(numReplicas).build();
assertNotNull("Failed Cluster Creation", cluster);
cluster.waitClusterUp();
dfs = cluster.getFileSystem();
@@ -1268,9 +1289,9 @@ public class TestFsck {
DistributedFileSystem dfs = null;
// Startup a minicluster
- cluster =
- new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts)
- .racks(racks).build();
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
+ .numDataNodes(numDn).hosts(hosts).racks(racks).build();
assertNotNull("Failed Cluster Creation", cluster);
cluster.waitClusterUp();
dfs = cluster.getFileSystem();
@@ -1377,7 +1398,9 @@ public class TestFsck {
conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY,
precision);
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(4).build();
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
+ .numDataNodes(4).build();
fs = cluster.getFileSystem();
final String fileName = "/srcdat";
util.createFiles(fs, fileName);
@@ -1404,7 +1427,8 @@ public class TestFsck {
*/
@Test
public void testFsckForSnapshotFiles() throws Exception {
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).numDataNodes(1)
.build();
String runFsck = runFsck(conf, 0, true, "/", "-includeSnapshots",
"-files");
@@ -1439,9 +1463,9 @@ public class TestFsck {
conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 2);
DistributedFileSystem dfs = null;
- cluster =
- new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts)
- .racks(racks).build();
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
+ .numDataNodes(numDn).hosts(hosts).racks(racks).build();
assertNotNull("Failed Cluster Creation", cluster);
cluster.waitClusterUp();
@@ -1494,9 +1518,9 @@ public class TestFsck {
conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 2);
DistributedFileSystem dfs;
- cluster =
- new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts)
- .racks(racks).build();
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
+ .numDataNodes(numDn).hosts(hosts).racks(racks).build();
assertNotNull("Failed Cluster Creation", cluster);
cluster.waitClusterUp();
@@ -1579,7 +1603,8 @@ public class TestFsck {
replFactor);
DistributedFileSystem dfs;
- cluster = new MiniDFSCluster.Builder(conf)
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
.numDataNodes(numDn)
.hosts(hosts)
.racks(racks)
@@ -1700,9 +1725,9 @@ public class TestFsck {
conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
DistributedFileSystem dfs = null;
- cluster =
- new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts)
- .racks(racks).build();
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
+ .numDataNodes(numDn).hosts(hosts).racks(racks).build();
assertNotNull("Failed Cluster Creation", cluster);
cluster.waitClusterUp();
@@ -1769,7 +1794,8 @@ public class TestFsck {
*/
@Test
public void testStoragePoliciesCK() throws Exception {
- cluster = new MiniDFSCluster.Builder(conf)
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
.numDataNodes(3)
.storageTypes(
new StorageType[] {StorageType.DISK, StorageType.ARCHIVE})
@@ -1812,9 +1838,9 @@ public class TestFsck {
conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
DistributedFileSystem dfs;
- cluster =
- new MiniDFSCluster.Builder(conf).numDataNodes(numDn).hosts(hosts)
- .racks(racks).build();
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
+ .numDataNodes(numDn).hosts(hosts).racks(racks).build();
assertNotNull("Failed Cluster Creation", cluster);
cluster.waitClusterUp();
@@ -1894,7 +1920,8 @@ public class TestFsck {
replFactor);
DistributedFileSystem dfs;
- cluster = new MiniDFSCluster.Builder(conf)
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
.numDataNodes(numDn)
.hosts(hosts)
.racks(racks)
@@ -2002,7 +2029,9 @@ public class TestFsck {
int parityBlocks =
StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
int totalSize = dataBlocks + parityBlocks;
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(totalSize).build();
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
+ .numDataNodes(totalSize).build();
fs = cluster.getFileSystem();
fs.enableErasureCodingPolicy(
StripedFileTestUtil.getDefaultECPolicy().getName());
@@ -2069,7 +2098,8 @@ public class TestFsck {
int numFiles = 3;
int numSnapshots = 0;
- cluster = new MiniDFSCluster.Builder(conf).build();
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).build();
cluster.waitActive();
hdfs = cluster.getFileSystem();
DFSTestUtil util = new DFSTestUtil.Builder().
@@ -2149,7 +2179,8 @@ public class TestFsck {
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000L);
conf.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, 1);
conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, replication);
- cluster = new MiniDFSCluster.Builder(conf).build();
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir).build();
DistributedFileSystem dfs = cluster.getFileSystem();
cluster.waitActive();
@@ -2244,6 +2275,7 @@ public class TestFsck {
HostsFileWriter hostsFileWriter = new HostsFileWriter();
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, replFactor);
+ conf.set(HDFS_MINIDFS_BASEDIR, GenericTestUtils.getRandomizedTempPath());
if (defineUpgradeDomain) {
conf.setClass(DFSConfigKeys.DFS_NAMENODE_HOSTS_PROVIDER_CLASSNAME_KEY,
CombinedHostFileManager.class, HostConfigManager.class);
@@ -2295,7 +2327,8 @@ public class TestFsck {
StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize();
int totalSize = dataBlocks + parityBlocks;
- cluster = new MiniDFSCluster.Builder(conf)
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
.numDataNodes(totalSize).build();
fs = cluster.getFileSystem();
fs.enableErasureCodingPolicy(
@@ -2366,7 +2399,8 @@ public class TestFsck {
StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize();
int totalSize = dataBlocks + parityBlocks;
- cluster = new MiniDFSCluster.Builder(conf)
+ File builderBaseDir = new File(GenericTestUtils.getRandomizedTempPath());
+ cluster = new MiniDFSCluster.Builder(conf, builderBaseDir)
.numDataNodes(totalSize).build();
fs = cluster.getFileSystem();
fs.enableErasureCodingPolicy(
@@ -2427,7 +2461,8 @@ public class TestFsck {
@Test(timeout = 300000)
public void testFsckCorruptWhenOneReplicaIsCorrupt()
throws Exception {
- try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf,
+ new File(GenericTestUtils.getRandomizedTempPath()))
.nnTopology(MiniDFSNNTopology.simpleHATopology()).numDataNodes(2)
.build()) {
cluster.waitActive();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org