You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2015/11/03 22:09:17 UTC
[01/50] hadoop git commit: HDFS-8545. Refactor FS#getUsed() to use
ContentSummary and add an API to fetch the total file length from a specific
path (Contributed by J.Andreina)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-8966 08aafacd0 -> 27009cf32
HDFS-8545. Refactor FS#getUsed() to use ContentSummary and add an API to fetch the total file length from a specific path (Contributed by J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7d2d16f4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7d2d16f4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7d2d16f4
Branch: refs/heads/HDFS-8966
Commit: 7d2d16f4ee87ae56dc20016a91c109dd5130f7d4
Parents: b0c818b
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Oct 29 11:28:17 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu Oct 29 11:28:17 2015 +0530
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileSystem.java | 19 ++++++-------
.../org/apache/hadoop/fs/FilterFileSystem.java | 8 +++++-
.../org/apache/hadoop/fs/HarFileSystem.java | 6 +++++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/TestDistributedFileSystem.java | 28 ++++++++++++++++++++
5 files changed, 54 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d2d16f4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index bd8aa2a..d3eb0ad 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -2079,16 +2079,17 @@ public abstract class FileSystem extends Configured implements Closeable {
CACHE.remove(this.key, this);
}
- /** Return the total size of all files in the filesystem.*/
- public long getUsed() throws IOException{
- long used = 0;
- RemoteIterator<LocatedFileStatus> files = listFiles(new Path("/"), true);
- while (files.hasNext()) {
- used += files.next().getLen();
- }
- return used;
+ /** Return the total size of all files in the filesystem. */
+ public long getUsed() throws IOException {
+ Path path = new Path("/");
+ return getUsed(path);
}
-
+
+ /** Return the total size of all files from a specified path. */
+ public long getUsed(Path path) throws IOException {
+ return getContentSummary(path).getLength();
+ }
+
/**
* Get the block size for a particular file.
* @param f the filename
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d2d16f4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
index f862c74..00f6778 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
@@ -389,7 +389,13 @@ public class FilterFileSystem extends FileSystem {
public long getUsed() throws IOException{
return fs.getUsed();
}
-
+
+ /** Return the total size of all files from a specified path.*/
+ @Override
+ public long getUsed(Path path) throws IOException {
+ return fs.getUsed(path);
+ }
+
@Override
public long getDefaultBlockSize() {
return fs.getDefaultBlockSize();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d2d16f4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
index 868b8dc..ea5e6a3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
@@ -1237,6 +1237,12 @@ public class HarFileSystem extends FileSystem {
return fs.getUsed();
}
+ /** Return the total size of all files from a specified path.*/
+ @Override
+ public long getUsed(Path path) throws IOException {
+ return fs.getUsed(path);
+ }
+
@SuppressWarnings("deprecation")
@Override
public long getDefaultBlockSize() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d2d16f4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e79d660..f8f7452 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1602,6 +1602,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9295. Add a thorough test of the full KMS code path.
(Daniel Templeton via zhz)
+ HDFS-8545. Refactor FS#getUsed() to use ContentSummary and add an API to fetch
+ the total file length from a specific path (J.Andreina via vinayakumarb)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d2d16f4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index 79da7b8..0543e9d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -1048,4 +1048,32 @@ public class TestDistributedFileSystem {
cluster.shutdown();
}
}
+
+ @Test(timeout = 30000)
+ public void testTotalDfsUsed() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ FileSystem fs = cluster.getFileSystem();
+ // create file under root
+ FSDataOutputStream File1 = fs.create(new Path("/File1"));
+ File1.write("hi".getBytes());
+ File1.close();
+ // create file under sub-folder
+ FSDataOutputStream File2 = fs.create(new Path("/Folder1/File2"));
+ File2.write("hi".getBytes());
+ File2.close();
+ // getUsed(Path) should return total len of all the files from a path
+ assertEquals(2, fs.getUsed(new Path("/Folder1")));
+ //getUsed() should return total length of all files in filesystem
+ assertEquals(4, fs.getUsed());
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+ }
+
}
[30/50] hadoop git commit: Revert "fix CHANGES.txt"
Posted by ji...@apache.org.
Revert "fix CHANGES.txt"
This reverts commit 3c0204a5866520e74917b26b6ac2061650a5bb6d.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ea4413b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ea4413b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ea4413b
Branch: refs/heads/HDFS-8966
Commit: 2ea4413b15f82a032d6dbd2532861d82a299461a
Parents: a4a6b5b
Author: yliu <yl...@apache.org>
Authored: Sat Oct 31 16:20:37 2015 +0800
Committer: yliu <yl...@apache.org>
Committed: Sat Oct 31 16:20:37 2015 +0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ea4413b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 5a61eed..211e7fc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2201,6 +2201,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9332. Fix Precondition failures from NameNodeEditLogRoller while
saving namespace. (wang)
+ HDFS-4937. ReplicationMonitor can infinite-loop in
+ BlockPlacementPolicyDefault#chooseRandom() (kihwal)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -2273,9 +2276,6 @@ Release 2.7.2 - UNRELEASED
HDFS-9317. Document fsck -blockId and -storagepolicy options in branch-2.7.
(aajisaka)
- HDFS-4937. ReplicationMonitor can infinite-loop in
- BlockPlacementPolicyDefault#chooseRandom() (kihwal)
-
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
[48/50] hadoop git commit: HDFS-9354. Fix
TestBalancer#testBalancerWithZeroThreadsForMove on Windows. Contributed by
Xiaoyu Yao.
Posted by ji...@apache.org.
HDFS-9354. Fix TestBalancer#testBalancerWithZeroThreadsForMove on Windows. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/095ac834
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/095ac834
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/095ac834
Branch: refs/heads/HDFS-8966
Commit: 095ac834022df6136b42961c507ec745c6cf8f97
Parents: 0783184
Author: cnauroth <cn...@apache.org>
Authored: Tue Nov 3 10:51:21 2015 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Tue Nov 3 11:21:08 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/server/balancer/TestBalancer.java | 572 +++++++++----------
2 files changed, 277 insertions(+), 298 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/095ac834/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 879c015..1729b73 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2219,6 +2219,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9313. Possible NullPointerException in BlockManager if no excess
replica can be chosen. (mingma)
+ HDFS-9354. Fix TestBalancer#testBalancerWithZeroThreadsForMove on Windows.
+ (Xiaoyu Yao via cnauroth)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/095ac834/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
index 332ae15..dd54345 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
@@ -86,6 +86,7 @@ import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.Tool;
import org.apache.log4j.Level;
+import org.junit.After;
import org.junit.Test;
/**
@@ -106,6 +107,14 @@ public class TestBalancer {
final static Path filePath = new Path(fileName);
private MiniDFSCluster cluster;
+ @After
+ public void shutdown() throws Exception {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
ClientProtocol client;
static final long TIMEOUT = 40000L; //msec
@@ -367,44 +376,38 @@ public class TestBalancer {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(capacities.length)
.hosts(hosts).racks(racks).simulatedCapacities(capacities).build();
- try {
- cluster.waitActive();
- client = NameNodeProxies.createProxy(conf,
- cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy();
-
- // fill up the cluster to be 80% full
- long totalCapacity = sum(capacities);
- long totalUsedSpace = totalCapacity * 8 / 10;
- InetSocketAddress[] favoredNodes = new InetSocketAddress[numOfDatanodes];
- for (int i = 0; i < favoredNodes.length; i++) {
- // DFSClient will attempt reverse lookup. In case it resolves
- // "127.0.0.1" to "localhost", we manually specify the hostname.
- int port = cluster.getDataNodes().get(i).getXferAddress().getPort();
- favoredNodes[i] = new InetSocketAddress(hosts[i], port);
- }
+ cluster.waitActive();
+ client = NameNodeProxies.createProxy(conf,
+ cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy();
+
+ // fill up the cluster to be 80% full
+ long totalCapacity = sum(capacities);
+ long totalUsedSpace = totalCapacity * 8 / 10;
+ InetSocketAddress[] favoredNodes = new InetSocketAddress[numOfDatanodes];
+ for (int i = 0; i < favoredNodes.length; i++) {
+ // DFSClient will attempt reverse lookup. In case it resolves
+ // "127.0.0.1" to "localhost", we manually specify the hostname.
+ int port = cluster.getDataNodes().get(i).getXferAddress().getPort();
+ favoredNodes[i] = new InetSocketAddress(hosts[i], port);
+ }
- DFSTestUtil.createFile(cluster.getFileSystem(0), filePath, false, 1024,
- totalUsedSpace / numOfDatanodes, DEFAULT_BLOCK_SIZE,
- (short) numOfDatanodes, 0, false, favoredNodes);
-
- // start up an empty node with the same capacity
- cluster.startDataNodes(conf, 1, true, null, new String[] { RACK2 },
- new long[] { CAPACITY });
-
- totalCapacity += CAPACITY;
-
- // run balancer and validate results
- waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
+ DFSTestUtil.createFile(cluster.getFileSystem(0), filePath, false, 1024,
+ totalUsedSpace / numOfDatanodes, DEFAULT_BLOCK_SIZE,
+ (short) numOfDatanodes, 0, false, favoredNodes);
- // start rebalancing
- Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
- int r = Balancer.run(namenodes, BalancerParameters.DEFAULT, conf);
- assertEquals(ExitStatus.NO_MOVE_PROGRESS.getExitCode(), r);
-
- } finally {
- cluster.shutdown();
- }
-
+ // start up an empty node with the same capacity
+ cluster.startDataNodes(conf, 1, true, null, new String[] { RACK2 },
+ new long[] { CAPACITY });
+
+ totalCapacity += CAPACITY;
+
+ // run balancer and validate results
+ waitForHeartBeat(totalUsedSpace, totalCapacity, client, cluster);
+
+ // start rebalancing
+ Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
+ int r = Balancer.run(namenodes, BalancerParameters.DEFAULT, conf);
+ assertEquals(ExitStatus.NO_MOVE_PROGRESS.getExitCode(), r);
}
/**
@@ -588,7 +591,7 @@ public class TestBalancer {
private void doTest(Configuration conf, long[] capacities,
String[] racks, long newCapacity, String newRack, NewNodeInfo nodes,
boolean useTool, boolean useFile) throws Exception {
- LOG.info("capacities = " + long2String(capacities));
+ LOG.info("capacities = " + long2String(capacities));
LOG.info("racks = " + Arrays.asList(racks));
LOG.info("newCapacity= " + newCapacity);
LOG.info("newRack = " + newRack);
@@ -606,7 +609,7 @@ public class TestBalancer {
ClientProtocol.class).getProxy();
long totalCapacity = sum(capacities);
-
+
// fill up the cluster to be 30% full
long totalUsedSpace = totalCapacity*3/10;
createFile(cluster, filePath, totalUsedSpace / numOfDatanodes,
@@ -830,7 +833,7 @@ public class TestBalancer {
/** one-node cluster test*/
private void oneNodeTest(Configuration conf, boolean useTool) throws Exception {
// add an empty node with half of the CAPACITY & the same rack
- doTest(conf, new long[]{CAPACITY}, new String[]{RACK0}, CAPACITY/2,
+ doTest(conf, new long[]{CAPACITY}, new String[]{RACK0}, CAPACITY/2,
RACK0, useTool);
}
@@ -884,31 +887,27 @@ public class TestBalancer {
.racks(racks)
.simulatedCapacities(capacities)
.build();
- try {
- cluster.waitActive();
- client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(),
- ClientProtocol.class).getProxy();
+ cluster.waitActive();
+ client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(),
+ ClientProtocol.class).getProxy();
- for(int i = 0; i < 3; i++) {
- cluster.injectBlocks(i, Arrays.asList(blocksDN[i]), null);
- }
+ for(int i = 0; i < 3; i++) {
+ cluster.injectBlocks(i, Arrays.asList(blocksDN[i]), null);
+ }
- cluster.startDataNodes(conf, 1, true, null,
- new String[]{RACK0}, null,new long[]{CAPACITY});
- cluster.triggerHeartbeats();
+ cluster.startDataNodes(conf, 1, true, null,
+ new String[]{RACK0}, null,new long[]{CAPACITY});
+ cluster.triggerHeartbeats();
- Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
- Set<String> datanodes = new HashSet<String>();
- datanodes.add(cluster.getDataNodes().get(0).getDatanodeId().getHostName());
- BalancerParameters.Builder pBuilder =
- new BalancerParameters.Builder();
- pBuilder.setExcludedNodes(datanodes);
- pBuilder.setRunDuringUpgrade(false);
- final int r = Balancer.run(namenodes, pBuilder.build(), conf);
- assertEquals(ExitStatus.SUCCESS.getExitCode(), r);
- } finally {
- cluster.shutdown();
- }
+ Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
+ Set<String> datanodes = new HashSet<String>();
+ datanodes.add(cluster.getDataNodes().get(0).getDatanodeId().getHostName());
+ BalancerParameters.Builder pBuilder =
+ new BalancerParameters.Builder();
+ pBuilder.setExcludedNodes(datanodes);
+ pBuilder.setRunDuringUpgrade(false);
+ final int r = Balancer.run(namenodes, pBuilder.build(), conf);
+ assertEquals(ExitStatus.SUCCESS.getExitCode(), r);
}
/**
@@ -1339,47 +1338,44 @@ public class TestBalancer {
.storageTypes(new StorageType[] { RAM_DISK, DEFAULT })
.build();
- try {
- cluster.waitActive();
- // Create few files on RAM_DISK
- final String METHOD_NAME = GenericTestUtils.getMethodName();
- final Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
- final Path path2 = new Path("/" + METHOD_NAME + ".02.dat");
-
- DistributedFileSystem fs = cluster.getFileSystem();
- DFSClient client = fs.getClient();
- DFSTestUtil.createFile(fs, path1, true,
- DEFAULT_RAM_DISK_BLOCK_SIZE, 4 * DEFAULT_RAM_DISK_BLOCK_SIZE,
- DEFAULT_RAM_DISK_BLOCK_SIZE, REPL_FACT, SEED, true);
- DFSTestUtil.createFile(fs, path2, true,
- DEFAULT_RAM_DISK_BLOCK_SIZE, 1 * DEFAULT_RAM_DISK_BLOCK_SIZE,
- DEFAULT_RAM_DISK_BLOCK_SIZE, REPL_FACT, SEED, true);
-
- // Sleep for a short time to allow the lazy writer thread to do its job
- Thread.sleep(6 * 1000);
-
- // Add another fresh DN with the same type/capacity without files on RAM_DISK
- StorageType[][] storageTypes = new StorageType[][] {{RAM_DISK, DEFAULT}};
- long[][] storageCapacities = new long[][]{{ramDiskStorageLimit, diskStorageLimit}};
- cluster.startDataNodes(conf, REPL_FACT, storageTypes, true, null,
- null, null, storageCapacities, null, false, false, false, null);
-
- cluster.triggerHeartbeats();
- Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
+ cluster.waitActive();
+ // Create few files on RAM_DISK
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ final Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
+ final Path path2 = new Path("/" + METHOD_NAME + ".02.dat");
+
+ DistributedFileSystem fs = cluster.getFileSystem();
+ DFSClient client = fs.getClient();
+ DFSTestUtil.createFile(fs, path1, true,
+ DEFAULT_RAM_DISK_BLOCK_SIZE, 4 * DEFAULT_RAM_DISK_BLOCK_SIZE,
+ DEFAULT_RAM_DISK_BLOCK_SIZE, REPL_FACT, SEED, true);
+ DFSTestUtil.createFile(fs, path2, true,
+ DEFAULT_RAM_DISK_BLOCK_SIZE, 1 * DEFAULT_RAM_DISK_BLOCK_SIZE,
+ DEFAULT_RAM_DISK_BLOCK_SIZE, REPL_FACT, SEED, true);
+
+ // Sleep for a short time to allow the lazy writer thread to do its job
+ Thread.sleep(6 * 1000);
+
+ // Add another fresh DN with the same type/capacity without files on RAM_DISK
+ StorageType[][] storageTypes = new StorageType[][] {{RAM_DISK, DEFAULT}};
+ long[][] storageCapacities = new long[][]{{ramDiskStorageLimit,
+ diskStorageLimit}};
+ cluster.startDataNodes(conf, REPL_FACT, storageTypes, true, null,
+ null, null, storageCapacities, null, false, false, false, null);
+
+ cluster.triggerHeartbeats();
+ Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
- // Run Balancer
- final BalancerParameters p = BalancerParameters.DEFAULT;
- final int r = Balancer.run(namenodes, p, conf);
+ // Run Balancer
+ final BalancerParameters p = BalancerParameters.DEFAULT;
+ final int r = Balancer.run(namenodes, p, conf);
- // Validate no RAM_DISK block should be moved
- assertEquals(ExitStatus.NO_MOVE_PROGRESS.getExitCode(), r);
+ // Validate no RAM_DISK block should be moved
+ assertEquals(ExitStatus.NO_MOVE_PROGRESS.getExitCode(), r);
- // Verify files are still on RAM_DISK
- DFSTestUtil.verifyFileReplicasOnStorageType(fs, client, path1, RAM_DISK);
- DFSTestUtil.verifyFileReplicasOnStorageType(fs, client, path2, RAM_DISK);
- } finally {
- cluster.shutdown();
- }
+ // Verify files are still on RAM_DISK
+ DFSTestUtil.verifyFileReplicasOnStorageType(fs, client, path1, RAM_DISK);
+ DFSTestUtil.verifyFileReplicasOnStorageType(fs, client, path2, RAM_DISK);
}
/**
@@ -1403,51 +1399,45 @@ public class TestBalancer {
.storageTypes(new StorageType[] { DEFAULT })
.storagesPerDatanode(1)
.build();
+ cluster.waitActive();
+ // Create a file on the single DN
+ final String METHOD_NAME = GenericTestUtils.getMethodName();
+ final Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
- try {
- cluster.waitActive();
- // Create a file on the single DN
- final String METHOD_NAME = GenericTestUtils.getMethodName();
- final Path path1 = new Path("/" + METHOD_NAME + ".01.dat");
-
- DistributedFileSystem fs = cluster.getFileSystem();
- DFSTestUtil.createFile(fs, path1, BLOCK_SIZE, BLOCK_SIZE * 2, BLOCK_SIZE,
- (short) 1, SEED);
-
- // Add another DN with the same capacity, cluster is now unbalanced
- cluster.startDataNodes(conf, 1, true, null, null);
- cluster.triggerHeartbeats();
- Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
+ DistributedFileSystem fs = cluster.getFileSystem();
+ DFSTestUtil.createFile(fs, path1, BLOCK_SIZE, BLOCK_SIZE * 2, BLOCK_SIZE,
+ (short) 1, SEED);
- // Run balancer
- final BalancerParameters p = BalancerParameters.DEFAULT;
+ // Add another DN with the same capacity, cluster is now unbalanced
+ cluster.startDataNodes(conf, 1, true, null, null);
+ cluster.triggerHeartbeats();
+ Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
- fs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
- fs.rollingUpgrade(HdfsConstants.RollingUpgradeAction.PREPARE);
- fs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
+ // Run balancer
+ final BalancerParameters p = BalancerParameters.DEFAULT;
- // Rolling upgrade should abort the balancer
- assertEquals(ExitStatus.UNFINALIZED_UPGRADE.getExitCode(),
- Balancer.run(namenodes, p, conf));
+ fs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
+ fs.rollingUpgrade(HdfsConstants.RollingUpgradeAction.PREPARE);
+ fs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
- // Should work with the -runDuringUpgrade flag.
- BalancerParameters.Builder b =
- new BalancerParameters.Builder();
- b.setRunDuringUpgrade(true);
- final BalancerParameters runDuringUpgrade = b.build();
- assertEquals(ExitStatus.SUCCESS.getExitCode(),
- Balancer.run(namenodes, runDuringUpgrade, conf));
+ // Rolling upgrade should abort the balancer
+ assertEquals(ExitStatus.UNFINALIZED_UPGRADE.getExitCode(),
+ Balancer.run(namenodes, p, conf));
- // Finalize the rolling upgrade
- fs.rollingUpgrade(HdfsConstants.RollingUpgradeAction.FINALIZE);
+ // Should work with the -runDuringUpgrade flag.
+ BalancerParameters.Builder b =
+ new BalancerParameters.Builder();
+ b.setRunDuringUpgrade(true);
+ final BalancerParameters runDuringUpgrade = b.build();
+ assertEquals(ExitStatus.SUCCESS.getExitCode(),
+ Balancer.run(namenodes, runDuringUpgrade, conf));
- // Should also work after finalization.
- assertEquals(ExitStatus.SUCCESS.getExitCode(),
- Balancer.run(namenodes, p, conf));
+ // Finalize the rolling upgrade
+ fs.rollingUpgrade(HdfsConstants.RollingUpgradeAction.FINALIZE);
- } finally {
- cluster.shutdown();
- }
+ // Should also work after finalization.
+ assertEquals(ExitStatus.SUCCESS.getExitCode(),
+ Balancer.run(namenodes, p, conf));
}
/**
@@ -1469,7 +1459,7 @@ public class TestBalancer {
conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1L);
int numOfDatanodes =2;
- final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(2)
.racks(new String[]{"/default/rack0", "/default/rack0"})
.storagesPerDatanode(2)
@@ -1480,39 +1470,33 @@ public class TestBalancer {
{100 * blockSize, 20 * blockSize},
{20 * blockSize, 100 * blockSize}})
.build();
+ cluster.waitActive();
- try {
- cluster.waitActive();
-
- //set "/bar" directory with ONE_SSD storage policy.
- DistributedFileSystem fs = cluster.getFileSystem();
- Path barDir = new Path("/bar");
- fs.mkdir(barDir,new FsPermission((short)777));
- fs.setStoragePolicy(barDir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
-
- // Insert 30 blocks. So (DN0,SSD) and (DN1,DISK) are about half full,
- // and (DN0,SSD) and (DN1,DISK) are about 15% full.
- long fileLen = 30 * blockSize;
- // fooFile has ONE_SSD policy. So
- // (DN0,SSD) and (DN1,DISK) have 2 replicas belong to same block.
- // (DN0,DISK) and (DN1,SSD) have 2 replicas belong to same block.
- Path fooFile = new Path(barDir, "foo");
- createFile(cluster, fooFile, fileLen, (short) numOfDatanodes, 0);
- // update space info
- cluster.triggerHeartbeats();
-
- BalancerParameters p = BalancerParameters.DEFAULT;
- Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
- final int r = Balancer.run(namenodes, p, conf);
-
- // Replica in (DN0,SSD) was not moved to (DN1,SSD), because (DN1,DISK)
- // already has one. Otherwise DN1 will have 2 replicas.
- // For same reason, no replicas were moved.
- assertEquals(ExitStatus.NO_MOVE_PROGRESS.getExitCode(), r);
+ //set "/bar" directory with ONE_SSD storage policy.
+ DistributedFileSystem fs = cluster.getFileSystem();
+ Path barDir = new Path("/bar");
+ fs.mkdir(barDir,new FsPermission((short)777));
+ fs.setStoragePolicy(barDir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
+
+ // Insert 30 blocks. So (DN0,SSD) and (DN1,DISK) are about half full,
+ // and (DN0,SSD) and (DN1,DISK) are about 15% full.
+ long fileLen = 30 * blockSize;
+ // fooFile has ONE_SSD policy. So
+ // (DN0,SSD) and (DN1,DISK) have 2 replicas belong to same block.
+ // (DN0,DISK) and (DN1,SSD) have 2 replicas belong to same block.
+ Path fooFile = new Path(barDir, "foo");
+ createFile(cluster, fooFile, fileLen, (short) numOfDatanodes, 0);
+ // update space info
+ cluster.triggerHeartbeats();
+
+ BalancerParameters p = BalancerParameters.DEFAULT;
+ Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
+ final int r = Balancer.run(namenodes, p, conf);
- } finally {
- cluster.shutdown();
- }
+ // Replica in (DN0,SSD) was not moved to (DN1,SSD), because (DN1,DISK)
+ // already has one. Otherwise DN1 will have 2 replicas.
+ // For same reason, no replicas were moved.
+ assertEquals(ExitStatus.NO_MOVE_PROGRESS.getExitCode(), r);
}
/**
@@ -1543,50 +1527,46 @@ public class TestBalancer {
int numOfDatanodes = capacities.length;
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(capacities.length)
.racks(racks).simulatedCapacities(capacities).build();
- try {
- cluster.waitActive();
- client = NameNodeProxies.createProxy(conf,
- cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy();
-
- long totalCapacity = sum(capacities);
-
- // fill up the cluster to be 30% full
- final long totalUsedSpace = totalCapacity * 3 / 10;
- createFile(cluster, filePath, totalUsedSpace / numOfDatanodes,
- (short) numOfDatanodes, 0);
- // start up an empty node with the same capacity and on the same rack
- cluster.startDataNodes(conf, 1, true, null, new String[] { newRack },
- new long[] { newCapacity });
-
- // Case1: Simulate first balancer by creating 'balancer.id' file. It
- // will keep this file until the balancing operation is completed.
- FileSystem fs = cluster.getFileSystem(0);
- final FSDataOutputStream out = fs
- .create(Balancer.BALANCER_ID_PATH, false);
- out.writeBytes(InetAddress.getLocalHost().getHostName());
- out.hflush();
- assertTrue("'balancer.id' file doesn't exist!",
- fs.exists(Balancer.BALANCER_ID_PATH));
-
- // start second balancer
- final String[] args = { "-policy", "datanode" };
- final Tool tool = new Cli();
- tool.setConf(conf);
- int exitCode = tool.run(args); // start balancing
- assertEquals("Exit status code mismatches",
- ExitStatus.IO_EXCEPTION.getExitCode(), exitCode);
-
- // Case2: Release lease so that another balancer would be able to
- // perform balancing.
- out.close();
- assertTrue("'balancer.id' file doesn't exist!",
- fs.exists(Balancer.BALANCER_ID_PATH));
- exitCode = tool.run(args); // start balancing
- assertEquals("Exit status code mismatches",
- ExitStatus.SUCCESS.getExitCode(), exitCode);
- } finally {
- cluster.shutdown();
- }
+ cluster.waitActive();
+ client = NameNodeProxies.createProxy(conf,
+ cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy();
+
+ long totalCapacity = sum(capacities);
+
+ // fill up the cluster to be 30% full
+ final long totalUsedSpace = totalCapacity * 3 / 10;
+ createFile(cluster, filePath, totalUsedSpace / numOfDatanodes,
+ (short) numOfDatanodes, 0);
+ // start up an empty node with the same capacity and on the same rack
+ cluster.startDataNodes(conf, 1, true, null, new String[] { newRack },
+ new long[] { newCapacity });
+
+ // Case1: Simulate first balancer by creating 'balancer.id' file. It
+ // will keep this file until the balancing operation is completed.
+ FileSystem fs = cluster.getFileSystem(0);
+ final FSDataOutputStream out = fs
+ .create(Balancer.BALANCER_ID_PATH, false);
+ out.writeBytes(InetAddress.getLocalHost().getHostName());
+ out.hflush();
+ assertTrue("'balancer.id' file doesn't exist!",
+ fs.exists(Balancer.BALANCER_ID_PATH));
+
+ // start second balancer
+ final String[] args = { "-policy", "datanode" };
+ final Tool tool = new Cli();
+ tool.setConf(conf);
+ int exitCode = tool.run(args); // start balancing
+ assertEquals("Exit status code mismatches",
+ ExitStatus.IO_EXCEPTION.getExitCode(), exitCode);
+
+ // Case2: Release lease so that another balancer would be able to
+ // perform balancing.
+ out.close();
+ assertTrue("'balancer.id' file doesn't exist!",
+ fs.exists(Balancer.BALANCER_ID_PATH));
+ exitCode = tool.run(args); // start balancing
+ assertEquals("Exit status code mismatches",
+ ExitStatus.SUCCESS.getExitCode(), exitCode);
}
/** Balancer should not move blocks with size < minBlockSize. */
@@ -1606,101 +1586,97 @@ public class TestBalancer {
.simulatedCapacities(capacities)
.build();
final DistributedFileSystem dfs = cluster.getFileSystem();
+ cluster.waitActive();
+ client = NameNodeProxies.createProxy(conf, dfs.getUri(),
+ ClientProtocol.class).getProxy();
- try {
- cluster.waitActive();
- client = NameNodeProxies.createProxy(conf, dfs.getUri(),
- ClientProtocol.class).getProxy();
-
- // fill up the cluster to be 80% full
- for(int i = 0; i < lengths.length; i++) {
- final long size = lengths[i];
- final Path p = new Path("/file" + i + "_size" + size);
- try(final OutputStream out = dfs.create(p)) {
- for(int j = 0; j < size; j++) {
- out.write(j);
- }
+ // fill up the cluster to be 80% full
+ for(int i = 0; i < lengths.length; i++) {
+ final long size = lengths[i];
+ final Path p = new Path("/file" + i + "_size" + size);
+ try(final OutputStream out = dfs.create(p)) {
+ for(int j = 0; j < size; j++) {
+ out.write(j);
}
}
-
- // start up an empty node with the same capacity
- cluster.startDataNodes(conf, capacities.length, true, null, null, capacities);
- LOG.info("capacities = " + Arrays.toString(capacities));
- LOG.info("totalUsedSpace= " + totalUsed);
- LOG.info("lengths = " + Arrays.toString(lengths) + ", #=" + lengths.length);
- waitForHeartBeat(totalUsed, 2*capacities[0]*capacities.length, client, cluster);
-
- final Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
-
- { // run Balancer with min-block-size=50
- BalancerParameters.Builder b =
- new BalancerParameters.Builder();
- b.setBalancingPolicy(BalancingPolicy.Node.INSTANCE);
- b.setThreshold(1);
- final BalancerParameters p = b.build();
-
- conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 50);
- final int r = Balancer.run(namenodes, p, conf);
- assertEquals(ExitStatus.NO_MOVE_PROGRESS.getExitCode(), r);
- }
-
- conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1);
+ }
- { // run Balancer with empty nodes as source nodes
- final Set<String> sourceNodes = new HashSet<>();
- final List<DataNode> datanodes = cluster.getDataNodes();
- for(int i = capacities.length; i < datanodes.size(); i++) {
- sourceNodes.add(datanodes.get(i).getDisplayName());
- }
- BalancerParameters.Builder b =
- new BalancerParameters.Builder();
- b.setBalancingPolicy(BalancingPolicy.Node.INSTANCE);
- b.setThreshold(1);
- b.setSourceNodes(sourceNodes);
- final BalancerParameters p = b.build();
-
- conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 50);
- final int r = Balancer.run(namenodes, p, conf);
- assertEquals(ExitStatus.NO_MOVE_BLOCK.getExitCode(), r);
- }
+ // start up an empty node with the same capacity
+ cluster.startDataNodes(conf, capacities.length, true, null, null, capacities);
+ LOG.info("capacities = " + Arrays.toString(capacities));
+ LOG.info("totalUsedSpace= " + totalUsed);
+ LOG.info("lengths = " + Arrays.toString(lengths) + ", #=" + lengths.length);
+ waitForHeartBeat(totalUsed, 2*capacities[0]*capacities.length, client, cluster);
+
+ final Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
+
+ { // run Balancer with min-block-size=50
+ BalancerParameters.Builder b =
+ new BalancerParameters.Builder();
+ b.setBalancingPolicy(BalancingPolicy.Node.INSTANCE);
+ b.setThreshold(1);
+ final BalancerParameters p = b.build();
- { // run Balancer with a filled node as a source node
- final Set<String> sourceNodes = new HashSet<>();
- final List<DataNode> datanodes = cluster.getDataNodes();
- sourceNodes.add(datanodes.get(0).getDisplayName());
- BalancerParameters.Builder b =
- new BalancerParameters.Builder();
- b.setBalancingPolicy(BalancingPolicy.Node.INSTANCE);
- b.setThreshold(1);
- b.setSourceNodes(sourceNodes);
- final BalancerParameters p = b.build();
-
- conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1);
- final int r = Balancer.run(namenodes, p, conf);
- assertEquals(ExitStatus.NO_MOVE_BLOCK.getExitCode(), r);
+ conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 50);
+ final int r = Balancer.run(namenodes, p, conf);
+ assertEquals(ExitStatus.NO_MOVE_PROGRESS.getExitCode(), r);
+ }
+
+ conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1);
+
+ { // run Balancer with empty nodes as source nodes
+ final Set<String> sourceNodes = new HashSet<>();
+ final List<DataNode> datanodes = cluster.getDataNodes();
+ for(int i = capacities.length; i < datanodes.size(); i++) {
+ sourceNodes.add(datanodes.get(i).getDisplayName());
}
+ BalancerParameters.Builder b =
+ new BalancerParameters.Builder();
+ b.setBalancingPolicy(BalancingPolicy.Node.INSTANCE);
+ b.setThreshold(1);
+ b.setSourceNodes(sourceNodes);
+ final BalancerParameters p = b.build();
- { // run Balancer with all filled node as source nodes
- final Set<String> sourceNodes = new HashSet<>();
- final List<DataNode> datanodes = cluster.getDataNodes();
- for(int i = 0; i < capacities.length; i++) {
- sourceNodes.add(datanodes.get(i).getDisplayName());
- }
- BalancerParameters.Builder b =
- new BalancerParameters.Builder();
- b.setBalancingPolicy(BalancingPolicy.Node.INSTANCE);
- b.setThreshold(1);
- b.setSourceNodes(sourceNodes);
- final BalancerParameters p = b.build();
-
- conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1);
- final int r = Balancer.run(namenodes, p, conf);
- assertEquals(ExitStatus.SUCCESS.getExitCode(), r);
+ conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 50);
+ final int r = Balancer.run(namenodes, p, conf);
+ assertEquals(ExitStatus.NO_MOVE_BLOCK.getExitCode(), r);
+ }
+
+ { // run Balancer with a filled node as a source node
+ final Set<String> sourceNodes = new HashSet<>();
+ final List<DataNode> datanodes = cluster.getDataNodes();
+ sourceNodes.add(datanodes.get(0).getDisplayName());
+ BalancerParameters.Builder b =
+ new BalancerParameters.Builder();
+ b.setBalancingPolicy(BalancingPolicy.Node.INSTANCE);
+ b.setThreshold(1);
+ b.setSourceNodes(sourceNodes);
+ final BalancerParameters p = b.build();
+
+ conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1);
+ final int r = Balancer.run(namenodes, p, conf);
+ assertEquals(ExitStatus.NO_MOVE_BLOCK.getExitCode(), r);
+ }
+
+ { // run Balancer with all filled node as source nodes
+ final Set<String> sourceNodes = new HashSet<>();
+ final List<DataNode> datanodes = cluster.getDataNodes();
+ for(int i = 0; i < capacities.length; i++) {
+ sourceNodes.add(datanodes.get(i).getDisplayName());
}
- } finally {
- cluster.shutdown();
+ BalancerParameters.Builder b =
+ new BalancerParameters.Builder();
+ b.setBalancingPolicy(BalancingPolicy.Node.INSTANCE);
+ b.setThreshold(1);
+ b.setSourceNodes(sourceNodes);
+ final BalancerParameters p = b.build();
+
+ conf.setLong(DFSConfigKeys.DFS_BALANCER_GETBLOCKS_MIN_BLOCK_SIZE_KEY, 1);
+ final int r = Balancer.run(namenodes, p, conf);
+ assertEquals(ExitStatus.SUCCESS.getExitCode(), r);
}
}
+
public void integrationTestWithStripedFile(Configuration conf) throws Exception {
initConfWithStripe(conf);
doTestBalancerWithStripedFile(conf);
@@ -1778,4 +1754,4 @@ public class TestBalancer {
balancerTest.testBalancer1();
balancerTest.testBalancer2();
}
-}
+}
\ No newline at end of file
[44/50] hadoop git commit: HDFS-9313. Possible NullPointerException
in BlockManager if no excess replica can be chosen. (mingma)
Posted by ji...@apache.org.
HDFS-9313. Possible NullPointerException in BlockManager if no excess replica can be chosen. (mingma)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d565480d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d565480d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d565480d
Branch: refs/heads/HDFS-8966
Commit: d565480da2f646b40c3180e1ccb2935c9863dfef
Parents: 8e05dbf
Author: Ming Ma <mi...@apache.org>
Authored: Mon Nov 2 19:36:37 2015 -0800
Committer: Ming Ma <mi...@apache.org>
Committed: Mon Nov 2 19:36:37 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../blockmanagement/BlockPlacementPolicy.java | 8 +++--
.../BlockPlacementPolicyDefault.java | 6 ++++
.../blockmanagement/TestReplicationPolicy.java | 31 ++++++++++++++++++++
4 files changed, 45 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d565480d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 19ea5c1..879c015 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2216,6 +2216,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9329. TestBootstrapStandby#testRateThrottling is flaky because fsimage
size is smaller than IO buffer size. (zhz)
+ HDFS-9313. Possible NullPointerException in BlockManager if no excess
+ replica can be chosen. (mingma)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d565480d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
index be169c3..526a5d7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
@@ -23,8 +23,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.StorageType;
@@ -33,13 +31,17 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.Node;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* This interface is used for choosing the desired number of targets
* for placing block replicas.
*/
@InterfaceAudience.Private
public abstract class BlockPlacementPolicy {
- static final Log LOG = LogFactory.getLog(BlockPlacementPolicy.class);
+ static final Logger LOG = LoggerFactory.getLogger(
+ BlockPlacementPolicy.class);
@InterfaceAudience.Private
public static class NotEnoughReplicasException extends Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d565480d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index d9b8d60..2723ed9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -981,6 +981,12 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
excessTypes);
}
firstOne = false;
+ if (cur == null) {
+ LOG.warn("No excess replica can be found. excessTypes: {}." +
+ " moreThanOne: {}. exactlyOne: {}.", excessTypes, moreThanOne,
+ exactlyOne);
+ break;
+ }
// adjust rackmap, moreThanOne, and exactlyOne
adjustSetsWithChosenReplica(rackMap, moreThanOne, exactlyOne, cur);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d565480d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index ef73001..37fcf34 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -1000,6 +1000,14 @@ public class TestReplicationPolicy extends BaseReplicationPolicyTest {
BlockStoragePolicySuite POLICY_SUITE = BlockStoragePolicySuite
.createDefaultSuite();
BlockStoragePolicy storagePolicy = POLICY_SUITE.getDefaultPolicy();
+ DatanodeStorageInfo excessSSD = DFSTestUtil.createDatanodeStorageInfo(
+ "Storage-excess-SSD-ID", "localhost",
+ storages[0].getDatanodeDescriptor().getNetworkLocation(),
+ "foo.com", StorageType.SSD);
+ updateHeartbeatWithUsage(excessSSD.getDatanodeDescriptor(),
+ 2* HdfsServerConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
+ 2* HdfsServerConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0L, 0L, 0,
+ 0);
// use delete hint case.
@@ -1022,6 +1030,29 @@ public class TestReplicationPolicy extends BaseReplicationPolicyTest {
excessReplicas = replicator.chooseReplicasToDelete(nonExcess, 3,
excessTypes, storages[3].getDatanodeDescriptor(), null);
assertTrue(excessReplicas.contains(excessStorage));
+
+
+ // The block was initially created on excessSSD(rack r1),
+ // storages[4](rack r3) and storages[5](rack r3) with
+ // ONESSD_STORAGE_POLICY_NAME storage policy.
+ // Right after balancer moves the block from storages[5] to
+ // storages[3](rack r2), the application changes the storage policy from
+ // ONESSD_STORAGE_POLICY_NAME to HOT_STORAGE_POLICY_ID. In this case,
+ // no replica can be chosen as the excessive replica as
+ // chooseReplicasToDelete only considers storages[4] and storages[5] that
+ // are the same rack. But neither's storage type is SSD.
+ // TODO BlockPlacementPolicyDefault should be able to delete excessSSD.
+ nonExcess.clear();
+ nonExcess.add(excessSSD);
+ nonExcess.add(storages[3]);
+ nonExcess.add(storages[4]);
+ nonExcess.add(storages[5]);
+ excessTypes = storagePolicy.chooseExcess((short) 3,
+ DatanodeStorageInfo.toStorageTypes(nonExcess));
+ excessReplicas = replicator.chooseReplicasToDelete(nonExcess, 3,
+ excessTypes, storages[3].getDatanodeDescriptor(),
+ storages[5].getDatanodeDescriptor());
+ assertTrue(excessReplicas.size() == 0);
}
@Test
[28/50] hadoop git commit: Addendum to MAPREDUCE-6451
Posted by ji...@apache.org.
Addendum to MAPREDUCE-6451
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b24fe064
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b24fe064
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b24fe064
Branch: refs/heads/HDFS-8966
Commit: b24fe0648348d325d14931f80cee8a170fb3358a
Parents: 2868ca0
Author: Kihwal Lee <ki...@apache.org>
Authored: Fri Oct 30 16:05:23 2015 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Fri Oct 30 16:05:23 2015 -0500
----------------------------------------------------------------------
.../mapred/lib/DynamicInputChunkContext.java | 113 +++++++++++++++++++
1 file changed, 113 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b24fe064/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/lib/DynamicInputChunkContext.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/lib/DynamicInputChunkContext.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/lib/DynamicInputChunkContext.java
new file mode 100644
index 0000000..043ff1c
--- /dev/null
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/lib/DynamicInputChunkContext.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.tools.mapred.lib;
+
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.tools.DistCpConstants;
+
+import java.io.IOException;
+
+/**
+ * Class to initialize the DynamicInputChunk invariants.
+ */
+class DynamicInputChunkContext<K, V> {
+
+ private static Log LOG = LogFactory.getLog(DynamicInputChunkContext.class);
+ private Configuration configuration;
+ private Path chunkRootPath = null;
+ private String chunkFilePrefix;
+ private FileSystem fs;
+ private int numChunksLeft = -1; // Un-initialized before 1st dir-scan.
+
+ public DynamicInputChunkContext(Configuration config)
+ throws IOException {
+ this.configuration = config;
+ Path listingFilePath = new Path(getListingFilePath(configuration));
+ chunkRootPath = new Path(listingFilePath.getParent(), "chunkDir");
+ fs = chunkRootPath.getFileSystem(configuration);
+ chunkFilePrefix = listingFilePath.getName() + ".chunk.";
+ }
+
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ public Path getChunkRootPath() {
+ return chunkRootPath;
+ }
+
+ public String getChunkFilePrefix() {
+ return chunkFilePrefix;
+ }
+
+ public FileSystem getFs() {
+ return fs;
+ }
+
+ private static String getListingFilePath(Configuration configuration) {
+ final String listingFileString = configuration.get(
+ DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, "");
+ assert !listingFileString.equals("") : "Listing file not found.";
+ return listingFileString;
+ }
+
+ public int getNumChunksLeft() {
+ return numChunksLeft;
+ }
+
+ public DynamicInputChunk acquire(TaskAttemptContext taskAttemptContext)
+ throws IOException, InterruptedException {
+
+ String taskId
+ = taskAttemptContext.getTaskAttemptID().getTaskID().toString();
+ Path acquiredFilePath = new Path(getChunkRootPath(), taskId);
+
+ if (fs.exists(acquiredFilePath)) {
+ LOG.info("Acquiring pre-assigned chunk: " + acquiredFilePath);
+ return new DynamicInputChunk(acquiredFilePath, taskAttemptContext, this);
+ }
+
+ for (FileStatus chunkFile : getListOfChunkFiles()) {
+ if (fs.rename(chunkFile.getPath(), acquiredFilePath)) {
+ LOG.info(taskId + " acquired " + chunkFile.getPath());
+ return new DynamicInputChunk(acquiredFilePath, taskAttemptContext,
+ this);
+ }
+ }
+ return null;
+ }
+
+ public DynamicInputChunk createChunkForWrite(String chunkId)
+ throws IOException {
+ return new DynamicInputChunk(chunkId, this);
+ }
+
+ public FileStatus [] getListOfChunkFiles() throws IOException {
+ Path chunkFilePattern = new Path(chunkRootPath, chunkFilePrefix + "*");
+ FileStatus chunkFiles[] = fs.globStatus(chunkFilePattern);
+ numChunksLeft = chunkFiles.length;
+ return chunkFiles;
+ }
+}
[29/50] hadoop git commit: Updated the 2.6.2 final release date.
Posted by ji...@apache.org.
Updated the 2.6.2 final release date.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a4a6b5b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a4a6b5b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a4a6b5b4
Branch: refs/heads/HDFS-8966
Commit: a4a6b5b4b470b1e7a3c5e2d38433429c455bc709
Parents: b24fe06
Author: Sangjin Lee <sj...@apache.org>
Authored: Fri Oct 30 18:47:16 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Fri Oct 30 18:47:16 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +-
hadoop-mapreduce-project/CHANGES.txt | 2 +-
hadoop-yarn-project/CHANGES.txt | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4a6b5b4/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index a1409f8..2560fe5 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -2170,7 +2170,7 @@ Release 2.6.3 - UNRELEASED
BUG FIXES
-Release 2.6.2 - 2015-10-21
+Release 2.6.2 - 2015-10-28
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4a6b5b4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 17df171..5a61eed 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -3310,7 +3310,7 @@ Release 2.6.3 - UNRELEASED
BUG FIXES
-Release 2.6.2 - 2015-10-21
+Release 2.6.2 - 2015-10-28
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4a6b5b4/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index e999659..4d6dcb8 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -913,7 +913,7 @@ Release 2.6.3 - UNRELEASED
MAPREDUCE-6528. Memory leak for HistoryFileManager.getJobSummary()
(Junping Du via jlowe)
-Release 2.6.2 - 2015-10-21
+Release 2.6.2 - 2015-10-28
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4a6b5b4/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cc8f5f3..1040f45 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1887,7 +1887,7 @@ Release 2.6.3 - UNRELEASED
BUG FIXES
-Release 2.6.2 - 2015-10-21
+Release 2.6.2 - 2015-10-28
INCOMPATIBLE CHANGES
[09/50] hadoop git commit: HDFS-9229. Expose size of NameNode
directory as a metric. Contributed by Surendra Singh Lilhore.
Posted by ji...@apache.org.
HDFS-9229. Expose size of NameNode directory as a metric. Contributed by Surendra Singh Lilhore.
Change-Id: I985627a5d1400249d72d084283ef366d5ac6e07b
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8def51a7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8def51a7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8def51a7
Branch: refs/heads/HDFS-8966
Commit: 8def51a708e5de8a57689b8c9b3fd034cfd2cd78
Parents: cf953b6
Author: Zhe Zhang <zh...@apache.org>
Authored: Thu Oct 29 11:14:00 2015 -0700
Committer: Zhe Zhang <zh...@apache.org>
Committed: Thu Oct 29 11:23:08 2015 -0700
----------------------------------------------------------------------
.../hadoop-common/src/site/markdown/Metrics.md | 1 +
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop/hdfs/server/common/Storage.java | 15 +++++
.../hadoop/hdfs/server/namenode/FSImage.java | 4 ++
.../hdfs/server/namenode/FSNamesystem.java | 5 ++
.../hadoop/hdfs/server/namenode/NNStorage.java | 25 ++++++++
.../hdfs/server/namenode/NameNodeMXBean.java | 5 ++
.../hdfs/server/namenode/ha/EditLogTailer.java | 4 +-
.../server/namenode/TestNameNodeMXBean.java | 62 +++++++++++++++++++-
9 files changed, 122 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8def51a7/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index 3e01fe3..2e24aba 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -240,6 +240,7 @@ Each metrics record contains tags such as HAState and Hostname as additional inf
| `LockQueueLength` | Number of threads waiting to acquire FSNameSystem lock |
| `TotalSyncCount` | Total number of sync operations performed by edit log |
| `TotalSyncTimes` | Total number of milliseconds spent by various edit logs in sync operation|
+| `NameDirSize` | NameNode name directories size in bytes |
JournalNode
-----------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8def51a7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index a87fe76..36a70b8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1608,6 +1608,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8545. Refactor FS#getUsed() to use ContentSummary and add an API to fetch
the total file length from a specific path (J.Andreina via vinayakumarb)
+ HDFS-9229. Expose size of NameNode directory as a metric.
+ (Surendra Singh Lilhore via zhz)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8def51a7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
index c630206..7b4b571 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Storage.java
@@ -30,6 +30,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Properties;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -310,6 +311,20 @@ public abstract class Storage extends StorageInfo {
return dirType;
}
+ /**
+ * Get storage directory size.
+ */
+ public long getDirecorySize() {
+ try {
+ if (!isShared() && root != null && root.exists()) {
+ return FileUtils.sizeOfDirectory(root);
+ }
+ } catch (Exception e) {
+ LOG.warn("Failed to get directory size :" + root, e);
+ }
+ return 0;
+ }
+
public void read(File from, Storage storage) throws IOException {
Properties props = readPropertiesFile(from);
storage.setFieldsFromProperties(props, this);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8def51a7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
index 93dc097..341dd98 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
@@ -1064,6 +1064,8 @@ public class FSImage implements Closeable {
} finally {
removeFromCheckpointing(imageTxId);
}
+ //Update NameDirSize Metric
+ getStorage().updateNameDirSize();
}
/**
@@ -1244,6 +1246,8 @@ public class FSImage implements Closeable {
// we won't miss this log segment on a restart if the edits directories
// go missing.
storage.writeTransactionIdFileToStorage(getEditLog().getCurSegmentTxId());
+ //Update NameDirSize Metric
+ getStorage().updateNameDirSize();
return new CheckpointSignature(this);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8def51a7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 969677a..29ee087 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -6402,6 +6402,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return VersionInfo.getVersion();
}
+ @Override // NameNodeStatusMXBean
+ public String getNameDirSize() {
+ return getFSImage().getStorage().getNNDirectorySize();
+ }
+
/**
* Verifies that the given identifier and password are valid and match.
* @param identifier Token identifier.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8def51a7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
index d872c03..9b63e72 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
@@ -29,6 +29,7 @@ import java.util.EnumSet;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import java.util.UUID;
import java.util.concurrent.CopyOnWriteArrayList;
@@ -52,6 +53,7 @@ import org.apache.hadoop.hdfs.util.PersistentLongFile;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.DNS;
import org.apache.hadoop.util.Time;
+import org.mortbay.util.ajax.JSON;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -149,6 +151,11 @@ public class NNStorage extends Storage implements Closeable,
private HashMap<String, String> deprecatedProperties;
/**
+ * Name directories size for metric.
+ */
+ private Map<String, Long> nameDirSizeMap = new HashMap<>();
+
+ /**
* Construct the NNStorage.
* @param conf Namenode configuration.
* @param imageDirs Directories the image can be stored in.
@@ -166,6 +173,8 @@ public class NNStorage extends Storage implements Closeable,
setStorageDirectories(imageDirs,
Lists.newArrayList(editsDirs),
FSNamesystem.getSharedEditsDirs(conf));
+ //Update NameDirSize metric value after NN start
+ updateNameDirSize();
}
@Override // Storage
@@ -1075,4 +1084,20 @@ public class NNStorage extends Storage implements Closeable,
getBlockPoolID(),
getCTime());
}
+
+ public String getNNDirectorySize() {
+ return JSON.toString(nameDirSizeMap);
+ }
+
+ public void updateNameDirSize() {
+ Map<String, Long> nnDirSizeMap = new HashMap<>();
+ for (Iterator<StorageDirectory> it = dirIterator(); it.hasNext();) {
+ StorageDirectory sd = it.next();
+ if (!sd.isShared()) {
+ nnDirSizeMap.put(sd.getRoot().getAbsolutePath(), sd.getDirecorySize());
+ }
+ }
+ nameDirSizeMap.clear();
+ nameDirSizeMap.putAll(nnDirSizeMap);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8def51a7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java
index 9dcef89..d127c98 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java
@@ -272,4 +272,9 @@ public interface NameNodeMXBean {
*/
public Map<String, Integer> getDistinctVersions();
+ /**
+ * Get namenode directory size.
+ */
+ String getNameDirSize();
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8def51a7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
index 59f90a6..6e60dba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
@@ -372,6 +372,8 @@ public class EditLogTailer {
} finally {
namesystem.cpUnlock();
}
+ //Update NameDirSize Metric
+ namesystem.getFSImage().getStorage().updateNameDirSize();
} catch (EditLogInputException elie) {
LOG.warn("Error while reading edits from disk. Will try again.", elie);
} catch (InterruptedException ie) {
@@ -463,4 +465,4 @@ public class EditLogTailer {
return cachedActiveProxy;
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8def51a7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
index 559aae6..5c865e1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
@@ -26,12 +26,16 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.hdfs.server.namenode.top.TopConf;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.io.nativeio.NativeIO.POSIX.NoMlockCacheManipulator;
+import org.apache.hadoop.net.ServerSocketUtil;
import org.apache.hadoop.util.VersionInfo;
import org.codehaus.jackson.map.ObjectMapper;
import org.junit.Test;
@@ -40,6 +44,7 @@ import org.mortbay.util.ajax.JSON;
import javax.management.MBeanServer;
import javax.management.ObjectName;
import java.io.File;
+import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.net.URI;
import java.util.Collection;
@@ -186,7 +191,7 @@ public class TestNameNodeMXBean {
}
assertEquals(2, statusMap.get("active").size());
assertEquals(0, statusMap.get("failed").size());
-
+
// This will cause the first dir to fail.
File failedNameDir = new File(nameDirUris.iterator().next());
assertEquals(0, FileUtil.chmod(
@@ -412,4 +417,59 @@ public class TestNameNodeMXBean {
}
}
}
+
+ @Test(timeout = 120000)
+ public void testNNDirectorySize() throws Exception{
+ Configuration conf = new Configuration();
+ conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+ // Have to specify IPC ports so the NNs can talk to each other.
+ MiniDFSNNTopology topology = new MiniDFSNNTopology()
+ .addNameservice(new MiniDFSNNTopology.NSConf("ns1")
+ .addNN(new MiniDFSNNTopology.NNConf("nn1")
+ .setIpcPort(ServerSocketUtil.getPort(0, 100)))
+ .addNN(new MiniDFSNNTopology.NNConf("nn2")
+ .setIpcPort(ServerSocketUtil.getPort(0, 100))));
+
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(topology).numDataNodes(0)
+ .build();
+ FileSystem fs = null;
+ try {
+ cluster.waitActive();
+
+ FSNamesystem nn0 = cluster.getNamesystem(0);
+ FSNamesystem nn1 = cluster.getNamesystem(1);
+ checkNNDirSize(cluster.getNameDirs(0), nn0.getNameDirSize());
+ checkNNDirSize(cluster.getNameDirs(1), nn1.getNameDirSize());
+ cluster.transitionToActive(0);
+ fs = cluster.getFileSystem(0);
+ DFSTestUtil.createFile(fs, new Path("/file"), 0, (short) 1, 0L);
+
+ //rollEditLog
+ HATestUtil.waitForStandbyToCatchUp(cluster.getNameNode(0),
+ cluster.getNameNode(1));
+ checkNNDirSize(cluster.getNameDirs(0), nn0.getNameDirSize());
+ checkNNDirSize(cluster.getNameDirs(1), nn1.getNameDirSize());
+
+ //Test metric after call saveNamespace
+ DFSTestUtil.createFile(fs, new Path("/file"), 0, (short) 1, 0L);
+ nn0.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ nn0.saveNamespace(0, 0);
+ checkNNDirSize(cluster.getNameDirs(0), nn0.getNameDirSize());
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private void checkNNDirSize(Collection<URI> nameDirUris, String metric){
+ Map<String, Long> nnDirMap =
+ (Map<String, Long>) JSON.parse(metric);
+ assertEquals(nameDirUris.size(), nnDirMap.size());
+ for (URI dirUrl : nameDirUris) {
+ File dir = new File(dirUrl);
+ assertEquals(nnDirMap.get(dir.getAbsolutePath()).longValue(),
+ FileUtils.sizeOfDirectory(dir));
+ }
+ }
}
[13/50] hadoop git commit: YARN-4183. Enabling generic application
history forces every job to get a timeline service delegation token (jeagles)
Posted by ji...@apache.org.
YARN-4183. Enabling generic application history forces every job to get a timeline service delegation token (jeagles)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c293c589
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c293c589
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c293c589
Branch: refs/heads/HDFS-8966
Commit: c293c58954cdab25c8c69418b0e839883b563fa4
Parents: 58d1df5
Author: Jonathan Eagles <je...@yahoo-inc.com>
Authored: Thu Oct 29 16:41:10 2015 -0500
Committer: Jonathan Eagles <je...@yahoo-inc.com>
Committed: Thu Oct 29 16:41:10 2015 -0500
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../server/resourcemanager/metrics/SystemMetricsPublisher.java | 4 ++--
.../resourcemanager/metrics/TestSystemMetricsPublisher.java | 2 +-
3 files changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c293c589/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index baa66c1..9fb2a9f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1138,6 +1138,9 @@ Release 2.7.2 - UNRELEASED
YARN-2902. Killing a container that is localizing can orphan resources in
the DOWNLOADING state (Varun Saxena via jlowe)
+ YARN-4183. Enabling generic application history forces every job to get a
+ timeline service delegation token (Mit Desai via jeagles)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c293c589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
index 0f09735..c73e835 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
@@ -77,8 +77,8 @@ public class SystemMetricsPublisher extends CompositeService {
@Override
protected void serviceInit(Configuration conf) throws Exception {
publishSystemMetrics =
- conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
- YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED) &&
+ conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
+ YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED) &&
conf.getBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED,
YarnConfiguration.DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c293c589/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index 98daae7..e2fbf8d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -73,7 +73,7 @@ public class TestSystemMetricsPublisher {
@BeforeClass
public static void setup() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
- conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true);
conf.setBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED, true);
conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
MemoryTimelineStore.class, TimelineStore.class);
[24/50] hadoop git commit: MAPREDUCE-6528. Memory leak for
HistoryFileManager.getJobSummary(). Contributed by Junping Du
Posted by ji...@apache.org.
MAPREDUCE-6528. Memory leak for HistoryFileManager.getJobSummary(). Contributed by Junping Du
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6344b6a7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6344b6a7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6344b6a7
Branch: refs/heads/HDFS-8966
Commit: 6344b6a7694c70f296392b6462dba452ff762109
Parents: 6ae9efa
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Oct 30 15:31:38 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Oct 30 15:31:38 2015 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 6 ++++++
.../hadoop/mapreduce/v2/hs/HistoryFileManager.java | 13 ++++++++++---
2 files changed, 16 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6344b6a7/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 22f9e89..32be987 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -671,6 +671,9 @@ Release 2.7.2 - UNRELEASED
avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED state.
(zhihai xu via devaraj)
+ MAPREDUCE-6528. Memory leak for HistoryFileManager.getJobSummary()
+ (Junping Du via jlowe)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
@@ -904,6 +907,9 @@ Release 2.6.3 - UNRELEASED
avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED state.
(zhihai xu via devaraj)
+ MAPREDUCE-6528. Memory leak for HistoryFileManager.getJobSummary()
+ (Junping Du via jlowe)
+
Release 2.6.2 - 2015-10-21
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6344b6a7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index f0786da..b221961 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -966,9 +966,16 @@ public class HistoryFileManager extends AbstractService {
private String getJobSummary(FileContext fc, Path path) throws IOException {
Path qPath = fc.makeQualified(path);
- FSDataInputStream in = fc.open(qPath);
- String jobSummaryString = in.readUTF();
- in.close();
+ FSDataInputStream in = null;
+ String jobSummaryString = null;
+ try {
+ in = fc.open(qPath);
+ jobSummaryString = in.readUTF();
+ } finally {
+ if (in != null) {
+ in.close();
+ }
+ }
return jobSummaryString;
}
[25/50] hadoop git commit: HADOOP-11919. Empty commit to test github
integration.
Posted by ji...@apache.org.
HADOOP-11919. Empty commit to test github integration.
closes apache/hadoop#40
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/45d39679
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/45d39679
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/45d39679
Branch: refs/heads/HDFS-8966
Commit: 45d39679dfc47e66fe2d69da09689fa62017637f
Parents: 6344b6a
Author: Owen O'Malley <om...@apache.org>
Authored: Fri Oct 30 10:04:30 2015 -0700
Committer: Owen O'Malley <om...@apache.org>
Committed: Fri Oct 30 10:21:03 2015 -0700
----------------------------------------------------------------------
----------------------------------------------------------------------
[14/50] hadoop git commit: YARN-4127. RM fail with noAuth error if
switched from failover to non-failover. Contributed by Varun Saxena
Posted by ji...@apache.org.
YARN-4127. RM fail with noAuth error if switched from failover to non-failover. Contributed by Varun Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e5b1733e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e5b1733e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e5b1733e
Branch: refs/heads/HDFS-8966
Commit: e5b1733e049dc0f1859b93618354e049a0efdc4a
Parents: c293c58
Author: Jian He <ji...@apache.org>
Authored: Thu Oct 29 15:42:57 2015 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Oct 29 15:42:57 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../recovery/ZKRMStateStore.java | 25 ++++---
.../recovery/TestZKRMStateStore.java | 68 ++++++++++++++++++++
3 files changed, 86 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5b1733e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9fb2a9f..0da15bd 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1039,6 +1039,9 @@ Release 2.8.0 - UNRELEASED
YARN-4288. Fixed RMProxy to retry on IOException from local host.
(Junping Du via jianhe)
+ YARN-4127. RM fail with noAuth error if switched from failover to non-failover.
+ (Varun Saxena via jianhe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5b1733e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
index 0550087..a86b60c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
@@ -282,8 +282,9 @@ public class ZKRMStateStore extends RMStateStore {
// ensure root dirs exist
createRootDirRecursively(znodeWorkingPath);
create(zkRootNodePath);
- if (HAUtil.isHAEnabled(getConfig())){
- fence();
+ setRootNodeAcls();
+ delete(fencingNodePath);
+ if (HAUtil.isHAEnabled(getConfig())) {
verifyActiveStatusThread = new VerifyActiveStatusThread();
verifyActiveStatusThread.start();
}
@@ -309,16 +310,19 @@ public class ZKRMStateStore extends RMStateStore {
LOG.debug(builder.toString());
}
- private synchronized void fence() throws Exception {
- if (LOG.isTraceEnabled()) {
- logRootNodeAcls("Before fencing\n");
+ private void setRootNodeAcls() throws Exception {
+ if (LOG.isDebugEnabled()) {
+ logRootNodeAcls("Before setting ACLs'\n");
}
- curatorFramework.setACL().withACL(zkRootNodeAcl).forPath(zkRootNodePath);
- delete(fencingNodePath);
+ if (HAUtil.isHAEnabled(getConfig())) {
+ curatorFramework.setACL().withACL(zkRootNodeAcl).forPath(zkRootNodePath);
+ } else {
+ curatorFramework.setACL().withACL(zkAcl).forPath(zkRootNodePath);
+ }
- if (LOG.isTraceEnabled()) {
- logRootNodeAcls("After fencing\n");
+ if (LOG.isDebugEnabled()) {
+ logRootNodeAcls("After setting ACLs'\n");
}
}
@@ -933,7 +937,8 @@ public class ZKRMStateStore extends RMStateStore {
return curatorFramework.getData().forPath(path);
}
- private List<ACL> getACL(final String path) throws Exception {
+ @VisibleForTesting
+ List<ACL> getACL(final String path) throws Exception {
return curatorFramework.getACL().forPath(path);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5b1733e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
index df96653..9a12ca8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
@@ -54,6 +54,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptS
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooDefs.Perms;
+import org.apache.zookeeper.data.ACL;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -65,6 +67,8 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
+import java.util.List;
+
import javax.crypto.SecretKey;
public class TestZKRMStateStore extends RMStateStoreTestBase {
@@ -248,6 +252,70 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
return conf;
}
+ private static boolean verifyZKACL(String id, String scheme, int perm,
+ List<ACL> acls) {
+ for (ACL acl : acls) {
+ if (acl.getId().getScheme().equals(scheme) &&
+ acl.getId().getId().startsWith(id) &&
+ acl.getPerms() == perm) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Test if RM can successfully start in HA disabled mode if it was previously
+ * running in HA enabled mode. And then start it in HA mode after running it
+ * with HA disabled. NoAuth Exception should not be sent by zookeeper and RM
+ * should start successfully.
+ */
+ @Test
+ public void testZKRootPathAcls() throws Exception {
+ StateChangeRequestInfo req = new StateChangeRequestInfo(
+ HAServiceProtocol.RequestSource.REQUEST_BY_USER);
+ String rootPath =
+ YarnConfiguration.DEFAULT_ZK_RM_STATE_STORE_PARENT_PATH + "/" +
+ ZKRMStateStore.ROOT_ZNODE_NAME;
+
+ // Start RM with HA enabled
+ Configuration conf = createHARMConf("rm1,rm2", "rm1", 1234);
+ conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
+ ResourceManager rm = new MockRM(conf);
+ rm.start();
+ rm.getRMContext().getRMAdminService().transitionToActive(req);
+ List<ACL> acls =
+ ((ZKRMStateStore)rm.getRMContext().getStateStore()).getACL(rootPath);
+ assertEquals(acls.size(), 2);
+ // CREATE and DELETE permissions for root node based on RM ID
+ verifyZKACL("digest", "localhost", Perms.CREATE | Perms.DELETE, acls);
+ verifyZKACL(
+ "world", "anyone", Perms.ALL ^ (Perms.CREATE | Perms.DELETE), acls);
+ rm.close();
+
+ // Now start RM with HA disabled. NoAuth Exception should not be thrown.
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, false);
+ rm = new MockRM(conf);
+ rm.start();
+ rm.getRMContext().getRMAdminService().transitionToActive(req);
+ acls = ((ZKRMStateStore)rm.getRMContext().getStateStore()).getACL(rootPath);
+ assertEquals(acls.size(), 1);
+ verifyZKACL("world", "anyone", Perms.ALL, acls);
+ rm.close();
+
+ // Start RM with HA enabled.
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ rm = new MockRM(conf);
+ rm.start();
+ rm.getRMContext().getRMAdminService().transitionToActive(req);
+ acls = ((ZKRMStateStore)rm.getRMContext().getStateStore()).getACL(rootPath);
+ assertEquals(acls.size(), 2);
+ verifyZKACL("digest", "localhost", Perms.CREATE | Perms.DELETE, acls);
+ verifyZKACL(
+ "world", "anyone", Perms.ALL ^ (Perms.CREATE | Perms.DELETE), acls);
+ rm.close();
+ }
+
@SuppressWarnings("unchecked")
@Test
public void testFencing() throws Exception {
[32/50] hadoop git commit: HDFS-9343. Empty caller context considered
invalid. (Contributed by Mingliang Liu)
Posted by ji...@apache.org.
HDFS-9343. Empty caller context considered invalid. (Contributed by Mingliang Liu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3cde6931
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3cde6931
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3cde6931
Branch: refs/heads/HDFS-8966
Commit: 3cde6931cb5055a9d92503f4ecefa35571e7b07f
Parents: 7fd6416
Author: Arpit Agarwal <ar...@apache.org>
Authored: Sun Nov 1 15:35:02 2015 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Sun Nov 1 15:35:02 2015 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/ipc/CallerContext.java | 13 ++++++++-----
.../main/java/org/apache/hadoop/util/ProtoUtil.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 8 ++++----
.../hadoop/hdfs/server/namenode/TestAuditLogger.java | 15 ++++++++++++---
5 files changed, 28 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cde6931/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
index 8be7e35..b197575 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallerContext.java
@@ -44,6 +44,7 @@ public class CallerContext {
* {@link org.apache.hadoop.fs.CommonConfigurationKeysPublic#HADOOP_CALLER_CONTEXT_MAX_SIZE_DEFAULT}
*/
private final String context;
+
/** The caller's signature for validation.
*
* The signature is optional. The null or empty signature will be abandoned.
@@ -58,10 +59,6 @@ public class CallerContext {
this.signature = builder.signature;
}
- public boolean isValid() {
- return context != null;
- }
-
public String getContext() {
return context;
}
@@ -71,6 +68,11 @@ public class CallerContext {
null : Arrays.copyOf(signature, signature.length);
}
+ @InterfaceAudience.Private
+ public boolean isContextValid() {
+ return context != null && !context.isEmpty();
+ }
+
@Override
public int hashCode() {
return new HashCodeBuilder().append(context).toHashCode();
@@ -92,9 +94,10 @@ public class CallerContext {
.isEquals();
}
}
+
@Override
public String toString() {
- if (!isValid()) {
+ if (!isContextValid()) {
return "";
}
String str = context;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cde6931/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProtoUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProtoUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProtoUtil.java
index 4bfcd66..1a5acba 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProtoUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProtoUtil.java
@@ -180,7 +180,7 @@ public abstract class ProtoUtil {
// Add caller context if it is not null
CallerContext callerContext = CallerContext.getCurrent();
- if (callerContext != null && callerContext.isValid()) {
+ if (callerContext != null && callerContext.isContextValid()) {
RPCCallerContextProto.Builder contextBuilder = RPCCallerContextProto
.newBuilder().setContext(callerContext.getContext());
if (callerContext.getSignature() != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cde6931/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 30cdfee..8e6634a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2201,6 +2201,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9332. Fix Precondition failures from NameNodeEditLogRoller while
saving namespace. (wang)
+ HDFS-9343. Empty caller context considered invalid. (Mingliang Liu via
+ Arpit Agarwal)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cde6931/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 8d70fbc..65b40c8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -7498,9 +7498,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
sb.append(NamenodeWebHdfsMethods.isWebHdfsInvocation() ? "webhdfs" : "rpc");
if (isCallerContextEnabled &&
callerContext != null &&
- callerContext.isValid() &&
- (callerContext.getSignature() == null ||
- callerContext.getSignature().length <= callerSignatureMaxLen)) {
+ callerContext.isContextValid()) {
sb.append("\t").append("callerContext=");
if (callerContext.getContext().length() > callerContextMaxLen) {
sb.append(callerContext.getContext().substring(0,
@@ -7508,7 +7506,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
} else {
sb.append(callerContext.getContext());
}
- if (callerContext.getSignature() != null) {
+ if (callerContext.getSignature() != null &&
+ callerContext.getSignature().length > 0 &&
+ callerContext.getSignature().length <= callerSignatureMaxLen) {
sb.append(":");
sb.append(new String(callerContext.getSignature(),
CallerContext.SIGNATURE_ENCODING));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cde6931/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
index 168b9d6..252f7af 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
@@ -243,7 +243,6 @@ public class TestAuditLogger {
CallerContext.setCurrent(context);
LOG.info("Set current caller context as {}", CallerContext.getCurrent());
fs.setTimes(p, time, time);
- System.out.println("LLLLLL" + auditlog.getOutput());
assertTrue(auditlog.getOutput().endsWith("callerContext=setTimes\n"));
auditlog.clearOutput();
@@ -270,6 +269,16 @@ public class TestAuditLogger {
"callerContext=" + longContext.substring(0, 128) + ":L\n"));
auditlog.clearOutput();
+ // empty context is ignored
+ context = new CallerContext.Builder("")
+ .setSignature("L".getBytes(CallerContext.SIGNATURE_ENCODING))
+ .build();
+ CallerContext.setCurrent(context);
+ LOG.info("Set empty caller context");
+ fs.setTimes(p, time, time);
+ assertFalse(auditlog.getOutput().contains("callerContext="));
+ auditlog.clearOutput();
+
// caller context is inherited in child thread
context = new CallerContext.Builder("setTimes")
.setSignature("L".getBytes(CallerContext.SIGNATURE_ENCODING))
@@ -333,14 +342,14 @@ public class TestAuditLogger {
assertTrue(auditlog.getOutput().endsWith("callerContext=mkdirs:L\n"));
auditlog.clearOutput();
- // caller context with too long signature is abandoned
+ // too long signature is ignored
context = new CallerContext.Builder("setTimes")
.setSignature(new byte[41])
.build();
CallerContext.setCurrent(context);
LOG.info("Set current caller context as {}", CallerContext.getCurrent());
fs.setTimes(p, time, time);
- assertFalse(auditlog.getOutput().contains("callerContext="));
+ assertTrue(auditlog.getOutput().endsWith("callerContext=setTimes\n"));
auditlog.clearOutput();
// null signature is ignored
[06/50] hadoop git commit: HADOOP-12327. Initialize output buffers
with ZERO bytes in erasure coder. Contributed by Kai Zheng.
Posted by ji...@apache.org.
HADOOP-12327. Initialize output buffers with ZERO bytes in erasure coder. Contributed by Kai Zheng.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5eca6dec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5eca6dec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5eca6dec
Branch: refs/heads/HDFS-8966
Commit: 5eca6dece67620f990f3306b6caaf09f317b38f6
Parents: 656c8f9
Author: Walter Su <wa...@apache.org>
Authored: Thu Oct 29 15:02:06 2015 +0800
Committer: Walter Su <wa...@apache.org>
Committed: Thu Oct 29 15:04:34 2015 +0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../rawcoder/AbstractRawErasureCoder.java | 47 +++++++++++++++-----
.../rawcoder/AbstractRawErasureDecoder.java | 8 ++--
.../rawcoder/AbstractRawErasureEncoder.java | 8 ++--
.../io/erasurecode/rawcoder/RSRawDecoder.java | 4 +-
.../io/erasurecode/rawcoder/XORRawDecoder.java | 1 -
.../io/erasurecode/rawcoder/XORRawEncoder.java | 1 -
.../hadoop/hdfs/DFSStripedOutputStream.java | 3 --
.../erasurecode/ErasureCodingWorker.java | 7 +--
9 files changed, 49 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5eca6dec/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 49f3d94..6b33a2c 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -603,6 +603,9 @@ Trunk (Unreleased)
HADOOP-11921. Enhance tests for erasure coders. (Kai Zheng)
+ HADOOP-12327. Initialize output buffers with ZERO bytes in erasure coder.
+ (Kai Zheng via waltersu4549)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5eca6dec/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
index 35e9492..d8a57eb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
@@ -32,6 +32,7 @@ import java.nio.ByteBuffer;
public abstract class AbstractRawErasureCoder
extends Configured implements RawErasureCoder {
+ private static byte[] emptyChunk = new byte[4096];
private final int numDataUnits;
private final int numParityUnits;
private final int numAllUnits;
@@ -42,6 +43,23 @@ public abstract class AbstractRawErasureCoder
this.numAllUnits = numDataUnits + numParityUnits;
}
+ /**
+ * Make sure to return an empty chunk buffer for the desired length.
+ * @param leastLength
+ * @return empty chunk of zero bytes
+ */
+ protected static byte[] getEmptyChunk(int leastLength) {
+ if (emptyChunk.length >= leastLength) {
+ return emptyChunk; // In most time
+ }
+
+ synchronized (AbstractRawErasureCoder.class) {
+ emptyChunk = new byte[leastLength];
+ }
+
+ return emptyChunk;
+ }
+
@Override
public int getNumDataUnits() {
return numDataUnits;
@@ -73,11 +91,9 @@ public abstract class AbstractRawErasureCoder
* @return the buffer itself, with ZERO bytes written, the position and limit
* are not changed after the call
*/
- protected ByteBuffer resetBuffer(ByteBuffer buffer) {
+ protected ByteBuffer resetBuffer(ByteBuffer buffer, int len) {
int pos = buffer.position();
- for (int i = pos; i < buffer.limit(); ++i) {
- buffer.put((byte) 0);
- }
+ buffer.put(getEmptyChunk(len), 0, len);
buffer.position(pos);
return buffer;
@@ -90,9 +106,8 @@ public abstract class AbstractRawErasureCoder
* @return the buffer itself
*/
protected byte[] resetBuffer(byte[] buffer, int offset, int len) {
- for (int i = offset; i < len; ++i) {
- buffer[i] = (byte) 0;
- }
+ byte[] empty = getEmptyChunk(len);
+ System.arraycopy(empty, 0, buffer, offset, len);
return buffer;
}
@@ -104,9 +119,10 @@ public abstract class AbstractRawErasureCoder
* @param allowNull whether to allow any element to be null or not
* @param dataLen the length of data available in the buffer to ensure with
* @param isDirectBuffer is direct buffer or not to ensure with
+ * @param isOutputs is output buffer or not
*/
- protected void ensureLengthAndType(ByteBuffer[] buffers, boolean allowNull,
- int dataLen, boolean isDirectBuffer) {
+ protected void checkParameterBuffers(ByteBuffer[] buffers, boolean
+ allowNull, int dataLen, boolean isDirectBuffer, boolean isOutputs) {
for (ByteBuffer buffer : buffers) {
if (buffer == null && !allowNull) {
throw new HadoopIllegalArgumentException(
@@ -120,18 +136,23 @@ public abstract class AbstractRawErasureCoder
throw new HadoopIllegalArgumentException(
"Invalid buffer, isDirect should be " + isDirectBuffer);
}
+ if (isOutputs) {
+ resetBuffer(buffer, dataLen);
+ }
}
}
}
/**
- * Check and ensure the buffers are of the length specified by dataLen.
+ * Check and ensure the buffers are of the length specified by dataLen. If is
+ * output buffers, ensure they will be ZEROed.
* @param buffers the buffers to check
* @param allowNull whether to allow any element to be null or not
* @param dataLen the length of data available in the buffer to ensure with
+ * @param isOutputs is output buffer or not
*/
- protected void ensureLength(byte[][] buffers,
- boolean allowNull, int dataLen) {
+ protected void checkParameterBuffers(byte[][] buffers, boolean allowNull,
+ int dataLen, boolean isOutputs) {
for (byte[] buffer : buffers) {
if (buffer == null && !allowNull) {
throw new HadoopIllegalArgumentException(
@@ -139,6 +160,8 @@ public abstract class AbstractRawErasureCoder
} else if (buffer != null && buffer.length != dataLen) {
throw new HadoopIllegalArgumentException(
"Invalid buffer not of length " + dataLen);
+ } else if (isOutputs) {
+ resetBuffer(buffer, 0, dataLen);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5eca6dec/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java
index a99730d..2cfb57c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java
@@ -48,8 +48,8 @@ public abstract class AbstractRawErasureDecoder extends AbstractRawErasureCoder
if (dataLen == 0) {
return;
}
- ensureLengthAndType(inputs, true, dataLen, usingDirectBuffer);
- ensureLengthAndType(outputs, false, dataLen, usingDirectBuffer);
+ checkParameterBuffers(inputs, true, dataLen, usingDirectBuffer, false);
+ checkParameterBuffers(outputs, false, dataLen, usingDirectBuffer, true);
if (usingDirectBuffer) {
doDecode(inputs, erasedIndexes, outputs);
@@ -106,8 +106,8 @@ public abstract class AbstractRawErasureDecoder extends AbstractRawErasureCoder
if (dataLen == 0) {
return;
}
- ensureLength(inputs, true, dataLen);
- ensureLength(outputs, false, dataLen);
+ checkParameterBuffers(inputs, true, dataLen, false);
+ checkParameterBuffers(outputs, false, dataLen, true);
int[] inputOffsets = new int[inputs.length]; // ALL ZERO
int[] outputOffsets = new int[outputs.length]; // ALL ZERO
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5eca6dec/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java
index 99c754e..13c895c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java
@@ -45,8 +45,8 @@ public abstract class AbstractRawErasureEncoder extends AbstractRawErasureCoder
if (dataLen == 0) {
return;
}
- ensureLengthAndType(inputs, false, dataLen, usingDirectBuffer);
- ensureLengthAndType(outputs, false, dataLen, usingDirectBuffer);
+ checkParameterBuffers(inputs, false, dataLen, usingDirectBuffer, false);
+ checkParameterBuffers(outputs, false, dataLen, usingDirectBuffer, true);
if (usingDirectBuffer) {
doEncode(inputs, outputs);
@@ -93,8 +93,8 @@ public abstract class AbstractRawErasureEncoder extends AbstractRawErasureCoder
if (dataLen == 0) {
return;
}
- ensureLength(inputs, false, dataLen);
- ensureLength(outputs, false, dataLen);
+ checkParameterBuffers(inputs, false, dataLen, false);
+ checkParameterBuffers(outputs, false, dataLen, true);
int[] inputOffsets = new int[inputs.length]; // ALL ZERO
int[] outputOffsets = new int[outputs.length]; // ALL ZERO
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5eca6dec/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java
index 1acaab9..87347c0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java
@@ -206,7 +206,7 @@ public class RSRawDecoder extends AbstractRawErasureDecoder {
if (erasedIndexes[i] == erasedOrNotToReadIndexes[j]) {
found = true;
adjustedDirectBufferOutputsParameter[j] =
- resetBuffer(outputs[outputIdx++]);
+ resetBuffer(outputs[outputIdx++], dataLen);
}
}
if (!found) {
@@ -220,7 +220,7 @@ public class RSRawDecoder extends AbstractRawErasureDecoder {
ByteBuffer buffer = checkGetDirectBuffer(bufferIdx, dataLen);
buffer.position(0);
buffer.limit(dataLen);
- adjustedDirectBufferOutputsParameter[i] = resetBuffer(buffer);
+ adjustedDirectBufferOutputsParameter[i] = resetBuffer(buffer, dataLen);
bufferIdx++;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5eca6dec/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java
index f11dd9f..61017dd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java
@@ -39,7 +39,6 @@ public class XORRawDecoder extends AbstractRawErasureDecoder {
protected void doDecode(ByteBuffer[] inputs, int[] erasedIndexes,
ByteBuffer[] outputs) {
ByteBuffer output = outputs[0];
- resetBuffer(output);
int erasedIdx = erasedIndexes[0];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5eca6dec/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java
index bc1ae90..646fc17 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java
@@ -37,7 +37,6 @@ public class XORRawEncoder extends AbstractRawErasureEncoder {
protected void doEncode(ByteBuffer[] inputs, ByteBuffer[] outputs) {
ByteBuffer output = outputs[0];
- resetBuffer(output);
// Get the first buffer's data.
int iIdx, oIdx;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5eca6dec/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index c595026..f5bae2a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -221,9 +221,6 @@ public class DFSStripedOutputStream extends DFSOutputStream {
private void clear() {
for (int i = 0; i< numAllBlocks; i++) {
buffers[i].clear();
- if (i >= numDataBlocks) {
- Arrays.fill(buffers[i].array(), (byte) 0);
- }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5eca6dec/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
index 64afcd0..834bd07 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
@@ -907,15 +907,10 @@ public final class ErasureCodingWorker {
for (int i = 0; i < targetBuffers.length; i++) {
if (targetBuffers[i] != null) {
- cleanBuffer(targetBuffers[i]);
+ targetBuffers[i].clear();
}
}
}
-
- private ByteBuffer cleanBuffer(ByteBuffer buffer) {
- Arrays.fill(buffer.array(), (byte) 0);
- return (ByteBuffer)buffer.clear();
- }
// send an empty packet to mark the end of the block
private void endTargetBlocks(boolean[] targetsStatus) {
[47/50] hadoop git commit: YARN-4326. Fix TestDistributedShell
timeout as AHS in MiniYarnCluster no longer binds to default port 8188. (Meng
Ding via wangda)
Posted by ji...@apache.org.
YARN-4326. Fix TestDistributedShell timeout as AHS in MiniYarnCluster no longer binds to default port 8188. (Meng Ding via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0783184f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0783184f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0783184f
Branch: refs/heads/HDFS-8966
Commit: 0783184f4b3f669f7211e42b395b62d63144100d
Parents: 957f031
Author: Wangda Tan <wa...@apache.org>
Authored: Tue Nov 3 11:18:34 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Nov 3 11:18:34 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../applications/distributedshell/TestDistributedShell.java | 9 +++++++--
2 files changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0783184f/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1040f45..d6ad672 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1039,6 +1039,9 @@ Release 2.8.0 - UNRELEASED
YARN-4127. RM fail with noAuth error if switched from failover to non-failover.
(Varun Saxena via jianhe)
+ YARN-4326. Fix TestDistributedShell timeout as AHS in MiniYarnCluster no longer
+ binds to default port 8188. (Meng Ding via wangda)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0783184f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index dcb6e72..3197875 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -92,9 +92,14 @@ public class TestDistributedShell {
yarnCluster.init(conf);
yarnCluster.start();
-
+
+ conf.set(
+ YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+ MiniYARNCluster.getHostname() + ":"
+ + yarnCluster.getApplicationHistoryServer().getPort());
+
waitForNMsToRegister();
-
+
URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml");
if (url == null) {
throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath");
[35/50] hadoop git commit: HDFS-8777. Erasure Coding: add tests for
taking snapshots on EC files. Contributed by Rakesh R.
Posted by ji...@apache.org.
HDFS-8777. Erasure Coding: add tests for taking snapshots on EC files. Contributed by Rakesh R.
Change-Id: Ia3ae582405e741ca8e90d9255ab9b95d085e5fa8
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/90e14055
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/90e14055
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/90e14055
Branch: refs/heads/HDFS-8966
Commit: 90e14055168afdb93fa8089158c03a6a694e066c
Parents: 2529464
Author: Zhe Zhang <zh...@apache.org>
Authored: Mon Nov 2 07:48:30 2015 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Mon Nov 2 07:48:30 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../TestErasureCodingPolicyWithSnapshot.java | 199 +++++++++++++++++++
2 files changed, 202 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90e14055/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0bbc60d..3b2d997 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -189,6 +189,9 @@ Trunk (Unreleased)
HDFS-9323. Randomize the DFSStripedOutputStreamWithFailure tests. (szetszwo)
+ HDFS-8777. Erasure Coding: add tests for taking snapshots on EC files.
+ (Rakesh R via zhz)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90e14055/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java
new file mode 100644
index 0000000..515763c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FsShell;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestErasureCodingPolicyWithSnapshot {
+ private MiniDFSCluster cluster;
+ private DistributedFileSystem fs;
+ private Configuration conf;
+
+ private final static short GROUP_SIZE = StripedFileTestUtil.NUM_DATA_BLOCKS
+ + StripedFileTestUtil.NUM_PARITY_BLOCKS;
+ private final static int SUCCESS = 0;
+ private final ErasureCodingPolicy sysDefaultPolicy = ErasureCodingPolicyManager
+ .getSystemDefaultPolicy();
+
+ @Before
+ public void setupCluster() throws IOException {
+ conf = new HdfsConfiguration();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE).build();
+ cluster.waitActive();
+ fs = cluster.getFileSystem();
+ }
+
+ @After
+ public void shutdownCluster() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ /**
+ * Test correctness of successive snapshot creation and deletion with erasure
+ * coding policies. Create snapshot of ecDir's parent directory.
+ */
+ @Test(timeout = 120000)
+ public void testSnapshotsOnErasureCodingDirsParentDir() throws Exception {
+ final int len = 1024;
+ final Path ecDirParent = new Path("/parent");
+ final Path ecDir = new Path(ecDirParent, "ecdir");
+ final Path ecFile = new Path(ecDir, "ecfile");
+ fs.mkdirs(ecDir);
+ fs.allowSnapshot(ecDirParent);
+ // set erasure coding policy
+ fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
+ DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED);
+ String contents = DFSTestUtil.readFile(fs, ecFile);
+ final Path snap1 = fs.createSnapshot(ecDirParent, "snap1");
+ final Path snap1ECDir = new Path(snap1, ecDir.getName());
+ assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+ fs.getErasureCodingPolicy(snap1ECDir));
+
+ // Now delete the dir which has erasure coding policy. Re-create the dir again, and
+ // take another snapshot
+ fs.delete(ecDir, true);
+ fs.mkdir(ecDir, FsPermission.getDirDefault());
+ final Path snap2 = fs.createSnapshot(ecDirParent, "snap2");
+ final Path snap2ECDir = new Path(snap2, ecDir.getName());
+ assertNull("Expected null erasure coding policy",
+ fs.getErasureCodingPolicy(snap2ECDir));
+
+ // Make dir again with system default ec policy
+ fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
+ final Path snap3 = fs.createSnapshot(ecDirParent, "snap3");
+ final Path snap3ECDir = new Path(snap3, ecDir.getName());
+ // Check that snap3's ECPolicy has the correct settings
+ ErasureCodingPolicy ezSnap3 = fs.getErasureCodingPolicy(snap3ECDir);
+ assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+ ezSnap3);
+
+ // Check that older snapshots still have the old ECPolicy settings
+ assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+ fs.getErasureCodingPolicy(snap1ECDir));
+ assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+ fs.getErasureCodingPolicy(snap2ECDir));
+
+ // Verify contents of the snapshotted file
+ final Path snapshottedECFile = new Path(snap1.toString() + "/"
+ + ecDir.getName() + "/" + ecFile.getName());
+ assertEquals("Contents of snapshotted file have changed unexpectedly",
+ contents, DFSTestUtil.readFile(fs, snapshottedECFile));
+
+ // Now delete the snapshots out of order and verify the EC policy
+ // correctness
+ fs.deleteSnapshot(ecDirParent, snap2.getName());
+ assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+ fs.getErasureCodingPolicy(snap1ECDir));
+ assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+ fs.getErasureCodingPolicy(snap3ECDir));
+ fs.deleteSnapshot(ecDirParent, snap1.getName());
+ assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+ fs.getErasureCodingPolicy(snap3ECDir));
+ }
+
+ /**
+ * Test creation of snapshot on directory has erasure coding policy.
+ */
+ @Test(timeout = 120000)
+ public void testSnapshotsOnErasureCodingDir() throws Exception {
+ final Path ecDir = new Path("/ecdir");
+ fs.mkdirs(ecDir);
+ fs.allowSnapshot(ecDir);
+
+ fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
+ final Path snap1 = fs.createSnapshot(ecDir, "snap1");
+ assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+ fs.getErasureCodingPolicy(snap1));
+ }
+
+ /**
+ * Test verify erasure coding policy is present after restarting the NameNode.
+ */
+ @Test(timeout = 120000)
+ public void testSnapshotsOnErasureCodingDirAfterNNRestart() throws Exception {
+ final Path ecDir = new Path("/ecdir");
+ fs.mkdirs(ecDir);
+ fs.allowSnapshot(ecDir);
+
+ // set erasure coding policy
+ fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
+ final Path snap1 = fs.createSnapshot(ecDir, "snap1");
+ ErasureCodingPolicy ecSnap = fs.getErasureCodingPolicy(snap1);
+ assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+ ecSnap);
+
+ // save namespace, restart namenode, and check ec policy correctness.
+ fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ fs.saveNamespace();
+ fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+ cluster.restartNameNode(true);
+
+ ErasureCodingPolicy ecSnap1 = fs.getErasureCodingPolicy(snap1);
+ assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+ ecSnap1);
+ assertEquals("Got unexpected ecSchema", ecSnap.getSchema(),
+ ecSnap1.getSchema());
+ }
+
+ /**
+ * Test copy a snapshot will not preserve its erasure coding policy info.
+ */
+ @Test(timeout = 120000)
+ public void testCopySnapshotWillNotPreserveErasureCodingPolicy()
+ throws Exception {
+ final int len = 1024;
+ final Path ecDir = new Path("/ecdir");
+ final Path ecFile = new Path(ecDir, "ecFile");
+ fs.mkdirs(ecDir);
+ fs.allowSnapshot(ecDir);
+
+ // set erasure coding policy
+ fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
+ DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED);
+ final Path snap1 = fs.createSnapshot(ecDir, "snap1");
+
+ Path snap1Copy = new Path(ecDir.toString() + "-copy");
+ final Path snap1CopyECDir = new Path("/ecdir-copy");
+ String[] argv = new String[] { "-cp", "-px", snap1.toUri().toString(),
+ snap1Copy.toUri().toString() };
+ int ret = ToolRunner.run(new FsShell(conf), argv);
+ assertEquals("cp -px is not working on a snapshot", SUCCESS, ret);
+
+ assertNull("Got unexpected erasure coding policy",
+ fs.getErasureCodingPolicy(snap1CopyECDir));
+ assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+ fs.getErasureCodingPolicy(snap1));
+ }
+}
[08/50] hadoop git commit: MAPREDUCE-6515. Update Application
priority in AM side from AM-RM heartbeat. Contributed by Sunil G
Posted by ji...@apache.org.
MAPREDUCE-6515. Update Application priority in AM side from AM-RM heartbeat. Contributed by Sunil G
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cf953b62
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cf953b62
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cf953b62
Branch: refs/heads/HDFS-8966
Commit: cf953b6258b036fc482456b4591cfb98054f48f2
Parents: e2267de
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Oct 29 18:05:01 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Oct 29 18:05:01 2015 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 ++
.../apache/hadoop/mapreduce/v2/app/job/Job.java | 2 +
.../mapreduce/v2/app/job/impl/JobImpl.java | 13 +++++-
.../v2/app/rm/RMContainerAllocator.java | 9 ++++
.../hadoop/mapreduce/v2/app/MockJobs.java | 6 +++
.../mapreduce/v2/app/TestRuntimeEstimators.java | 6 +++
.../mapreduce/v2/app/job/impl/TestJobImpl.java | 35 ++++++++++++++++
.../v2/app/rm/TestRMContainerAllocator.java | 2 +
.../apache/hadoop/mapreduce/TypeConverter.java | 34 +++++++++++----
.../mapreduce/v2/api/records/JobReport.java | 4 ++
.../v2/api/records/impl/pb/JobReportPBImpl.java | 44 ++++++++++++++++++--
.../mapreduce/v2/util/MRBuilderUtils.java | 15 ++++++-
.../src/main/proto/mr_protos.proto | 1 +
.../hadoop/mapreduce/TestTypeConverter.java | 8 +++-
.../org/apache/hadoop/mapred/JobPriority.java | 5 ++-
.../apache/hadoop/mapreduce/JobPriority.java | 4 +-
.../hadoop/mapreduce/v2/hs/CompletedJob.java | 7 ++++
.../hadoop/mapreduce/v2/hs/PartialJob.java | 7 ++++
.../v2/hs/webapp/TestHsWebServicesAcls.java | 5 +++
19 files changed, 191 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf953b62/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 66a45fb..42ff113 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -620,6 +620,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6508. TestNetworkedJob fails consistently due to delegation
token changes on RM. (Akira AJISAKA via junping_du)
+ MAPREDUCE-6515. Update Application priority in AM side from AM-RM heartbeat
+ (Sunil G via jlowe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf953b62/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java
index a40e5d2..7738810 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.yarn.api.records.Priority;
/**
@@ -100,4 +101,5 @@ public interface Job {
boolean checkAccess(UserGroupInformation callerUGI, JobACL jobOperation);
public void setQueueName(String queueName);
+ public void setJobPriority(Priority priority);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf953b62/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
index fc9a3a5..5ed0762 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
@@ -120,6 +120,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
@@ -653,6 +654,8 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
private JobState lastNonFinalState = JobState.NEW;
+ private volatile Priority jobPriority = Priority.newInstance(0);
+
public JobImpl(JobId jobId, ApplicationAttemptId applicationAttemptId,
Configuration conf, EventHandler eventHandler,
TaskAttemptListener taskAttemptListener,
@@ -878,7 +881,8 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
reporterUserName,
state, appSubmitTime, startTime, finishTime, setupProgress,
this.mapProgress, this.reduceProgress,
- cleanupProgress, jobFile, amInfos, isUber, diagsb.toString());
+ cleanupProgress, jobFile, amInfos, isUber, diagsb.toString(),
+ jobPriority);
return report;
} finally {
readLock.unlock();
@@ -2166,7 +2170,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
}
}
}
-
+
private static class InternalTerminationTransition implements
SingleArcTransition<JobImpl, JobEvent> {
JobStateInternal terminationState = null;
@@ -2219,4 +2223,9 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
public int getMaxFetchFailuresNotifications() {
return maxFetchFailuresNotifications;
}
+
+ @Override
+ public void setJobPriority(Priority priority) {
+ this.jobPriority = priority;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf953b62/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
index bf9b1f8..496886e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
@@ -797,6 +797,7 @@ public class RMContainerAllocator extends RMContainerRequestor
computeIgnoreBlacklisting();
handleUpdatedNodes(response);
+ handleJobPriorityChange(response);
for (ContainerStatus cont : finishedContainers) {
LOG.info("Received completed container " + cont.getContainerId());
@@ -921,6 +922,14 @@ public class RMContainerAllocator extends RMContainerRequestor
}
}
+ private void handleJobPriorityChange(AllocateResponse response) {
+ Priority priorityFromResponse = Priority.newInstance(response
+ .getApplicationPriority().getPriority());
+
+ // Update the job priority to Job directly.
+ getJob().setJobPriority(priorityFromResponse);
+ }
+
@Private
public Resource getResourceLimit() {
Resource headRoom = getAvailableResources();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf953b62/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
index fd9c094..ccacf1c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.util.Records;
import com.google.common.collect.Iterators;
@@ -634,6 +635,11 @@ public class MockJobs extends MockApps {
public void setQueueName(String queueName) {
// do nothing
}
+
+ @Override
+ public void setJobPriority(Priority priority) {
+ // do nothing
+ }
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf953b62/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
index 475cd1f..0b7d1b1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
@@ -69,6 +69,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -526,6 +527,11 @@ public class TestRuntimeEstimators {
public void setQueueName(String queueName) {
// do nothing
}
+
+ @Override
+ public void setJobPriority(Priority priority) {
+ // do nothing
+ }
}
/*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf953b62/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
index 2af4380..9e18920 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
@@ -72,6 +72,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.JobDiagnosticsUpdateEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobFinishEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobSetupCompletedEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobStartEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskAttemptCompletedEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskEvent;
@@ -92,6 +93,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
@@ -889,6 +891,39 @@ public class TestJobImpl {
job.getDiagnostics().toString().contains(EXCEPTIONMSG));
}
+ @Test
+ public void testJobPriorityUpdate() throws Exception {
+ Configuration conf = new Configuration();
+ AsyncDispatcher dispatcher = new AsyncDispatcher();
+ Priority submittedPriority = Priority.newInstance(5);
+
+ AppContext mockContext = mock(AppContext.class);
+ when(mockContext.hasSuccessfullyUnregistered()).thenReturn(false);
+ JobImpl job = createStubbedJob(conf, dispatcher, 2, mockContext);
+
+ JobId jobId = job.getID();
+ job.handle(new JobEvent(jobId, JobEventType.JOB_INIT));
+ assertJobState(job, JobStateInternal.INITED);
+ job.handle(new JobStartEvent(jobId));
+ assertJobState(job, JobStateInternal.SETUP);
+ // Update priority of job to 5, and it will be updated
+ job.setJobPriority(submittedPriority);
+ Assert.assertEquals(submittedPriority, job.getReport().getJobPriority());
+
+ job.handle(new JobSetupCompletedEvent(jobId));
+ assertJobState(job, JobStateInternal.RUNNING);
+
+ // Update priority of job to 8, and see whether its updated
+ Priority updatedPriority = Priority.newInstance(5);
+ job.setJobPriority(updatedPriority);
+ assertJobState(job, JobStateInternal.RUNNING);
+ Priority jobPriority = job.getReport().getJobPriority();
+ Assert.assertNotNull(jobPriority);
+
+ // Verify whether changed priority is same as what is set in Job.
+ Assert.assertEquals(updatedPriority, jobPriority);
+ }
+
private static CommitterEventHandler createCommitterEventHandler(
Dispatcher dispatcher, OutputCommitter committer) {
final SystemClock clock = new SystemClock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf953b62/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
index e4421a8..0a10434 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
@@ -2948,6 +2948,8 @@ public class TestRMContainerAllocator {
Collections.<NodeReport>emptyList(),
Resource.newInstance(512000, 1024), null, 10, null,
Collections.<NMToken>emptyList());
+ // RM will always ensure that a default priority is sent to AM
+ response.setApplicationPriority(Priority.newInstance(0));
containersToComplete.clear();
containersToAllocate.clear();
return response;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf953b62/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
index 5b8d3a7..88f61b0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
@@ -296,21 +296,41 @@ public class TypeConverter {
}
return yCntrs;
}
-
+
public static JobStatus fromYarn(JobReport jobreport, String trackingUrl) {
- JobPriority jobPriority = JobPriority.NORMAL;
+ JobPriority jobPriority = (jobreport.getJobPriority() == null)
+ ? JobPriority.DEFAULT
+ : fromYarnPriority(jobreport.getJobPriority().getPriority());
JobStatus jobStatus = new org.apache.hadoop.mapred.JobStatus(
- fromYarn(jobreport.getJobId()), jobreport.getSetupProgress(), jobreport
- .getMapProgress(), jobreport.getReduceProgress(), jobreport
- .getCleanupProgress(), fromYarn(jobreport.getJobState()),
- jobPriority, jobreport.getUser(), jobreport.getJobName(), jobreport
- .getJobFile(), trackingUrl, jobreport.isUber());
+ fromYarn(jobreport.getJobId()), jobreport.getSetupProgress(),
+ jobreport.getMapProgress(), jobreport.getReduceProgress(),
+ jobreport.getCleanupProgress(), fromYarn(jobreport.getJobState()),
+ jobPriority, jobreport.getUser(), jobreport.getJobName(),
+ jobreport.getJobFile(), trackingUrl, jobreport.isUber());
jobStatus.setStartTime(jobreport.getStartTime());
jobStatus.setFinishTime(jobreport.getFinishTime());
jobStatus.setFailureInfo(jobreport.getDiagnostics());
return jobStatus;
}
+ private static JobPriority fromYarnPriority(int priority) {
+ switch (priority) {
+ case 5 :
+ return JobPriority.VERY_HIGH;
+ case 4 :
+ return JobPriority.HIGH;
+ case 3 :
+ return JobPriority.NORMAL;
+ case 2 :
+ return JobPriority.LOW;
+ case 1 :
+ return JobPriority.VERY_LOW;
+ case 0 :
+ return JobPriority.DEFAULT;
+ }
+ return JobPriority.UNDEFINED_PRIORITY;
+ }
+
public static org.apache.hadoop.mapreduce.QueueState fromYarn(
QueueState state) {
org.apache.hadoop.mapreduce.QueueState qState =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf953b62/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/JobReport.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/JobReport.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/JobReport.java
index b2f2cc1..38dfcae 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/JobReport.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/JobReport.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.mapreduce.v2.api.records;
import java.util.List;
+import org.apache.hadoop.yarn.api.records.Priority;
+
public interface JobReport {
public abstract JobId getJobId();
public abstract JobState getJobState();
@@ -37,6 +39,7 @@ public interface JobReport {
public abstract String getJobFile();
public abstract List<AMInfo> getAMInfos();
public abstract boolean isUber();
+ public abstract Priority getJobPriority();
public abstract void setJobId(JobId jobId);
public abstract void setJobState(JobState jobState);
@@ -54,4 +57,5 @@ public interface JobReport {
public abstract void setJobFile(String jobFile);
public abstract void setAMInfos(List<AMInfo> amInfos);
public abstract void setIsUber(boolean isUber);
+ public abstract void setJobPriority(Priority priority);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf953b62/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/JobReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/JobReportPBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/JobReportPBImpl.java
index 5c90942..f4cb0a6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/JobReportPBImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/impl/pb/JobReportPBImpl.java
@@ -32,7 +32,10 @@ import org.apache.hadoop.mapreduce.v2.proto.MRProtos.JobReportProto;
import org.apache.hadoop.mapreduce.v2.proto.MRProtos.JobReportProtoOrBuilder;
import org.apache.hadoop.mapreduce.v2.proto.MRProtos.JobStateProto;
import org.apache.hadoop.mapreduce.v2.util.MRProtoUtils;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.impl.pb.PriorityPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase;
+import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto;
@@ -41,11 +44,11 @@ public class JobReportPBImpl extends ProtoBase<JobReportProto> implements
JobReportProto proto = JobReportProto.getDefaultInstance();
JobReportProto.Builder builder = null;
boolean viaProto = false;
-
+
private JobId jobId = null;
private List<AMInfo> amInfos = null;
-
-
+ private Priority jobPriority = null;
+
public JobReportPBImpl() {
builder = JobReportProto.newBuilder();
}
@@ -69,6 +72,9 @@ public class JobReportPBImpl extends ProtoBase<JobReportProto> implements
if (this.amInfos != null) {
addAMInfosToProto();
}
+ if (this.jobPriority != null) {
+ builder.setJobPriority(convertToProtoFormat(this.jobPriority));
+ }
}
private synchronized void mergeLocalToProto() {
@@ -333,6 +339,14 @@ public class JobReportPBImpl extends ProtoBase<JobReportProto> implements
return MRProtoUtils.convertFromProtoFormat(e);
}
+ private PriorityPBImpl convertFromProtoFormat(PriorityProto p) {
+ return new PriorityPBImpl(p);
+ }
+
+ private PriorityProto convertToProtoFormat(Priority t) {
+ return ((PriorityPBImpl)t).getProto();
+ }
+
@Override
public synchronized boolean isUber() {
JobReportProtoOrBuilder p = viaProto ? proto : builder;
@@ -344,4 +358,26 @@ public class JobReportPBImpl extends ProtoBase<JobReportProto> implements
maybeInitBuilder();
builder.setIsUber(isUber);
}
-}
+
+ @Override
+ public synchronized Priority getJobPriority() {
+ JobReportProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.jobPriority != null) {
+ return this.jobPriority;
+ }
+ if (!p.hasJobPriority()) {
+ return null;
+ }
+ this.jobPriority = convertFromProtoFormat(p.getJobPriority());
+ return this.jobPriority;
+ }
+
+ @Override
+ public synchronized void setJobPriority(Priority priority) {
+ maybeInitBuilder();
+ if (priority == null) {
+ builder.clearJobPriority();
+ }
+ this.jobPriority = priority;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf953b62/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRBuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRBuilderUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRBuilderUtils.java
index 95e0083..893f76a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRBuilderUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRBuilderUtils.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.util.Records;
public class MRBuilderUtils {
@@ -63,10 +64,21 @@ public class MRBuilderUtils {
}
public static JobReport newJobReport(JobId jobId, String jobName,
+ String userName, JobState state, long submitTime, long startTime,
+ long finishTime, float setupProgress, float mapProgress,
+ float reduceProgress, float cleanupProgress, String jobFile,
+ List<AMInfo> amInfos, boolean isUber, String diagnostics) {
+ return newJobReport(jobId, jobName, userName, state, submitTime, startTime,
+ finishTime, setupProgress, mapProgress, reduceProgress,
+ cleanupProgress, jobFile, amInfos, isUber, diagnostics,
+ Priority.newInstance(0));
+ }
+
+ public static JobReport newJobReport(JobId jobId, String jobName,
String userName, JobState state, long submitTime, long startTime, long finishTime,
float setupProgress, float mapProgress, float reduceProgress,
float cleanupProgress, String jobFile, List<AMInfo> amInfos,
- boolean isUber, String diagnostics) {
+ boolean isUber, String diagnostics, Priority priority) {
JobReport report = Records.newRecord(JobReport.class);
report.setJobId(jobId);
report.setJobName(jobName);
@@ -83,6 +95,7 @@ public class MRBuilderUtils {
report.setAMInfos(amInfos);
report.setIsUber(isUber);
report.setDiagnostics(diagnostics);
+ report.setJobPriority(priority);
return report;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf953b62/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_protos.proto b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_protos.proto
index b74eef6..5a4bac1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_protos.proto
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_protos.proto
@@ -146,6 +146,7 @@ message JobReportProto {
repeated AMInfoProto am_infos = 14;
optional int64 submit_time = 15;
optional bool is_uber = 16 [default = false];
+ optional hadoop.yarn.PriorityProto jobPriority = 17;
}
message AMInfoProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf953b62/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
index e36efec..60ce170 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.mapreduce;
import org.apache.hadoop.util.StringUtils;
+
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -35,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
@@ -201,10 +203,12 @@ public class TestTypeConverter {
jobReport.setJobState(state);
jobReport.setStartTime(jobStartTime);
jobReport.setFinishTime(jobFinishTime);
- jobReport.setUser("TestTypeConverter-user");
+ jobReport.setUser("TestTypeConverter-user");
+ jobReport.setJobPriority(Priority.newInstance(0));
JobStatus jobStatus = TypeConverter.fromYarn(jobReport, "dummy-jobfile");
Assert.assertEquals(jobStartTime, jobStatus.getStartTime());
Assert.assertEquals(jobFinishTime, jobStatus.getFinishTime());
Assert.assertEquals(state.toString(), jobStatus.getState().toString());
- }
+ Assert.assertEquals(JobPriority.DEFAULT, jobStatus.getPriority());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf953b62/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobPriority.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobPriority.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobPriority.java
index 376d8a4..b76d46d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobPriority.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobPriority.java
@@ -31,6 +31,7 @@ public enum JobPriority {
HIGH,
NORMAL,
LOW,
- VERY_LOW;
-
+ VERY_LOW,
+ DEFAULT,
+ UNDEFINED_PRIORITY;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf953b62/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobPriority.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobPriority.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobPriority.java
index b0d232a..7178568 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobPriority.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobPriority.java
@@ -31,5 +31,7 @@ public enum JobPriority {
HIGH,
NORMAL,
LOW,
- VERY_LOW;
+ VERY_LOW,
+ DEFAULT,
+ UNDEFINED_PRIORITY;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf953b62/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
index 0f1f391..3c12bdf 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
@@ -64,6 +64,7 @@ import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
import org.apache.hadoop.mapreduce.v2.util.MRWebAppUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.util.Records;
@@ -481,4 +482,10 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job
public void setQueueName(String queueName) {
throw new UnsupportedOperationException("Can't set job's queue name in history");
}
+
+ @Override
+ public void setJobPriority(Priority priority) {
+ throw new UnsupportedOperationException(
+ "Can't set job's priority in history");
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf953b62/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java
index 0725f46..b3b181c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -196,4 +197,10 @@ public class PartialJob implements org.apache.hadoop.mapreduce.v2.app.job.Job {
throw new UnsupportedOperationException("Can't set job's queue name in history");
}
+ @Override
+ public void setJobPriority(Priority priority) {
+ throw new UnsupportedOperationException(
+ "Can't set job's priority in history");
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf953b62/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java
index fd87b94..14961d2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAcls.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.security.GroupMappingServiceProvider;
import org.apache.hadoop.security.Groups;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.junit.Before;
import org.junit.Test;
@@ -419,5 +420,9 @@ public class TestHsWebServicesAcls {
@Override
public void setQueueName(String queueName) {
}
+
+ @Override
+ public void setJobPriority(Priority priority) {
+ }
}
}
[27/50] hadoop git commit: MAPREDUCE-6451. DistCp has incorrect
chunkFilePath for multiple jobs when strategy is dynamic. Contributed by Kuhu
Shukla.
Posted by ji...@apache.org.
MAPREDUCE-6451. DistCp has incorrect chunkFilePath for multiple jobs when strategy is dynamic. Contributed by Kuhu Shukla.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2868ca03
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2868ca03
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2868ca03
Branch: refs/heads/HDFS-8966
Commit: 2868ca0328d908056745223fb38d9a90fd2811ba
Parents: 18727c6
Author: Kihwal Lee <ki...@apache.org>
Authored: Fri Oct 30 14:56:41 2015 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Fri Oct 30 14:56:41 2015 -0500
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../tools/mapred/lib/DynamicInputChunk.java | 137 +++----------------
.../tools/mapred/lib/DynamicInputFormat.java | 31 +++--
.../tools/mapred/lib/DynamicRecordReader.java | 13 +-
.../org/apache/hadoop/tools/StubContext.java | 4 +
.../mapred/lib/TestDynamicInputFormat.java | 33 ++++-
6 files changed, 83 insertions(+), 138 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2868ca03/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 32be987..e999659 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -674,6 +674,9 @@ Release 2.7.2 - UNRELEASED
MAPREDUCE-6528. Memory leak for HistoryFileManager.getJobSummary()
(Junping Du via jlowe)
+ MAPREDUCE-6451. DistCp has incorrect chunkFilePath for multiple jobs when
+ strategy is dynamic (Kuhu Shukla via kihwal)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2868ca03/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/lib/DynamicInputChunk.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/lib/DynamicInputChunk.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/lib/DynamicInputChunk.java
index 8482e7d..9bf8e47 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/lib/DynamicInputChunk.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/lib/DynamicInputChunk.java
@@ -20,14 +20,10 @@ package org.apache.hadoop.tools.mapred.lib;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.tools.CopyListingFileStatus;
-import org.apache.hadoop.tools.DistCpConstants;
import org.apache.hadoop.tools.util.DistCpUtils;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileRecordReader;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
@@ -47,72 +43,28 @@ import java.io.IOException;
*/
class DynamicInputChunk<K, V> {
private static Log LOG = LogFactory.getLog(DynamicInputChunk.class);
-
- private static Configuration configuration;
- private static Path chunkRootPath;
- private static String chunkFilePrefix;
- private static int numChunksLeft = -1; // Un-initialized before 1st dir-scan.
- private static FileSystem fs;
-
private Path chunkFilePath;
private SequenceFileRecordReader<K, V> reader;
private SequenceFile.Writer writer;
+ private DynamicInputChunkContext chunkContext;
- private static void initializeChunkInvariants(Configuration config)
- throws IOException {
- configuration = config;
- Path listingFilePath = new Path(getListingFilePath(configuration));
- chunkRootPath = new Path(listingFilePath.getParent(), "chunkDir");
- fs = chunkRootPath.getFileSystem(configuration);
- chunkFilePrefix = listingFilePath.getName() + ".chunk.";
- }
-
- private static String getListingFilePath(Configuration configuration) {
- final String listingFileString = configuration.get(
- DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, "");
- assert !listingFileString.equals("") : "Listing file not found.";
- return listingFileString;
- }
-
- private static boolean areInvariantsInitialized() {
- return chunkRootPath != null;
- }
-
- private DynamicInputChunk(String chunkId, Configuration configuration)
+ DynamicInputChunk(String chunkId, DynamicInputChunkContext chunkContext)
throws IOException {
- if (!areInvariantsInitialized())
- initializeChunkInvariants(configuration);
-
- chunkFilePath = new Path(chunkRootPath, chunkFilePrefix + chunkId);
+ this.chunkContext = chunkContext;
+ chunkFilePath = new Path(chunkContext.getChunkRootPath(),
+ chunkContext.getChunkFilePrefix() + chunkId);
openForWrite();
}
-
private void openForWrite() throws IOException {
writer = SequenceFile.createWriter(
- chunkFilePath.getFileSystem(configuration), configuration,
+ chunkContext.getFs(), chunkContext.getConfiguration(),
chunkFilePath, Text.class, CopyListingFileStatus.class,
SequenceFile.CompressionType.NONE);
}
/**
- * Factory method to create chunk-files for writing to.
- * (For instance, when the DynamicInputFormat splits the input-file into
- * chunks.)
- * @param chunkId String to identify the chunk.
- * @param configuration Configuration, describing the location of the listing-
- * file, file-system for the map-job, etc.
- * @return A DynamicInputChunk, corresponding to a chunk-file, with the name
- * incorporating the chunk-id.
- * @throws IOException Exception on failure to create the chunk.
- */
- public static DynamicInputChunk createChunkForWrite(String chunkId,
- Configuration configuration) throws IOException {
- return new DynamicInputChunk(chunkId, configuration);
- }
-
- /**
* Method to write records into a chunk.
* @param key Key from the listing file.
* @param value Corresponding value from the listing file.
@@ -135,19 +87,19 @@ class DynamicInputChunk<K, V> {
* @throws IOException Exception on failure to reassign.
*/
public void assignTo(TaskID taskId) throws IOException {
- Path newPath = new Path(chunkRootPath, taskId.toString());
- if (!fs.rename(chunkFilePath, newPath)) {
+ Path newPath = new Path(chunkContext.getChunkRootPath(), taskId.toString());
+ if (!chunkContext.getFs().rename(chunkFilePath, newPath)) {
LOG.warn(chunkFilePath + " could not be assigned to " + taskId);
}
}
- private DynamicInputChunk(Path chunkFilePath,
- TaskAttemptContext taskAttemptContext)
- throws IOException, InterruptedException {
- if (!areInvariantsInitialized())
- initializeChunkInvariants(taskAttemptContext.getConfiguration());
+ public DynamicInputChunk(Path chunkFilePath,
+ TaskAttemptContext taskAttemptContext,
+ DynamicInputChunkContext chunkContext) throws IOException,
+ InterruptedException {
this.chunkFilePath = chunkFilePath;
+ this.chunkContext = chunkContext;
openForRead(taskAttemptContext);
}
@@ -155,45 +107,8 @@ class DynamicInputChunk<K, V> {
throws IOException, InterruptedException {
reader = new SequenceFileRecordReader<K, V>();
reader.initialize(new FileSplit(chunkFilePath, 0,
- DistCpUtils.getFileSize(chunkFilePath, configuration), null),
- taskAttemptContext);
- }
-
- /**
- * Factory method that
- * 1. acquires a chunk for the specified map-task attempt
- * 2. returns a DynamicInputChunk associated with the acquired chunk-file.
- * @param taskAttemptContext The attempt-context for the map task that's
- * trying to acquire a chunk.
- * @return The acquired dynamic-chunk. The chunk-file is renamed to the
- * attempt-id (from the attempt-context.)
- * @throws IOException Exception on failure.
- * @throws InterruptedException Exception on failure.
- */
- public static DynamicInputChunk acquire(TaskAttemptContext taskAttemptContext)
- throws IOException, InterruptedException {
- if (!areInvariantsInitialized())
- initializeChunkInvariants(taskAttemptContext.getConfiguration());
-
- String taskId
- = taskAttemptContext.getTaskAttemptID().getTaskID().toString();
- Path acquiredFilePath = new Path(chunkRootPath, taskId);
-
- if (fs.exists(acquiredFilePath)) {
- LOG.info("Acquiring pre-assigned chunk: " + acquiredFilePath);
- return new DynamicInputChunk(acquiredFilePath, taskAttemptContext);
- }
-
- for (FileStatus chunkFile : getListOfChunkFiles()) {
- if (fs.rename(chunkFile.getPath(), acquiredFilePath)) {
- LOG.info(taskId + " acquired " + chunkFile.getPath());
- return new DynamicInputChunk(acquiredFilePath, taskAttemptContext);
- }
- else
- LOG.warn(taskId + " could not acquire " + chunkFile.getPath());
- }
-
- return null;
+ DistCpUtils.getFileSize(chunkFilePath,
+ chunkContext.getConfiguration()), null), taskAttemptContext);
}
/**
@@ -204,19 +119,13 @@ class DynamicInputChunk<K, V> {
*/
public void release() throws IOException {
close();
- if (!fs.delete(chunkFilePath, false)) {
+ if (!chunkContext.getFs().delete(chunkFilePath, false)) {
LOG.error("Unable to release chunk at path: " + chunkFilePath);
- throw new IOException("Unable to release chunk at path: " + chunkFilePath);
+ throw new IOException("Unable to release chunk at path: " +
+ chunkFilePath);
}
}
- static FileStatus [] getListOfChunkFiles() throws IOException {
- Path chunkFilePattern = new Path(chunkRootPath, chunkFilePrefix + "*");
- FileStatus chunkFiles[] = fs.globStatus(chunkFilePattern);
- numChunksLeft = chunkFiles.length;
- return chunkFiles;
- }
-
/**
* Getter for the chunk-file's path, on HDFS.
* @return The qualified path to the chunk-file.
@@ -234,14 +143,4 @@ class DynamicInputChunk<K, V> {
return reader;
}
- /**
- * Getter for the number of chunk-files left in the chunk-file directory.
- * Useful to determine how many chunks (and hence, records) are left to be
- * processed.
- * @return Before the first scan of the directory, the number returned is -1.
- * Otherwise, the number of chunk-files seen from the last scan is returned.
- */
- public static int getNumChunksLeft() {
- return numChunksLeft;
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2868ca03/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/lib/DynamicInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/lib/DynamicInputFormat.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/lib/DynamicInputFormat.java
index 38269c7..fe8604a 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/lib/DynamicInputFormat.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/lib/DynamicInputFormat.java
@@ -57,7 +57,8 @@ public class DynamicInputFormat<K, V> extends InputFormat<K, V> {
= "mapred.num.splits";
private static final String CONF_LABEL_NUM_ENTRIES_PER_CHUNK
= "mapred.num.entries.per.chunk";
-
+ private DynamicInputChunkContext<K, V> chunkContext = null;
+
/**
* Implementation of InputFormat::getSplits(). This method splits up the
* copy-listing file into chunks, and assigns the first batch to different
@@ -72,6 +73,7 @@ public class DynamicInputFormat<K, V> extends InputFormat<K, V> {
throws IOException, InterruptedException {
LOG.info("DynamicInputFormat: Getting splits for job:"
+ jobContext.getJobID());
+ chunkContext = getChunkContext(jobContext.getConfiguration());
return createSplits(jobContext,
splitCopyListingIntoChunksWithShuffle(jobContext));
}
@@ -101,6 +103,13 @@ public class DynamicInputFormat<K, V> extends InputFormat<K, V> {
private static int N_CHUNKS_OPEN_AT_ONCE_DEFAULT = 16;
+ public DynamicInputChunkContext<K, V> getChunkContext(
+ Configuration configuration) throws IOException {
+ if(chunkContext == null) {
+ chunkContext = new DynamicInputChunkContext<K, V>(configuration);
+ }
+ return chunkContext;
+ }
private List<DynamicInputChunk> splitCopyListingIntoChunksWithShuffle
(JobContext context) throws IOException {
@@ -146,8 +155,8 @@ public class DynamicInputFormat<K, V> extends InputFormat<K, V> {
closeAll(openChunks);
chunksFinal.addAll(openChunks);
- openChunks = createChunks(
- configuration, chunkCount, nChunksTotal, nChunksOpenAtOnce);
+ openChunks = createChunks(chunkCount, nChunksTotal,
+ nChunksOpenAtOnce);
chunkCount += openChunks.size();
@@ -183,9 +192,9 @@ public class DynamicInputFormat<K, V> extends InputFormat<K, V> {
chunk.close();
}
- private static List<DynamicInputChunk> createChunks(Configuration config,
- int chunkCount, int nChunksTotal, int nChunksOpenAtOnce)
- throws IOException {
+ private List<DynamicInputChunk> createChunks(int chunkCount,
+ int nChunksTotal, int nChunksOpenAtOnce)
+ throws IOException {
List<DynamicInputChunk> chunks = new ArrayList<DynamicInputChunk>();
int chunkIdUpperBound
= Math.min(nChunksTotal, chunkCount + nChunksOpenAtOnce);
@@ -197,14 +206,13 @@ public class DynamicInputFormat<K, V> extends InputFormat<K, V> {
chunkIdUpperBound = nChunksTotal;
for (int i=chunkCount; i < chunkIdUpperBound; ++i)
- chunks.add(createChunk(i, config));
+ chunks.add(createChunk(i));
return chunks;
}
- private static DynamicInputChunk createChunk(int chunkId, Configuration config)
+ private DynamicInputChunk createChunk(int chunkId)
throws IOException {
- return DynamicInputChunk.createChunkForWrite(String.format("%05d", chunkId),
- config);
+ return chunkContext.createChunkForWrite(String.format("%05d", chunkId));
}
@@ -351,6 +359,7 @@ public class DynamicInputFormat<K, V> extends InputFormat<K, V> {
InputSplit inputSplit,
TaskAttemptContext taskAttemptContext)
throws IOException, InterruptedException {
- return new DynamicRecordReader<K, V>();
+ chunkContext = getChunkContext(taskAttemptContext.getConfiguration());
+ return new DynamicRecordReader<K, V>(chunkContext);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2868ca03/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/lib/DynamicRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/lib/DynamicRecordReader.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/lib/DynamicRecordReader.java
index 00b3c69..87b8f08 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/lib/DynamicRecordReader.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/lib/DynamicRecordReader.java
@@ -49,9 +49,14 @@ public class DynamicRecordReader<K, V> extends RecordReader<K, V> {
private int numRecordsProcessedByThisMap = 0;
private long timeOfLastChunkDirScan = 0;
private boolean isChunkDirAlreadyScanned = false;
+ private DynamicInputChunkContext<K, V> chunkContext;
private static long TIME_THRESHOLD_FOR_DIR_SCANS = TimeUnit.MINUTES.toMillis(5);
+ DynamicRecordReader(DynamicInputChunkContext<K, V> chunkContext) {
+ this.chunkContext = chunkContext;
+ }
+
/**
* Implementation for RecordReader::initialize(). Initializes the internal
* RecordReader to read from chunks.
@@ -69,7 +74,7 @@ public class DynamicRecordReader<K, V> extends RecordReader<K, V> {
this.taskAttemptContext = taskAttemptContext;
configuration = taskAttemptContext.getConfiguration();
taskId = taskAttemptContext.getTaskAttemptID().getTaskID();
- chunk = DynamicInputChunk.acquire(this.taskAttemptContext);
+ chunk = chunkContext.acquire(this.taskAttemptContext);
timeOfLastChunkDirScan = System.currentTimeMillis();
isChunkDirAlreadyScanned = false;
@@ -114,7 +119,7 @@ public class DynamicRecordReader<K, V> extends RecordReader<K, V> {
timeOfLastChunkDirScan = System.currentTimeMillis();
isChunkDirAlreadyScanned = false;
- chunk = DynamicInputChunk.acquire(taskAttemptContext);
+ chunk = chunkContext.acquire(taskAttemptContext);
if (chunk == null) return false;
@@ -182,12 +187,12 @@ public class DynamicRecordReader<K, V> extends RecordReader<K, V> {
|| (!isChunkDirAlreadyScanned &&
numRecordsProcessedByThisMap%numRecordsPerChunk
> numRecordsPerChunk/2)) {
- DynamicInputChunk.getListOfChunkFiles();
+ chunkContext.getListOfChunkFiles();
isChunkDirAlreadyScanned = true;
timeOfLastChunkDirScan = now;
}
- return DynamicInputChunk.getNumChunksLeft();
+ return chunkContext.getNumChunksLeft();
}
/**
* Implementation of RecordReader::close().
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2868ca03/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/StubContext.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/StubContext.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/StubContext.java
index 1a2227c..4bb6c98 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/StubContext.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/StubContext.java
@@ -64,6 +64,10 @@ public class StubContext {
return reader;
}
+ public void setReader(RecordReader<Text, CopyListingFileStatus> reader) {
+ this.reader = reader;
+ }
+
public StubInMemoryWriter getWriter() {
return writer;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2868ca03/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java
index 8cc8317..bb2dd9d 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/lib/TestDynamicInputFormat.java
@@ -40,6 +40,7 @@ import org.junit.BeforeClass;
import org.junit.Test;
import java.io.DataOutputStream;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -126,13 +127,14 @@ public class TestDynamicInputFormat {
int taskId = 0;
for (InputSplit split : splits) {
- RecordReader<Text, CopyListingFileStatus> recordReader =
- inputFormat.createRecordReader(split, null);
StubContext stubContext = new StubContext(jobContext.getConfiguration(),
- recordReader, taskId);
+ null, taskId);
final TaskAttemptContext taskAttemptContext
= stubContext.getContext();
-
+
+ RecordReader<Text, CopyListingFileStatus> recordReader =
+ inputFormat.createRecordReader(split, taskAttemptContext);
+ stubContext.setReader(recordReader);
recordReader.initialize(splits.get(0), taskAttemptContext);
float previousProgressValue = 0f;
while (recordReader.nextKeyValue()) {
@@ -182,4 +184,27 @@ public class TestDynamicInputFormat {
conf.setInt(DistCpConstants.CONF_LABEL_SPLIT_RATIO, 53);
Assert.assertEquals(53, DynamicInputFormat.getSplitRatio(3, 200, conf));
}
+
+ @Test
+ public void testDynamicInputChunkContext() throws IOException {
+ Configuration configuration = new Configuration();
+ configuration.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH,
+ "/tmp/test/file1.seq");
+ DynamicInputFormat firstInputFormat = new DynamicInputFormat();
+ DynamicInputFormat secondInputFormat = new DynamicInputFormat();
+ DynamicInputChunkContext firstContext =
+ firstInputFormat.getChunkContext(configuration);
+ DynamicInputChunkContext secondContext =
+ firstInputFormat.getChunkContext(configuration);
+ DynamicInputChunkContext thirdContext =
+ secondInputFormat.getChunkContext(configuration);
+ DynamicInputChunkContext fourthContext =
+ secondInputFormat.getChunkContext(configuration);
+ Assert.assertTrue("Chunk contexts from the same DynamicInputFormat " +
+ "object should be the same.",firstContext.equals(secondContext));
+ Assert.assertTrue("Chunk contexts from the same DynamicInputFormat " +
+ "object should be the same.",thirdContext.equals(fourthContext));
+ Assert.assertTrue("Contexts from different DynamicInputFormat " +
+ "objects should be different.",!firstContext.equals(thirdContext));
+ }
}
[10/50] hadoop git commit: HDFS-9332. Fix Precondition failures from
NameNodeEditLogRoller while saving namespace.
Posted by ji...@apache.org.
HDFS-9332. Fix Precondition failures from NameNodeEditLogRoller while saving namespace.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/888c6245
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/888c6245
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/888c6245
Branch: refs/heads/HDFS-8966
Commit: 888c6245e20ba6bdaa57d16b5c62b4a9eda2cdaf
Parents: 8def51a
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Oct 29 11:30:47 2015 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu Oct 29 11:30:47 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 4 +---
2 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/888c6245/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 36a70b8..fd1d6de 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2196,6 +2196,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9044. Give Priority to FavouredNodes , before selecting
nodes from FavouredNode's Node Group (J.Andreina via vinayakumarb)
+ HDFS-9332. Fix Precondition failures from NameNodeEditLogRoller while
+ saving namespace. (wang)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/888c6245/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 29ee087..8d70fbc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -3729,9 +3729,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
public void run() {
while (fsRunning && shouldRun) {
try {
- FSEditLog editLog = getFSImage().getEditLog();
- long numEdits =
- editLog.getLastWrittenTxId() - editLog.getCurSegmentTxId();
+ long numEdits = getTransactionsSinceLastLogRoll();
if (numEdits > rollThreshold) {
FSNamesystem.LOG.info("NameNode rolling its own edit log because"
+ " number of edits in open segment exceeds threshold of "
[38/50] hadoop git commit: HADOOP-12533. Introduce
FileNotFoundException in WASB for read and seek API. Contributed by
Dushyanth.
Posted by ji...@apache.org.
HADOOP-12533. Introduce FileNotFoundException in WASB for read and seek API. Contributed by Dushyanth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3ce0a650
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3ce0a650
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3ce0a650
Branch: refs/heads/HDFS-8966
Commit: 3ce0a6502e78240f551c29bb27a2324ce359cd70
Parents: 259bea3
Author: cnauroth <cn...@apache.org>
Authored: Mon Nov 2 09:38:37 2015 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Mon Nov 2 10:17:41 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../hadoop/fs/azure/NativeAzureFileSystem.java | 125 ++++++++++---
...estFileSystemOperationExceptionHandling.java | 131 +++++++++++++
...perationsExceptionHandlingMultiThreaded.java | 185 +++++++++++++++++++
4 files changed, 422 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ce0a650/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 5c8daad..c8d60b0 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1304,6 +1304,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12519. hadoop-azure tests should avoid creating a metrics
configuration file in the module root directory. (cnauroth)
+ HADOOP-12533. Introduce FileNotFoundException in WASB for read and seek API.
+ (Dushyanth via cnauroth)
+
OPTIMIZATIONS
HADOOP-12051. ProtobufRpcEngine.invoke() should use Exception.toString()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ce0a650/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 7c5a504..73bc6b3 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs.azure;
import java.io.DataInputStream;
+import java.io.EOFException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
@@ -49,6 +50,7 @@ import org.apache.hadoop.fs.BufferedFSInputStream;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSExceptionMessages;
import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -62,7 +64,6 @@ import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Progressable;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.codehaus.jackson.JsonNode;
@@ -74,9 +75,11 @@ import org.codehaus.jackson.map.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
import com.microsoft.azure.storage.AccessCondition;
import com.microsoft.azure.storage.OperationContext;
+import com.microsoft.azure.storage.StorageErrorCode;
import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.blob.CloudBlob;
-import com.microsoft.azure.storage.core.*;
+import com.microsoft.azure.storage.StorageErrorCodeStrings;
+import org.apache.hadoop.io.IOUtils;
/**
* A {@link FileSystem} for reading and writing files stored on <a
@@ -88,7 +91,6 @@ import com.microsoft.azure.storage.core.*;
@InterfaceStability.Stable
public class NativeAzureFileSystem extends FileSystem {
private static final int USER_WX_PERMISION = 0300;
-
/**
* A description of a folder rename operation, including the source and
* destination keys, and descriptions of the files in the source folder.
@@ -712,7 +714,7 @@ public class NativeAzureFileSystem extends FileSystem {
* @returns int An integer corresponding to the byte read.
*/
@Override
- public synchronized int read() throws IOException {
+ public synchronized int read() throws FileNotFoundException, IOException {
try {
int result = 0;
result = in.read();
@@ -726,13 +728,21 @@ public class NativeAzureFileSystem extends FileSystem {
//
return result;
} catch(IOException e) {
- if (e.getCause() instanceof StorageException) {
- StorageException storageExcp = (StorageException) e.getCause();
+
+ Throwable innerException = checkForAzureStorageException(e);
+
+ if (innerException instanceof StorageException) {
+
LOG.error("Encountered Storage Exception for read on Blob : {}"
+ " Exception details: {} Error Code : {}",
- key, e.getMessage(), storageExcp.getErrorCode());
+ key, e, ((StorageException) innerException).getErrorCode());
+
+ if (isFileNotFoundException((StorageException) innerException)) {
+ throw new FileNotFoundException(String.format("%s is not found", key));
+ }
}
- throw e;
+
+ throw e;
}
}
@@ -757,7 +767,7 @@ public class NativeAzureFileSystem extends FileSystem {
* there is no more data because the end of stream is reached.
*/
@Override
- public synchronized int read(byte[] b, int off, int len) throws IOException {
+ public synchronized int read(byte[] b, int off, int len) throws FileNotFoundException, IOException {
try {
int result = 0;
result = in.read(b, off, len);
@@ -772,29 +782,56 @@ public class NativeAzureFileSystem extends FileSystem {
// Return to the caller with the result.
return result;
} catch(IOException e) {
- if (e.getCause() instanceof StorageException) {
- StorageException storageExcp = (StorageException) e.getCause();
+
+ Throwable innerException = checkForAzureStorageException(e);
+
+ if (innerException instanceof StorageException) {
+
LOG.error("Encountered Storage Exception for read on Blob : {}"
+ " Exception details: {} Error Code : {}",
- key, e.getMessage(), storageExcp.getErrorCode());
+ key, e, ((StorageException) innerException).getErrorCode());
+
+ if (isFileNotFoundException((StorageException) innerException)) {
+ throw new FileNotFoundException(String.format("%s is not found", key));
+ }
}
- throw e;
+
+ throw e;
}
}
@Override
- public void close() throws IOException {
- in.close();
- closed = true;
+ public synchronized void close() throws IOException {
+ if (!closed) {
+ closed = true;
+ IOUtils.closeStream(in);
+ in = null;
+ }
}
@Override
- public synchronized void seek(long pos) throws IOException {
- in.close();
- in = store.retrieve(key);
- this.pos = in.skip(pos);
- LOG.debug("Seek to position {}. Bytes skipped {}", pos,
- this.pos);
+ public synchronized void seek(long pos) throws FileNotFoundException, EOFException, IOException {
+ try {
+ checkNotClosed();
+ if (pos < 0) {
+ throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK);
+ }
+ IOUtils.closeStream(in);
+ in = store.retrieve(key);
+ this.pos = in.skip(pos);
+ LOG.debug("Seek to position {}. Bytes skipped {}", pos,
+ this.pos);
+ } catch(IOException e) {
+
+ Throwable innerException = checkForAzureStorageException(e);
+
+ if (innerException instanceof StorageException
+ && isFileNotFoundException((StorageException) innerException)) {
+ throw new FileNotFoundException(String.format("%s is not found", key));
+ }
+
+ throw e;
+ }
}
@Override
@@ -806,6 +843,50 @@ public class NativeAzureFileSystem extends FileSystem {
public boolean seekToNewSource(long targetPos) throws IOException {
return false;
}
+
+ /*
+ * Helper method to recursively check if the cause of the exception is
+ * a Azure storage exception.
+ */
+ private Throwable checkForAzureStorageException(IOException e) {
+
+ Throwable innerException = e.getCause();
+
+ while (innerException != null
+ && !(innerException instanceof StorageException)) {
+ innerException = innerException.getCause();
+ }
+
+ return innerException;
+ }
+
+ /*
+ * Helper method to check if the AzureStorageException is
+ * because backing blob was not found.
+ */
+ private boolean isFileNotFoundException(StorageException e) {
+
+ String errorCode = ((StorageException) e).getErrorCode();
+ if (errorCode != null
+ && (errorCode.equals(StorageErrorCodeStrings.BLOB_NOT_FOUND)
+ || errorCode.equals(StorageErrorCodeStrings.RESOURCE_NOT_FOUND)
+ || errorCode.equals(StorageErrorCode.BLOB_NOT_FOUND.toString())
+ || errorCode.equals(StorageErrorCode.RESOURCE_NOT_FOUND.toString()))) {
+
+ return true;
+ }
+
+ return false;
+ }
+
+ /*
+ * Helper method to check if a stream is closed.
+ */
+ private void checkNotClosed() throws IOException {
+ if (closed) {
+ throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+ }
+ }
}
private class NativeAzureFsOutputStream extends OutputStream {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ce0a650/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionHandling.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionHandling.java
new file mode 100644
index 0000000..35a1f50
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationExceptionHandling.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.io.FileNotFoundException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.junit.After;
+import org.junit.Test;
+
+
+public class TestFileSystemOperationExceptionHandling extends
+ NativeAzureFileSystemBaseTest {
+
+ FSDataInputStream inputStream = null;
+ /*
+ * Helper method to create a PageBlob test storage account.
+ */
+ private AzureBlobStorageTestAccount getPageBlobTestStorageAccount()
+ throws Exception {
+
+ Configuration conf = new Configuration();
+
+ // Configure the page blob directories key so every file created is a page blob.
+ conf.set(AzureNativeFileSystemStore.KEY_PAGE_BLOB_DIRECTORIES, "/");
+
+ // Configure the atomic rename directories key so every folder will have
+ // atomic rename applied.
+ conf.set(AzureNativeFileSystemStore.KEY_ATOMIC_RENAME_DIRECTORIES, "/");
+ return AzureBlobStorageTestAccount.create(conf);
+ }
+
+
+ /*
+ * Helper method that creates a InputStream to validate exceptions
+ * for various scenarios
+ */
+ private void setupInputStreamToTest(AzureBlobStorageTestAccount testAccount)
+ throws Exception {
+
+ fs = testAccount.getFileSystem();
+
+ // Step 1: Create a file and write dummy data.
+ Path testFilePath1 = new Path("test1.dat");
+ Path testFilePath2 = new Path("test2.dat");
+ FSDataOutputStream outputStream = fs.create(testFilePath1);
+ String testString = "This is a test string";
+ outputStream.write(testString.getBytes());
+ outputStream.close();
+
+ // Step 2: Open a read stream on the file.
+ inputStream = fs.open(testFilePath1);
+
+ // Step 3: Rename the file
+ fs.rename(testFilePath1, testFilePath2);
+ }
+
+ /*
+ * Tests a basic single threaded read scenario for Page blobs.
+ */
+ @Test(expected=FileNotFoundException.class)
+ public void testSingleThreadedPageBlobReadScenario() throws Throwable {
+ AzureBlobStorageTestAccount testAccount = getPageBlobTestStorageAccount();
+ setupInputStreamToTest(testAccount);
+ byte[] readBuffer = new byte[512];
+ inputStream.read(readBuffer);
+ }
+
+ /*
+ * Tests a basic single threaded seek scenario for Page blobs.
+ */
+ @Test(expected=FileNotFoundException.class)
+ public void testSingleThreadedPageBlobSeekScenario() throws Throwable {
+ AzureBlobStorageTestAccount testAccount = getPageBlobTestStorageAccount();
+ setupInputStreamToTest(testAccount);
+ inputStream.seek(5);
+ }
+
+ /*
+ * Test a basic single thread seek scenario for Block blobs.
+ */
+ @Test(expected=FileNotFoundException.class)
+ public void testSingleThreadBlockBlobSeekScenario() throws Throwable {
+
+ AzureBlobStorageTestAccount testAccount = createTestAccount();
+ setupInputStreamToTest(testAccount);
+ inputStream.seek(5);
+ }
+
+ /*
+ * Tests a basic single threaded read scenario for Block blobs.
+ */
+ @Test(expected=FileNotFoundException.class)
+ public void testSingledThreadBlockBlobReadScenario() throws Throwable{
+ AzureBlobStorageTestAccount testAccount = createTestAccount();
+ setupInputStreamToTest(testAccount);
+ byte[] readBuffer = new byte[512];
+ inputStream.read(readBuffer);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (inputStream != null) {
+ inputStream.close();
+ }
+ }
+
+ @Override
+ protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
+ return AzureBlobStorageTestAccount.create();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ce0a650/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsExceptionHandlingMultiThreaded.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsExceptionHandlingMultiThreaded.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsExceptionHandlingMultiThreaded.java
new file mode 100644
index 0000000..0f91500
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsExceptionHandlingMultiThreaded.java
@@ -0,0 +1,185 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azure;
+
+import java.io.FileNotFoundException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.junit.After;
+import org.junit.Test;
+
+public class TestFileSystemOperationsExceptionHandlingMultiThreaded extends
+ NativeAzureFileSystemBaseTest {
+
+ FSDataInputStream inputStream = null;
+ /*
+ * Helper method to creates an input stream to test various scenarios.
+ */
+ private void getInputStreamToTest(FileSystem fs, Path testPath) throws Throwable {
+
+ FSDataOutputStream outputStream = fs.create(testPath);
+ String testString = "This is a test string";
+ outputStream.write(testString.getBytes());
+ outputStream.close();
+
+ inputStream = fs.open(testPath);
+ }
+
+ /*
+ * Test to validate correct exception is thrown for Multithreaded read
+ * scenario for block blobs
+ */
+ @Test(expected=FileNotFoundException.class)
+ public void testMultiThreadedBlockBlobReadScenario() throws Throwable {
+
+ AzureBlobStorageTestAccount testAccount = createTestAccount();
+ fs = testAccount.getFileSystem();
+ Path testFilePath1 = new Path("test1.dat");
+
+ getInputStreamToTest(fs, testFilePath1);
+ Thread renameThread = new Thread(new RenameThread(fs, testFilePath1));
+ renameThread.start();
+
+ renameThread.join();
+
+ byte[] readBuffer = new byte[512];
+ inputStream.read(readBuffer);
+ }
+
+ /*
+ * Test to validate correct exception is thrown for Multithreaded seek
+ * scenario for block blobs
+ */
+
+ @Test(expected=FileNotFoundException.class)
+ public void testMultiThreadBlockBlobSeekScenario() throws Throwable {
+
+ AzureBlobStorageTestAccount testAccount = createTestAccount();
+ fs = testAccount.getFileSystem();
+ Path testFilePath1 = new Path("test1.dat");
+
+ getInputStreamToTest(fs, testFilePath1);
+ Thread renameThread = new Thread(new RenameThread(fs, testFilePath1));
+ renameThread.start();
+
+ renameThread.join();
+
+ inputStream.seek(5);
+ }
+
+ /*
+ * Test to validate correct exception is thrown for Multithreaded read
+ * scenario for page blobs
+ */
+
+ @Test(expected=FileNotFoundException.class)
+ public void testMultiThreadedPageBlobReadScenario() throws Throwable {
+
+ AzureBlobStorageTestAccount testAccount = getPageBlobTestStorageAccount();
+ fs = testAccount.getFileSystem();
+ Path testFilePath1 = new Path("test1.dat");
+
+ getInputStreamToTest(fs, testFilePath1);
+ Thread renameThread = new Thread(new RenameThread(fs, testFilePath1));
+ renameThread.start();
+
+ renameThread.join();
+ byte[] readBuffer = new byte[512];
+ inputStream.read(readBuffer);
+ }
+
+ /*
+ * Test to validate correct exception is thrown for Multithreaded seek
+ * scenario for page blobs
+ */
+
+ @Test(expected=FileNotFoundException.class)
+ public void testMultiThreadedPageBlobSeekScenario() throws Throwable {
+
+ AzureBlobStorageTestAccount testAccount = getPageBlobTestStorageAccount();
+ fs = testAccount.getFileSystem();
+ Path testFilePath1 = new Path("test1.dat");
+
+ getInputStreamToTest(fs, testFilePath1);
+ Thread renameThread = new Thread(new RenameThread(fs, testFilePath1));
+ renameThread.start();
+
+ renameThread.join();
+ inputStream.seek(5);
+ }
+
+ @Override
+ protected AzureBlobStorageTestAccount createTestAccount() throws Exception {
+ return AzureBlobStorageTestAccount.create();
+ }
+
+ /*
+ * Helper method to create a PageBlob test storage account.
+ */
+ private AzureBlobStorageTestAccount getPageBlobTestStorageAccount()
+ throws Exception {
+
+ Configuration conf = new Configuration();
+
+ // Configure the page blob directories key so every file created is a page blob.
+ conf.set(AzureNativeFileSystemStore.KEY_PAGE_BLOB_DIRECTORIES, "/");
+
+ // Configure the atomic rename directories key so every folder will have
+ // atomic rename applied.
+ conf.set(AzureNativeFileSystemStore.KEY_ATOMIC_RENAME_DIRECTORIES, "/");
+ return AzureBlobStorageTestAccount.create(conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (inputStream != null) {
+ inputStream.close();
+ }
+ }
+}
+
+/*
+ * Helper thread that just renames the test file.
+ */
+class RenameThread implements Runnable {
+
+ private FileSystem fs;
+ private Path testPath;
+ private Path renamePath = new Path("test2.dat");
+
+ public RenameThread(FileSystem fs, Path testPath) {
+ this.fs = fs;
+ this.testPath = testPath;
+ }
+
+ @Override
+ public void run(){
+ try {
+ fs.rename(testPath, renamePath);
+ }catch (Exception e) {
+ // Swallowing the exception as the
+ // correctness of the test is controlled
+ // by the other thread
+ }
+ }
+}
[21/50] hadoop git commit: fix CHANGES.txt
Posted by ji...@apache.org.
fix CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3c0204a5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3c0204a5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3c0204a5
Branch: refs/heads/HDFS-8966
Commit: 3c0204a5866520e74917b26b6ac2061650a5bb6d
Parents: 43539b5
Author: Kihwal Lee <ki...@apache.org>
Authored: Fri Oct 30 09:40:41 2015 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Fri Oct 30 09:40:41 2015 -0500
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c0204a5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f6a22a1..c5846b3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2201,9 +2201,6 @@ Release 2.8.0 - UNRELEASED
HDFS-9332. Fix Precondition failures from NameNodeEditLogRoller while
saving namespace. (wang)
- HDFS-4937. ReplicationMonitor can infinite-loop in
- BlockPlacementPolicyDefault#chooseRandom() (kihwal)
-
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -2276,6 +2273,9 @@ Release 2.7.2 - UNRELEASED
HDFS-9317. Document fsck -blockId and -storagepolicy options in branch-2.7.
(aajisaka)
+ HDFS-4937. ReplicationMonitor can infinite-loop in
+ BlockPlacementPolicyDefault#chooseRandom() (kihwal)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
[49/50] hadoop git commit: HDFS-9362.
TestAuditLogger#testAuditLoggerWithCallContext assumes Unix line endings,
fails on Windows. Contributed by Chris Nauroth.
Posted by ji...@apache.org.
HDFS-9362. TestAuditLogger#testAuditLoggerWithCallContext assumes Unix line endings, fails on Windows. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7e282966
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7e282966
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7e282966
Branch: refs/heads/HDFS-8966
Commit: 7e2829662b4c4bf33ebaf2fa09312d0bed3d6f92
Parents: 095ac83
Author: cnauroth <cn...@apache.org>
Authored: Tue Nov 3 11:54:57 2015 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Tue Nov 3 11:54:57 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hdfs/server/namenode/TestAuditLogger.java | 24 ++++++++++++--------
2 files changed, 18 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e282966/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1729b73..fbf211f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2222,6 +2222,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9354. Fix TestBalancer#testBalancerWithZeroThreadsForMove on Windows.
(Xiaoyu Yao via cnauroth)
+ HDFS-9362. TestAuditLogger#testAuditLoggerWithCallContext assumes Unix line
+ endings, fails on Windows. (cnauroth)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e282966/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
index 252f7af..d637abc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogger.java
@@ -243,7 +243,8 @@ public class TestAuditLogger {
CallerContext.setCurrent(context);
LOG.info("Set current caller context as {}", CallerContext.getCurrent());
fs.setTimes(p, time, time);
- assertTrue(auditlog.getOutput().endsWith("callerContext=setTimes\n"));
+ assertTrue(auditlog.getOutput().endsWith(
+ String.format("callerContext=setTimes%n")));
auditlog.clearOutput();
// context with signature
@@ -254,7 +255,7 @@ public class TestAuditLogger {
LOG.info("Set current caller context as {}", CallerContext.getCurrent());
fs.setTimes(p, time, time);
assertTrue(auditlog.getOutput().endsWith(
- "callerContext=setTimes:L\n"));
+ String.format("callerContext=setTimes:L%n")));
auditlog.clearOutput();
// long context is truncated
@@ -266,7 +267,7 @@ public class TestAuditLogger {
LOG.info("Set current caller context as {}", CallerContext.getCurrent());
fs.setTimes(p, time, time);
assertTrue(auditlog.getOutput().endsWith(
- "callerContext=" + longContext.substring(0, 128) + ":L\n"));
+ String.format("callerContext=%s:L%n", longContext.substring(0, 128))));
auditlog.clearOutput();
// empty context is ignored
@@ -302,7 +303,8 @@ public class TestAuditLogger {
} catch (InterruptedException ignored) {
// Ignore
}
- assertTrue(auditlog.getOutput().endsWith("callerContext=setTimes:L\n"));
+ assertTrue(auditlog.getOutput().endsWith(
+ String.format("callerContext=setTimes:L%n")));
auditlog.clearOutput();
// caller context is overridden in child thread
@@ -330,7 +332,7 @@ public class TestAuditLogger {
// Ignore
}
assertTrue(auditlog.getOutput().endsWith(
- "callerContext=setPermission:L\n"));
+ String.format("callerContext=setPermission:L%n")));
auditlog.clearOutput();
// reuse the current context's signature
@@ -339,7 +341,8 @@ public class TestAuditLogger {
CallerContext.setCurrent(context);
LOG.info("Set current caller context as {}", CallerContext.getCurrent());
fs.mkdirs(new Path("/reuse-context-signature"));
- assertTrue(auditlog.getOutput().endsWith("callerContext=mkdirs:L\n"));
+ assertTrue(auditlog.getOutput().endsWith(
+ String.format("callerContext=mkdirs:L%n")));
auditlog.clearOutput();
// too long signature is ignored
@@ -349,7 +352,8 @@ public class TestAuditLogger {
CallerContext.setCurrent(context);
LOG.info("Set current caller context as {}", CallerContext.getCurrent());
fs.setTimes(p, time, time);
- assertTrue(auditlog.getOutput().endsWith("callerContext=setTimes\n"));
+ assertTrue(auditlog.getOutput().endsWith(
+ String.format("callerContext=setTimes%n")));
auditlog.clearOutput();
// null signature is ignored
@@ -358,7 +362,8 @@ public class TestAuditLogger {
CallerContext.setCurrent(context);
LOG.info("Set current caller context as {}", CallerContext.getCurrent());
fs.setTimes(p, time, time);
- assertTrue(auditlog.getOutput().endsWith("callerContext=setTimes\n"));
+ assertTrue(auditlog.getOutput().endsWith(
+ String.format("callerContext=setTimes%n")));
auditlog.clearOutput();
// empty signature is ignored
@@ -368,7 +373,8 @@ public class TestAuditLogger {
CallerContext.setCurrent(context);
LOG.info("Set current caller context as {}", CallerContext.getCurrent());
fs.mkdirs(new Path("/empty-signature"));
- assertTrue(auditlog.getOutput().endsWith("callerContext=mkdirs\n"));
+ assertTrue(auditlog.getOutput().endsWith(
+ String.format("callerContext=mkdirs%n")));
auditlog.clearOutput();
// invalid context is not passed to the rpc
[31/50] hadoop git commit: Revert "HDFS-4937. ReplicationMonitor can
infinite-loop in BlockPlacementPolicyDefault#chooseRandom(). Contributed by
Kihwal Lee."
Posted by ji...@apache.org.
Revert "HDFS-4937. ReplicationMonitor can infinite-loop in BlockPlacementPolicyDefault#chooseRandom(). Contributed by Kihwal Lee."
This reverts commit 43539b5ff4ac0874a8a454dc93a2a782b0e0ea8f.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7fd64167
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7fd64167
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7fd64167
Branch: refs/heads/HDFS-8966
Commit: 7fd6416759cbb202ed21b47d28c1587e04a5cdc6
Parents: 2ea4413
Author: yliu <yl...@apache.org>
Authored: Sat Oct 31 16:20:48 2015 +0800
Committer: yliu <yl...@apache.org>
Committed: Sat Oct 31 16:20:48 2015 +0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ---
.../blockmanagement/BlockPlacementPolicyDefault.java | 12 ------------
2 files changed, 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7fd64167/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 211e7fc..30cdfee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2201,9 +2201,6 @@ Release 2.8.0 - UNRELEASED
HDFS-9332. Fix Precondition failures from NameNodeEditLogRoller while
saving namespace. (wang)
- HDFS-4937. ReplicationMonitor can infinite-loop in
- BlockPlacementPolicyDefault#chooseRandom() (kihwal)
-
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7fd64167/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index f610574..d9b8d60 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -659,7 +659,6 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
int numOfAvailableNodes = clusterMap.countNumOfAvailableNodes(
scope, excludedNodes);
- int refreshCounter = numOfAvailableNodes;
StringBuilder builder = null;
if (LOG.isDebugEnabled()) {
builder = debugLoggingBuilder.get();
@@ -709,17 +708,6 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
// If no candidate storage was found on this DN then set badTarget.
badTarget = (storage == null);
}
- // Refresh the node count. If the live node count became smaller,
- // but it is not reflected in this loop, it may loop forever in case
- // the replicas/rack cannot be satisfied.
- if (--refreshCounter == 0) {
- refreshCounter = clusterMap.countNumOfAvailableNodes(scope,
- excludedNodes);
- // It has already gone through enough number of nodes.
- if (refreshCounter <= excludedNodes.size()) {
- break;
- }
- }
}
if (numOfReplicas>0) {
[07/50] hadoop git commit: YARN-2902. Killing a container that is
localizing can orphan resources in the DOWNLOADING state. Contributed by
Varun Saxena
Posted by ji...@apache.org.
YARN-2902. Killing a container that is localizing can orphan resources in the DOWNLOADING state. Contributed by Varun Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e2267de2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e2267de2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e2267de2
Branch: refs/heads/HDFS-8966
Commit: e2267de2076245bd9857f6a30e3c731df017fef8
Parents: c416999
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Oct 29 16:33:29 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Oct 29 16:34:25 2015 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../nodemanager/DefaultContainerExecutor.java | 9 +-
.../nodemanager/DockerContainerExecutor.java | 9 +-
.../localizer/LocalResourcesTrackerImpl.java | 10 +
.../localizer/ResourceLocalizationService.java | 13 +
.../impl/container-executor.c | 9 +-
.../test/test-container-executor.c | 11 +
.../TestLocalResourcesTrackerImpl.java | 6 +-
.../TestResourceLocalizationService.java | 293 ++++++++++++++++++-
9 files changed, 349 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2267de2/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8dc1fd0..4bdf6ec 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1133,6 +1133,9 @@ Release 2.7.2 - UNRELEASED
YARN-4041. Slow delegation token renewal can severely prolong RM recovery
(Sunil G via jlowe)
+ YARN-2902. Killing a container that is localizing can orphan resources in
+ the DOWNLOADING state (Varun Saxena via jlowe)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2267de2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index 5819f23..9ccde5f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -25,6 +25,7 @@ import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
import java.io.DataOutputStream;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.net.InetSocketAddress;
@@ -486,8 +487,12 @@ public class DefaultContainerExecutor extends ContainerExecutor {
for (Path baseDir : baseDirs) {
Path del = subDir == null ? baseDir : new Path(baseDir, subDir);
LOG.info("Deleting path : " + del);
- if (!lfs.delete(del, true)) {
- LOG.warn("delete returned false for path: [" + del + "]");
+ try {
+ if (!lfs.delete(del, true)) {
+ LOG.warn("delete returned false for path: [" + del + "]");
+ }
+ } catch (FileNotFoundException e) {
+ continue;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2267de2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
index 9dffff3..96f8b51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
@@ -24,6 +24,7 @@ import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
@@ -469,8 +470,12 @@ public class DockerContainerExecutor extends ContainerExecutor {
for (Path baseDir : baseDirs) {
Path del = subDir == null ? baseDir : new Path(baseDir, subDir);
LOG.info("Deleting path : " + del);
- if (!lfs.delete(del, true)) {
- LOG.warn("delete returned false for path: [" + del + "]");
+ try {
+ if (!lfs.delete(del, true)) {
+ LOG.warn("delete returned false for path: [" + del + "]");
+ }
+ } catch (FileNotFoundException e) {
+ continue;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2267de2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
index 51dbcaa..fb8f767 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
@@ -187,6 +187,16 @@ class LocalResourcesTrackerImpl implements LocalResourcesTracker {
rsrc.handle(event);
+ // Remove the resource if its downloading and its reference count has
+ // become 0 after RELEASE. This maybe because a container was killed while
+ // localizing and no other container is referring to the resource.
+ if (event.getType() == ResourceEventType.RELEASE) {
+ if (rsrc.getState() == ResourceState.DOWNLOADING &&
+ rsrc.getRefCount() <= 0) {
+ removeResource(req);
+ }
+ }
+
if (event.getType() == ResourceEventType.LOCALIZED) {
if (rsrc.getLocalPath() != null) {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2267de2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 2cc5683..5db5145 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -1165,9 +1165,22 @@ public class ResourceLocalizationService extends CompositeService
dispatcher.getEventHandler().handle(new ContainerResourceFailedEvent(
cId, null, exception.getMessage()));
}
+ List<Path> paths = new ArrayList<Path>();
for (LocalizerResourceRequestEvent event : scheduled.values()) {
+ // This means some resources were in downloading state. Schedule
+ // deletion task for localization dir and tmp dir used for downloading
+ Path locRsrcPath = event.getResource().getLocalPath();
+ if (locRsrcPath != null) {
+ Path locRsrcDirPath = locRsrcPath.getParent();
+ paths.add(locRsrcDirPath);
+ paths.add(new Path(locRsrcDirPath + "_tmp"));
+ }
event.getResource().unlock();
}
+ if (!paths.isEmpty()) {
+ delService.delete(context.getUser(),
+ null, paths.toArray(new Path[paths.size()]));
+ }
delService.delete(null, nmPrivateCTokensPath, new Path[] {});
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2267de2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index e81f0c7..102111b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -1729,8 +1729,13 @@ int delete_as_user(const char *user,
char* full_path = NULL;
struct stat sb;
if (stat(*ptr, &sb) != 0) {
- fprintf(LOGFILE, "Could not stat %s\n", *ptr);
- return -1;
+ if (errno == ENOENT) {
+ // Ignore missing dir. Continue deleting other directories.
+ continue;
+ } else {
+ fprintf(LOGFILE, "Could not stat %s - %s\n", *ptr, strerror(errno));
+ return -1;
+ }
}
if (!S_ISDIR(sb.st_mode)) {
if (!subDirEmptyStr) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2267de2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 001a37d..3db75ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -410,6 +410,17 @@ void test_delete_user() {
exit(1);
}
+ sprintf(buffer, "%s", app_dir);
+ char missing_dir[20];
+ strcpy(missing_dir, "/some/missing/dir");
+ char * dirs_with_missing[] = {missing_dir, buffer, 0};
+ ret = delete_as_user(yarn_username, "" , dirs_with_missing);
+ printf("%d" , ret);
+ if (access(buffer, R_OK) == 0) {
+ printf("FAIL: directory not deleted\n");
+ exit(1);
+ }
+
sprintf(buffer, "%s/local-1/usercache/%s", TEST_ROOT, yarn_username);
if (access(buffer, R_OK) != 0) {
printf("FAIL: directory missing before test\n");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2267de2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
index e6aeae0..4eb8675 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
@@ -141,12 +141,12 @@ public class TestLocalResourcesTrackerImpl {
tracker.handle(rel21Event);
dispatcher.await();
- verifyTrackedResourceCount(tracker, 2);
+ verifyTrackedResourceCount(tracker, 1);
// Verify resource with non zero ref count is not removed.
Assert.assertEquals(2, lr1.getRefCount());
Assert.assertFalse(tracker.remove(lr1, mockDelService));
- verifyTrackedResourceCount(tracker, 2);
+ verifyTrackedResourceCount(tracker, 1);
// Localize resource1
ResourceLocalizedEvent rle =
@@ -161,7 +161,7 @@ public class TestLocalResourcesTrackerImpl {
// Verify resources in state LOCALIZED with ref-count=0 is removed.
Assert.assertTrue(tracker.remove(lr1, mockDelService));
- verifyTrackedResourceCount(tracker, 1);
+ verifyTrackedResourceCount(tracker, 0);
} finally {
if (dispatcher != null) {
dispatcher.stop();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2267de2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
index e7a9db8..f9e4188 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
@@ -43,6 +43,8 @@ import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
+import java.io.NotSerializableException;
+import java.io.ObjectInputStream;
import java.lang.reflect.Constructor;
import java.net.InetSocketAddress;
import java.net.URI;
@@ -101,6 +103,7 @@ import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
@@ -150,9 +153,12 @@ import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.ArgumentMatcher;
import org.mockito.Mockito;
+import org.mockito.internal.matchers.VarargMatcher;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
+import com.google.common.collect.Sets;
+
public class TestResourceLocalizationService {
static final Path basedir =
@@ -481,16 +487,14 @@ public class TestResourceLocalizationService {
Assert.assertEquals("Incorrect reference count", 0, lr.getRefCount());
pubRsrcs.remove(lr.getRequest());
}
- Assert.assertEquals(0, pubRsrcs.size());
- Assert.assertEquals(2, pubRsrcCount);
+ Assert.assertEquals(2, pubRsrcs.size());
+ Assert.assertEquals(0, pubRsrcCount);
appRsrcCount = 0;
for (LocalizedResource lr : appTracker) {
appRsrcCount++;
- Assert.assertEquals("Incorrect reference count", 0, lr.getRefCount());
- Assert.assertEquals(appReq, lr.getRequest());
}
- Assert.assertEquals(1, appRsrcCount);
+ Assert.assertEquals(0, appRsrcCount);
} finally {
dispatcher.stop();
delService.stop();
@@ -1066,6 +1070,285 @@ public class TestResourceLocalizationService {
}
}
+ private static class DownloadingPathsMatcher extends ArgumentMatcher<Path[]>
+ implements VarargMatcher {
+ static final long serialVersionUID = 0;
+
+ private transient Set<Path> matchPaths;
+
+ DownloadingPathsMatcher(Set<Path> matchPaths) {
+ this.matchPaths = matchPaths;
+ }
+
+ @Override
+ public boolean matches(Object varargs) {
+ Path[] downloadingPaths = (Path[]) varargs;
+ if (matchPaths.size() != downloadingPaths.length) {
+ return false;
+ }
+ for (Path downloadingPath : downloadingPaths) {
+ if (!matchPaths.contains(downloadingPath)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private void readObject(ObjectInputStream os) throws NotSerializableException {
+ throw new NotSerializableException(this.getClass().getName());
+ }
+ }
+
+ private static class DummyExecutor extends DefaultContainerExecutor {
+ private volatile boolean stopLocalization = false;
+ @Override
+ public void startLocalizer(LocalizerStartContext ctx)
+ throws IOException, InterruptedException {
+ while (!stopLocalization) {
+ Thread.yield();
+ }
+ }
+ void setStopLocalization() {
+ stopLocalization = true;
+ }
+ }
+
+ @Test(timeout = 20000)
+ @SuppressWarnings("unchecked")
+ public void testDownloadingResourcesOnContainerKill() throws Exception {
+ List<Path> localDirs = new ArrayList<Path>();
+ String[] sDirs = new String[1];
+ localDirs.add(lfs.makeQualified(new Path(basedir, 0 + "")));
+ sDirs[0] = localDirs.get(0).toString();
+
+ conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
+ DrainDispatcher dispatcher = new DrainDispatcher();
+ dispatcher.init(conf);
+ dispatcher.start();
+ EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
+ dispatcher.register(ApplicationEventType.class, applicationBus);
+ EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
+ dispatcher.register(ContainerEventType.class, containerBus);
+
+ DummyExecutor exec = new DummyExecutor();
+ LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
+ dirsHandler.init(conf);
+
+ DeletionService delServiceReal = new DeletionService(exec);
+ DeletionService delService = spy(delServiceReal);
+ delService.init(new Configuration());
+ delService.start();
+
+ ResourceLocalizationService rawService = new ResourceLocalizationService(
+ dispatcher, exec, delService, dirsHandler, nmContext);
+ ResourceLocalizationService spyService = spy(rawService);
+ doReturn(mockServer).when(spyService).createServer();
+ doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
+ FsPermission defaultPermission =
+ FsPermission.getDirDefault().applyUMask(lfs.getUMask());
+ FsPermission nmPermission =
+ ResourceLocalizationService.NM_PRIVATE_PERM.applyUMask(lfs.getUMask());
+ final Path userDir =
+ new Path(sDirs[0].substring("file:".length()),
+ ContainerLocalizer.USERCACHE);
+ final Path fileDir =
+ new Path(sDirs[0].substring("file:".length()),
+ ContainerLocalizer.FILECACHE);
+ final Path sysDir =
+ new Path(sDirs[0].substring("file:".length()),
+ ResourceLocalizationService.NM_PRIVATE_DIR);
+ final FileStatus fs =
+ new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0,
+ defaultPermission, "", "", new Path(sDirs[0]));
+ final FileStatus nmFs =
+ new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0,
+ nmPermission, "", "", sysDir);
+
+ doAnswer(new Answer<FileStatus>() {
+ @Override
+ public FileStatus answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ if (args.length > 0) {
+ if (args[0].equals(userDir) || args[0].equals(fileDir)) {
+ return fs;
+ }
+ }
+ return nmFs;
+ }
+ }).when(spylfs).getFileStatus(isA(Path.class));
+
+ try {
+ spyService.init(conf);
+ spyService.start();
+
+ final Application app = mock(Application.class);
+ final ApplicationId appId =
+ BuilderUtils.newApplicationId(314159265358979L, 3);
+ String user = "user0";
+ when(app.getUser()).thenReturn(user);
+ when(app.getAppId()).thenReturn(appId);
+ spyService.handle(new ApplicationLocalizationEvent(
+ LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
+ ArgumentMatcher<ApplicationEvent> matchesAppInit =
+ new ArgumentMatcher<ApplicationEvent>() {
+ @Override
+ public boolean matches(Object o) {
+ ApplicationEvent evt = (ApplicationEvent) o;
+ return evt.getType() == ApplicationEventType.APPLICATION_INITED
+ && appId == evt.getApplicationID();
+ }
+ };
+ dispatcher.await();
+ verify(applicationBus).handle(argThat(matchesAppInit));
+
+ // Initialize localizer.
+ Random r = new Random();
+ long seed = r.nextLong();
+ System.out.println("SEED: " + seed);
+ r.setSeed(seed);
+ final Container c1 = getMockContainer(appId, 42, "user0");
+ final Container c2 = getMockContainer(appId, 43, "user0");
+ FSDataOutputStream out =
+ new FSDataOutputStream(new DataOutputBuffer(), null);
+ doReturn(out).when(spylfs).createInternal(isA(Path.class),
+ isA(EnumSet.class), isA(FsPermission.class), anyInt(), anyShort(),
+ anyLong(), isA(Progressable.class), isA(ChecksumOpt.class),
+ anyBoolean());
+ final LocalResource resource1 = getPrivateMockedResource(r);
+ LocalResource resource2 = null;
+ do {
+ resource2 = getPrivateMockedResource(r);
+ } while (resource2 == null || resource2.equals(resource1));
+ LocalResource resource3 = null;
+ do {
+ resource3 = getPrivateMockedResource(r);
+ } while (resource3 == null || resource3.equals(resource1)
+ || resource3.equals(resource2));
+
+ // Send localization requests for container c1 and c2.
+ final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
+ final LocalResourceRequest req2 = new LocalResourceRequest(resource2);
+ final LocalResourceRequest req3 = new LocalResourceRequest(resource3);
+ Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
+ new HashMap<LocalResourceVisibility,
+ Collection<LocalResourceRequest>>();
+ List<LocalResourceRequest> privateResourceList =
+ new ArrayList<LocalResourceRequest>();
+ privateResourceList.add(req1);
+ privateResourceList.add(req2);
+ privateResourceList.add(req3);
+ rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList);
+ spyService.handle(new ContainerLocalizationRequestEvent(c1, rsrcs));
+
+ final LocalResourceRequest req1_1 = new LocalResourceRequest(resource2);
+ Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs1 =
+ new HashMap<LocalResourceVisibility,
+ Collection<LocalResourceRequest>>();
+ List<LocalResourceRequest> privateResourceList1 =
+ new ArrayList<LocalResourceRequest>();
+ privateResourceList1.add(req1_1);
+ rsrcs1.put(LocalResourceVisibility.PRIVATE, privateResourceList1);
+ spyService.handle(new ContainerLocalizationRequestEvent(c2, rsrcs1));
+
+ dispatcher.await();
+ final String containerIdStr = c1.getContainerId().toString();
+ // Heartbeats from container localizer
+ LocalResourceStatus rsrc1success = mock(LocalResourceStatus.class);
+ LocalResourceStatus rsrc2pending = mock(LocalResourceStatus.class);
+ LocalizerStatus stat = mock(LocalizerStatus.class);
+ when(stat.getLocalizerId()).thenReturn(containerIdStr);
+ when(rsrc1success.getResource()).thenReturn(resource1);
+ when(rsrc2pending.getResource()).thenReturn(resource2);
+ when(rsrc1success.getLocalSize()).thenReturn(4344L);
+ URL locPath = getPath("/some/path");
+ when(rsrc1success.getLocalPath()).thenReturn(locPath);
+ when(rsrc1success.getStatus()).
+ thenReturn(ResourceStatusType.FETCH_SUCCESS);
+ when(rsrc2pending.getStatus()).
+ thenReturn(ResourceStatusType.FETCH_PENDING);
+
+ when(stat.getResources())
+ .thenReturn(Collections.<LocalResourceStatus>emptyList())
+ .thenReturn(Collections.singletonList(rsrc1success))
+ .thenReturn(Collections.singletonList(rsrc2pending))
+ .thenReturn(Collections.singletonList(rsrc2pending))
+ .thenReturn(Collections.<LocalResourceStatus>emptyList());
+
+ // First heartbeat which schedules first resource.
+ LocalizerHeartbeatResponse response = spyService.heartbeat(stat);
+ assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
+
+ // Second heartbeat which reports first resource as success.
+ // Second resource is scheduled.
+ response = spyService.heartbeat(stat);
+ assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
+ final String locPath1 = response.getResourceSpecs().get(0).
+ getDestinationDirectory().getFile();
+
+ // Third heartbeat which reports second resource as pending.
+ // Third resource is scheduled.
+ response = spyService.heartbeat(stat);
+ assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
+ final String locPath2 = response.getResourceSpecs().get(0).
+ getDestinationDirectory().getFile();
+
+ // Container c1 is killed which leads to cleanup
+ spyService.handle(new ContainerLocalizationCleanupEvent(c1, rsrcs));
+
+ // This heartbeat will indicate to container localizer to die as localizer
+ // runner has stopped.
+ response = spyService.heartbeat(stat);
+ assertEquals(LocalizerAction.DIE, response.getLocalizerAction());
+
+ exec.setStopLocalization();
+ dispatcher.await();
+ // verify container notification
+ ArgumentMatcher<ContainerEvent> successContainerLoc =
+ new ArgumentMatcher<ContainerEvent>() {
+ @Override
+ public boolean matches(Object o) {
+ ContainerEvent evt = (ContainerEvent) o;
+ return evt.getType() == ContainerEventType.RESOURCE_LOCALIZED
+ && c1.getContainerId() == evt.getContainerID();
+ }
+ };
+ // Only one resource gets localized for container c1.
+ verify(containerBus).handle(argThat(successContainerLoc));
+
+ Set<Path> paths =
+ Sets.newHashSet(new Path(locPath1), new Path(locPath1 + "_tmp"),
+ new Path(locPath2), new Path(locPath2 + "_tmp"));
+ // Verify if downloading resources were submitted for deletion.
+ verify(delService).delete(eq(user),
+ (Path) eq(null), argThat(new DownloadingPathsMatcher(paths)));
+
+ LocalResourcesTracker tracker = spyService.getLocalResourcesTracker(
+ LocalResourceVisibility.PRIVATE, "user0", appId);
+ // Container c1 was killed but this resource was localized before kill
+ // hence its not removed despite ref cnt being 0.
+ LocalizedResource rsrc1 = tracker.getLocalizedResource(req1);
+ assertNotNull(rsrc1);
+ assertEquals(rsrc1.getState(), ResourceState.LOCALIZED);
+ assertEquals(rsrc1.getRefCount(), 0);
+
+ // Container c1 was killed but this resource is referenced by container c2
+ // as well hence its ref cnt is 1.
+ LocalizedResource rsrc2 = tracker.getLocalizedResource(req2);
+ assertNotNull(rsrc2);
+ assertEquals(rsrc2.getState(), ResourceState.DOWNLOADING);
+ assertEquals(rsrc2.getRefCount(), 1);
+
+ // As container c1 was killed and this resource was not referenced by any
+ // other container, hence its removed.
+ LocalizedResource rsrc3 = tracker.getLocalizedResource(req3);
+ assertNull(rsrc3);
+ } finally {
+ spyService.stop();
+ dispatcher.stop();
+ delService.stop();
+ }
+ }
+
@Test
@SuppressWarnings("unchecked")
public void testPublicResourceInitializesLocalDir() throws Exception {
[19/50] hadoop git commit: YARN-4320. TestJobHistoryEventHandler
fails as AHS in MiniYarnCluster no longer binds to default port 8188.
Contributed by Varun Saxena.
Posted by ji...@apache.org.
YARN-4320. TestJobHistoryEventHandler fails as AHS in MiniYarnCluster no longer binds to default port 8188. Contributed by Varun Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ce31b227
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ce31b227
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ce31b227
Branch: refs/heads/HDFS-8966
Commit: ce31b22739512804da38cf87e0ce1059e3128da3
Parents: d21214c
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Fri Oct 30 17:51:39 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Fri Oct 30 18:00:20 2015 +0900
----------------------------------------------------------------------
.../mapreduce/jobhistory/TestJobHistoryEventHandler.java | 10 +++++++---
hadoop-yarn-project/CHANGES.txt | 3 +++
.../ApplicationHistoryServer.java | 2 +-
3 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce31b227/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
index 2b07efb..f213b32 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
@@ -489,9 +489,6 @@ public class TestJobHistoryEventHandler {
TestParams t = new TestParams(false);
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
- conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, true);
- JHEvenHandlerForTest jheh = new JHEvenHandlerForTest(t.mockAppContext, 0);
- jheh.init(conf);
MiniYARNCluster yarnCluster = null;
long currentTime = System.currentTimeMillis();
try {
@@ -499,6 +496,13 @@ public class TestJobHistoryEventHandler {
TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
+ Configuration confJHEH = new YarnConfiguration(conf);
+ confJHEH.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, true);
+ confJHEH.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+ MiniYARNCluster.getHostname() + ":" +
+ yarnCluster.getApplicationHistoryServer().getPort());
+ JHEvenHandlerForTest jheh = new JHEvenHandlerForTest(t.mockAppContext, 0);
+ jheh.init(confJHEH);
jheh.start();
TimelineStore ts = yarnCluster.getApplicationHistoryServer()
.getTimelineStore();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce31b227/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2151136..874397d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1150,6 +1150,9 @@ Release 2.7.2 - UNRELEASED
YARN-4312. TestSubmitApplicationWithRMHA fails on branch-2.7 and branch-2.6
as some of the test cases time out. (Varun Saxena via ozawa)
+ YARN-4320. TestJobHistoryEventHandler fails as AHS in MiniYarnCluster no longer
+ binds to default port 8188. (Varun Saxena via ozawa)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce31b227/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
index 111a85f..21e1c1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
@@ -143,7 +143,7 @@ public class ApplicationHistoryServer extends CompositeService {
@Private
@VisibleForTesting
- int getPort() {
+ public int getPort() {
return this.webApp.httpServer().getConnectorAddress(0).getPort();
}
[17/50] hadoop git commit: HDFS-9323. Randomize the
DFSStripedOutputStreamWithFailure tests.
Posted by ji...@apache.org.
HDFS-9323. Randomize the DFSStripedOutputStreamWithFailure tests.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f072eb5a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f072eb5a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f072eb5a
Branch: refs/heads/HDFS-8966
Commit: f072eb5a206d34d8af39d65c3ef1f39faaebfdd0
Parents: d2e01f4
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Fri Oct 30 15:38:38 2015 +0800
Committer: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Committed: Fri Oct 30 15:41:03 2015 +0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../TestDFSStripedOutputStreamWithFailure.java | 33 +++++++++-----------
...estDFSStripedOutputStreamWithFailure020.java | 22 +++++++++++++
...estDFSStripedOutputStreamWithFailure030.java | 22 +++++++++++++
...estDFSStripedOutputStreamWithFailure040.java | 22 +++++++++++++
...estDFSStripedOutputStreamWithFailure050.java | 22 +++++++++++++
...estDFSStripedOutputStreamWithFailure060.java | 22 +++++++++++++
...estDFSStripedOutputStreamWithFailure070.java | 22 +++++++++++++
...estDFSStripedOutputStreamWithFailure080.java | 22 +++++++++++++
...estDFSStripedOutputStreamWithFailure090.java | 22 +++++++++++++
...estDFSStripedOutputStreamWithFailure100.java | 22 +++++++++++++
...estDFSStripedOutputStreamWithFailure110.java | 22 +++++++++++++
...estDFSStripedOutputStreamWithFailure120.java | 22 +++++++++++++
...estDFSStripedOutputStreamWithFailure130.java | 22 +++++++++++++
...estDFSStripedOutputStreamWithFailure140.java | 22 +++++++++++++
...estDFSStripedOutputStreamWithFailure150.java | 22 +++++++++++++
...estDFSStripedOutputStreamWithFailure160.java | 22 +++++++++++++
...estDFSStripedOutputStreamWithFailure170.java | 22 +++++++++++++
...estDFSStripedOutputStreamWithFailure180.java | 22 +++++++++++++
...estDFSStripedOutputStreamWithFailure190.java | 22 +++++++++++++
...estDFSStripedOutputStreamWithFailure200.java | 22 +++++++++++++
...estDFSStripedOutputStreamWithFailure210.java | 23 ++++++++++++++
22 files changed, 458 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f072eb5a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index fd1d6de..38b9e55 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -187,6 +187,8 @@ Trunk (Unreleased)
HDFS-9261. Erasure Coding: Skip encoding the data cells if all the parity data
streamers are failed for the current block group. (Rakesh R via umamahesh)
+ HDFS-9323. Randomize the DFSStripedOutputStreamWithFailure tests. (szetszwo)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f072eb5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
index 7bd976f..b60d0f2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
@@ -131,15 +131,16 @@ public class TestDFSStripedOutputStreamWithFailure {
private static final List<Integer> LENGTHS = newLengths();
- static int getLength(int i) {
- return LENGTHS.get(i);
+ static Integer getLength(int i) {
+ return i >= 0 && i < LENGTHS.size()? LENGTHS.get(i): null;
}
+ private static final Random RANDOM = new Random();
+
private MiniDFSCluster cluster;
private DistributedFileSystem dfs;
private final Path dir = new Path("/"
+ TestDFSStripedOutputStreamWithFailure.class.getSimpleName());
- private final Random random = new Random();
private void setup(Configuration conf) throws IOException {
final int numDNs = NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS;
@@ -167,19 +168,6 @@ public class TestDFSStripedOutputStreamWithFailure {
}
@Test(timeout=240000)
- public void testDatanodeFailure56() throws Exception {
- runTest(getLength(56));
- }
-
- @Test(timeout=240000)
- public void testDatanodeFailureRandomLength() throws Exception {
- int lenIndex = random.nextInt(LENGTHS.size());
- LOG.info("run testMultipleDatanodeFailureRandomLength with length index: "
- + lenIndex);
- runTest(getLength(lenIndex));
- }
-
- @Test(timeout=240000)
public void testMultipleDatanodeFailure56() throws Exception {
runTestWithMultipleFailure(getLength(56));
}
@@ -190,7 +178,7 @@ public class TestDFSStripedOutputStreamWithFailure {
*/
//@Test(timeout=240000)
public void testMultipleDatanodeFailureRandomLength() throws Exception {
- int lenIndex = random.nextInt(LENGTHS.size());
+ int lenIndex = RANDOM.nextInt(LENGTHS.size());
LOG.info("run testMultipleDatanodeFailureRandomLength with length index: "
+ lenIndex);
runTestWithMultipleFailure(getLength(lenIndex));
@@ -484,7 +472,16 @@ public class TestDFSStripedOutputStreamWithFailure {
= new TestDFSStripedOutputStreamWithFailure();
private void run(int offset) {
final int i = offset + getBase();
- final int length = getLength(i);
+ final Integer length = getLength(i);
+ if (length == null) {
+ System.out.println("Skip test " + i + " since length=null.");
+ return;
+ }
+ if (RANDOM.nextInt(16) != 0) {
+ System.out.println("Test " + i + ", length=" + length
+ + ", is not chosen to run.");
+ return;
+ }
System.out.println("Run test " + i + ", length=" + length);
test.runTest(length);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f072eb5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure020.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure020.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure020.java
new file mode 100644
index 0000000..21feee6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure020.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.TestDFSStripedOutputStreamWithFailure.TestBase;
+
+public class TestDFSStripedOutputStreamWithFailure020 extends TestBase {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f072eb5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure030.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure030.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure030.java
new file mode 100644
index 0000000..01e6e73
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure030.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.TestDFSStripedOutputStreamWithFailure.TestBase;
+
+public class TestDFSStripedOutputStreamWithFailure030 extends TestBase {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f072eb5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure040.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure040.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure040.java
new file mode 100644
index 0000000..8519fed
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure040.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.TestDFSStripedOutputStreamWithFailure.TestBase;
+
+public class TestDFSStripedOutputStreamWithFailure040 extends TestBase {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f072eb5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure050.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure050.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure050.java
new file mode 100644
index 0000000..d750cbf
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure050.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.TestDFSStripedOutputStreamWithFailure.TestBase;
+
+public class TestDFSStripedOutputStreamWithFailure050 extends TestBase {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f072eb5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure060.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure060.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure060.java
new file mode 100644
index 0000000..cb8ce23
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure060.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.TestDFSStripedOutputStreamWithFailure.TestBase;
+
+public class TestDFSStripedOutputStreamWithFailure060 extends TestBase {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f072eb5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure070.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure070.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure070.java
new file mode 100644
index 0000000..ee72c92
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure070.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.TestDFSStripedOutputStreamWithFailure.TestBase;
+
+public class TestDFSStripedOutputStreamWithFailure070 extends TestBase {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f072eb5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure080.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure080.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure080.java
new file mode 100644
index 0000000..90ff587
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure080.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.TestDFSStripedOutputStreamWithFailure.TestBase;
+
+public class TestDFSStripedOutputStreamWithFailure080 extends TestBase {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f072eb5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure090.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure090.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure090.java
new file mode 100644
index 0000000..ce56cd2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure090.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.TestDFSStripedOutputStreamWithFailure.TestBase;
+
+public class TestDFSStripedOutputStreamWithFailure090 extends TestBase {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f072eb5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure100.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure100.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure100.java
new file mode 100644
index 0000000..d63c19d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure100.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.TestDFSStripedOutputStreamWithFailure.TestBase;
+
+public class TestDFSStripedOutputStreamWithFailure100 extends TestBase {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f072eb5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure110.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure110.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure110.java
new file mode 100644
index 0000000..a590623
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure110.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.TestDFSStripedOutputStreamWithFailure.TestBase;
+
+public class TestDFSStripedOutputStreamWithFailure110 extends TestBase {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f072eb5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure120.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure120.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure120.java
new file mode 100644
index 0000000..0e641ff
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure120.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.TestDFSStripedOutputStreamWithFailure.TestBase;
+
+public class TestDFSStripedOutputStreamWithFailure120 extends TestBase {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f072eb5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure130.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure130.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure130.java
new file mode 100644
index 0000000..e84ad1f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure130.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.TestDFSStripedOutputStreamWithFailure.TestBase;
+
+public class TestDFSStripedOutputStreamWithFailure130 extends TestBase {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f072eb5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure140.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure140.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure140.java
new file mode 100644
index 0000000..b128c85
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure140.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.TestDFSStripedOutputStreamWithFailure.TestBase;
+
+public class TestDFSStripedOutputStreamWithFailure140 extends TestBase {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f072eb5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure150.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure150.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure150.java
new file mode 100644
index 0000000..41940af
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure150.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.TestDFSStripedOutputStreamWithFailure.TestBase;
+
+public class TestDFSStripedOutputStreamWithFailure150 extends TestBase {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f072eb5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure160.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure160.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure160.java
new file mode 100644
index 0000000..8e6c39b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure160.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.TestDFSStripedOutputStreamWithFailure.TestBase;
+
+public class TestDFSStripedOutputStreamWithFailure160 extends TestBase {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f072eb5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure170.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure170.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure170.java
new file mode 100644
index 0000000..d54be46
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure170.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.TestDFSStripedOutputStreamWithFailure.TestBase;
+
+public class TestDFSStripedOutputStreamWithFailure170 extends TestBase {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f072eb5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure180.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure180.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure180.java
new file mode 100644
index 0000000..93f00b4
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure180.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.TestDFSStripedOutputStreamWithFailure.TestBase;
+
+public class TestDFSStripedOutputStreamWithFailure180 extends TestBase {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f072eb5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure190.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure190.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure190.java
new file mode 100644
index 0000000..04f5e0d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure190.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.TestDFSStripedOutputStreamWithFailure.TestBase;
+
+public class TestDFSStripedOutputStreamWithFailure190 extends TestBase {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f072eb5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure200.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure200.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure200.java
new file mode 100644
index 0000000..196d743
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure200.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.TestDFSStripedOutputStreamWithFailure.TestBase;
+
+public class TestDFSStripedOutputStreamWithFailure200 extends TestBase {}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f072eb5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure210.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure210.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure210.java
new file mode 100644
index 0000000..ef5d65b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure210.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.TestDFSStripedOutputStreamWithFailure.TestBase;
+
+public class TestDFSStripedOutputStreamWithFailure210 extends TestBase {
+}
\ No newline at end of file
[11/50] hadoop git commit: Update CHANGES.txt to reflect commit of
MR-6273 to branch-2.7
Posted by ji...@apache.org.
Update CHANGES.txt to reflect commit of MR-6273 to branch-2.7
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ecad1253
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ecad1253
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ecad1253
Branch: refs/heads/HDFS-8966
Commit: ecad1253b782070adc1d825af3ffd2e214823046
Parents: 888c624
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Oct 29 18:31:27 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Oct 29 18:32:25 2015 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecad1253/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 42ff113..af5c3f6 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -513,10 +513,6 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-5708. Duplicate String.format in YarnOutputFiles.getSpillFileForWrite.
(Konstantin Weitz via devaraj)
- MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists to
- avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED state.
- (zhihai xu via devaraj)
-
MAPREDUCE-6204. TestJobCounters should use new properties instead of
JobConf.MAPRED_TASK_JAVA_OPTS. (Sam Liu via ozawa)
@@ -671,6 +667,10 @@ Release 2.7.2 - UNRELEASED
MAPREDUCE-6518. Set SO_KEEPALIVE on shuffle connections (Chang Li via
jlowe)
+ MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists to
+ avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED state.
+ (zhihai xu via devaraj)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
[33/50] hadoop git commit: HADOOP-12047. Indicate preference not to
affect input buffers during coding in erasure coder. (Contributed by Kai
Zheng)
Posted by ji...@apache.org.
HADOOP-12047. Indicate preference not to affect input buffers during coding in erasure coder. (Contributed by Kai Zheng)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6e4f8a46
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6e4f8a46
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6e4f8a46
Branch: refs/heads/HDFS-8966
Commit: 6e4f8a46c5ce983493cb0ac2234fceafdb3a5613
Parents: 3cde693
Author: Walter Su <wa...@apache.org>
Authored: Mon Nov 2 10:40:14 2015 +0800
Committer: Walter Su <wa...@apache.org>
Committed: Mon Nov 2 10:40:14 2015 +0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../rawcoder/AbstractRawErasureCoder.java | 60 ++++++++++++++++++--
.../io/erasurecode/rawcoder/CoderOption.java | 43 ++++++++++++++
.../io/erasurecode/rawcoder/RSRawEncoder.java | 48 +++++++++++++---
.../erasurecode/rawcoder/RawErasureCoder.java | 23 +++++---
.../erasurecode/rawcoder/RawErasureDecoder.java | 32 ++++++-----
.../erasurecode/rawcoder/RawErasureEncoder.java | 35 ++++++------
.../hadoop/io/erasurecode/TestCoderBase.java | 34 +++++++++--
.../erasurecode/rawcoder/TestRawCoderBase.java | 37 ++++++++++--
.../erasurecode/rawcoder/TestXORRawCoder.java | 1 +
.../hadoop/hdfs/DFSStripedInputStream.java | 1 -
11 files changed, 253 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e4f8a46/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2560fe5..5c8daad 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -608,6 +608,9 @@ Trunk (Unreleased)
HADOOP-12327. Initialize output buffers with ZERO bytes in erasure coder.
(Kai Zheng via waltersu4549)
+ HADOOP-12047. Indicate preference not to affect input buffers during
+ coding in erasure coder. (Kai Zheng via waltersu4549)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e4f8a46/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
index d8a57eb..b195216 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
@@ -22,6 +22,8 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configured;
import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
/**
* A common class of basic facilities to be shared by encoder and decoder
@@ -36,11 +38,39 @@ public abstract class AbstractRawErasureCoder
private final int numDataUnits;
private final int numParityUnits;
private final int numAllUnits;
+ private final Map<CoderOption, Object> coderOptions;
public AbstractRawErasureCoder(int numDataUnits, int numParityUnits) {
this.numDataUnits = numDataUnits;
this.numParityUnits = numParityUnits;
this.numAllUnits = numDataUnits + numParityUnits;
+ this.coderOptions = new HashMap<>(3);
+
+ coderOptions.put(CoderOption.PREFER_DIRECT_BUFFER, preferDirectBuffer());
+ coderOptions.put(CoderOption.ALLOW_CHANGE_INPUTS, false);
+ coderOptions.put(CoderOption.ALLOW_VERBOSE_DUMP, false);
+ }
+
+ @Override
+ public Object getCoderOption(CoderOption option) {
+ if (option == null) {
+ throw new HadoopIllegalArgumentException("Invalid option");
+ }
+ return coderOptions.get(option);
+ }
+
+ @Override
+ public void setCoderOption(CoderOption option, Object value) {
+ if (option == null || value == null) {
+ throw new HadoopIllegalArgumentException(
+ "Invalid option or option value");
+ }
+ if (option.isReadOnly()) {
+ throw new HadoopIllegalArgumentException(
+ "The option is read-only: " + option.name());
+ }
+
+ coderOptions.put(option, value);
}
/**
@@ -75,13 +105,35 @@ public abstract class AbstractRawErasureCoder
}
@Override
- public boolean preferDirectBuffer() {
+ public void release() {
+ // Nothing to do by default
+ }
+
+ /**
+ * Tell if direct buffer is preferred or not. It's for callers to
+ * decide how to allocate coding chunk buffers, using DirectByteBuffer or
+ * bytes array. It will return false by default.
+ * @return true if native buffer is preferred for performance consideration,
+ * otherwise false.
+ */
+ protected boolean preferDirectBuffer() {
return false;
}
- @Override
- public void release() {
- // Nothing to do by default
+ protected boolean isAllowingChangeInputs() {
+ Object value = getCoderOption(CoderOption.ALLOW_CHANGE_INPUTS);
+ if (value != null && value instanceof Boolean) {
+ return (boolean) value;
+ }
+ return false;
+ }
+
+ protected boolean isAllowingVerboseDump() {
+ Object value = getCoderOption(CoderOption.ALLOW_VERBOSE_DUMP);
+ if (value != null && value instanceof Boolean) {
+ return (boolean) value;
+ }
+ return false;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e4f8a46/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/CoderOption.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/CoderOption.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/CoderOption.java
new file mode 100644
index 0000000..e4d97ca
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/CoderOption.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+/**
+ * Supported erasure coder options.
+ */
+public enum CoderOption {
+ /* If direct buffer is preferred, for perf consideration */
+ PREFER_DIRECT_BUFFER(true), // READ-ONLY
+ /**
+ * Allow changing input buffer content (not positions).
+ * Maybe better perf if allowed
+ */
+ ALLOW_CHANGE_INPUTS(false), // READ-WRITE
+ /* Allow dump verbose debug info or not */
+ ALLOW_VERBOSE_DUMP(false); // READ-WRITE
+
+ private boolean isReadOnly = false;
+
+ CoderOption(boolean isReadOnly) {
+ this.isReadOnly = isReadOnly;
+ }
+
+ public boolean isReadOnly() {
+ return isReadOnly;
+ }
+};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e4f8a46/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawEncoder.java
index 813a312..0f25b2a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawEncoder.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.erasurecode.rawcoder.util.RSUtil;
import java.nio.ByteBuffer;
+import java.util.Arrays;
/**
* A raw erasure encoder in RS code scheme in pure Java in case native one
@@ -54,8 +55,26 @@ public class RSRawEncoder extends AbstractRawErasureEncoder {
protected void doEncode(ByteBuffer[] inputs, ByteBuffer[] outputs) {
// parity units + data units
ByteBuffer[] all = new ByteBuffer[outputs.length + inputs.length];
- System.arraycopy(outputs, 0, all, 0, outputs.length);
- System.arraycopy(inputs, 0, all, outputs.length, inputs.length);
+
+ if (isAllowingChangeInputs()) {
+ System.arraycopy(outputs, 0, all, 0, outputs.length);
+ System.arraycopy(inputs, 0, all, outputs.length, inputs.length);
+ } else {
+ System.arraycopy(outputs, 0, all, 0, outputs.length);
+
+ /**
+ * Note when this coder would be really (rarely) used in a production
+ * system, this can be optimized to cache and reuse the new allocated
+ * buffers avoiding reallocating.
+ */
+ ByteBuffer tmp;
+ for (int i = 0; i < inputs.length; i++) {
+ tmp = ByteBuffer.allocate(inputs[i].remaining());
+ tmp.put(inputs[i]);
+ tmp.flip();
+ all[outputs.length + i] = tmp;
+ }
+ }
// Compute the remainder
RSUtil.GF.remainder(all, generatingPolynomial);
@@ -67,15 +86,26 @@ public class RSRawEncoder extends AbstractRawErasureEncoder {
int[] outputOffsets) {
// parity units + data units
byte[][] all = new byte[outputs.length + inputs.length][];
- System.arraycopy(outputs, 0, all, 0, outputs.length);
- System.arraycopy(inputs, 0, all, outputs.length, inputs.length);
+ int[] allOffsets = new int[outputOffsets.length + inputOffsets.length];
- int[] offsets = new int[inputOffsets.length + outputOffsets.length];
- System.arraycopy(outputOffsets, 0, offsets, 0, outputOffsets.length);
- System.arraycopy(inputOffsets, 0, offsets,
- outputOffsets.length, inputOffsets.length);
+ if (isAllowingChangeInputs()) {
+ System.arraycopy(outputs, 0, all, 0, outputs.length);
+ System.arraycopy(inputs, 0, all, outputs.length, inputs.length);
+
+ System.arraycopy(outputOffsets, 0, allOffsets, 0, outputOffsets.length);
+ System.arraycopy(inputOffsets, 0, allOffsets,
+ outputOffsets.length, inputOffsets.length);
+ } else {
+ System.arraycopy(outputs, 0, all, 0, outputs.length);
+ System.arraycopy(outputOffsets, 0, allOffsets, 0, outputOffsets.length);
+
+ for (int i = 0; i < inputs.length; i++) {
+ all[outputs.length + i] = Arrays.copyOfRange(inputs[i],
+ inputOffsets[i], inputOffsets[i] + dataLen);
+ }
+ }
// Compute the remainder
- RSUtil.GF.remainder(all, offsets, dataLen, generatingPolynomial);
+ RSUtil.GF.remainder(all, allOffsets, dataLen, generatingPolynomial);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e4f8a46/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java
index 082c9c3..dbe2da9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java
@@ -38,6 +38,20 @@ import org.apache.hadoop.conf.Configurable;
public interface RawErasureCoder extends Configurable {
/**
+ * Get a coder option value.
+ * @param option
+ * @return
+ */
+ public Object getCoderOption(CoderOption option);
+
+ /**
+ * Set a coder option value.
+ * @param option
+ * @param value
+ */
+ public void setCoderOption(CoderOption option, Object value);
+
+ /**
* The number of data input units for the coding. A unit can be a byte,
* chunk or buffer or even a block.
* @return count of data input units
@@ -52,15 +66,6 @@ public interface RawErasureCoder extends Configurable {
public int getNumParityUnits();
/**
- * Tell if direct buffer is preferred or not. It's for callers to
- * decide how to allocate coding chunk buffers, using DirectByteBuffer or
- * bytes array. It will return false by default.
- * @return true if native buffer is preferred for performance consideration,
- * otherwise false.
- */
- public boolean preferDirectBuffer();
-
- /**
* Should be called when release this coder. Good chance to release encoding
* or decoding buffers
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e4f8a46/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java
index 7a07b71..ab322fa 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java
@@ -54,24 +54,27 @@ public interface RawErasureDecoder extends RawErasureCoder {
* Note, for both inputs and outputs, no mixing of on-heap buffers and direct
* buffers are allowed.
*
- * @param inputs inputs to read data from, contents may change after the call
+ * If the coder option ALLOW_CHANGE_INPUTS is set true (false by default), the
+ * content of input buffers may change after the call, subject to concrete
+ * implementation. Anyway the positions of input buffers will move forward.
+ *
+ * @param inputs input buffers to read data from
* @param erasedIndexes indexes of erased units in the inputs array
- * @param outputs outputs to write into for data generated according to
- * erasedIndexes, ready for reading the result data from after
- * the call
+ * @param outputs output buffers to put decoded data into according to
+ * erasedIndexes, ready for read after the call
*/
- public void decode(ByteBuffer[] inputs, int[] erasedIndexes,
+ void decode(ByteBuffer[] inputs, int[] erasedIndexes,
ByteBuffer[] outputs);
/**
* Decode with inputs and erasedIndexes, generates outputs. More see above.
- * @param inputs inputs to read data from, contents may change after the call
+ *
+ * @param inputs input buffers to read data from
* @param erasedIndexes indexes of erased units in the inputs array
- * @param outputs outputs to write into for data generated according to
- * erasedIndexes, ready for reading the result data from after
- * the call
+ * @param outputs output buffers to put decoded data into according to
+ * erasedIndexes, ready for read after the call
*/
- public void decode(byte[][] inputs, int[] erasedIndexes, byte[][] outputs);
+ void decode(byte[][] inputs, int[] erasedIndexes, byte[][] outputs);
/**
* Decode with inputs and erasedIndexes, generates outputs. More see above.
@@ -79,12 +82,11 @@ public interface RawErasureDecoder extends RawErasureCoder {
* Note, for both input and output ECChunks, no mixing of on-heap buffers and
* direct buffers are allowed.
*
- * @param inputs inputs to read data from, contents may change after the call
+ * @param inputs input buffers to read data from
* @param erasedIndexes indexes of erased units in the inputs array
- * @param outputs outputs to write into for data generated according to
- * erasedIndexes, ready for reading the result data from after
- * the call
+ * @param outputs output buffers to put decoded data into according to
+ * erasedIndexes, ready for read after the call
*/
- public void decode(ECChunk[] inputs, int[] erasedIndexes, ECChunk[] outputs);
+ void decode(ECChunk[] inputs, int[] erasedIndexes, ECChunk[] outputs);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e4f8a46/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureEncoder.java
index d451eb9..91ef714 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureEncoder.java
@@ -38,29 +38,32 @@ public interface RawErasureEncoder extends RawErasureCoder {
* Note, for both inputs and outputs, no mixing of on-heap buffers and direct
* buffers are allowed.
*
- * @param inputs inputs to read data from, contents may change after the call
- * @param outputs
+ * If the coder option ALLOW_CHANGE_INPUTS is set true (false by default), the
+ * content of input buffers may change after the call, subject to concrete
+ * implementation. Anyway the positions of input buffers will move forward.
+ *
+ * @param inputs input buffers to read data from
+ * @param outputs output buffers to put the encoded data into, read to read
+ * after the call
*/
- public void encode(ByteBuffer[] inputs, ByteBuffer[] outputs);
+ void encode(ByteBuffer[] inputs, ByteBuffer[] outputs);
/**
- * Encode with inputs and generates outputs
- * @param inputs inputs to read data from, contents may change after the call
- * @param outputs outputs to write into for data generated, ready for reading
- * the result data from after the call
+ * Encode with inputs and generates outputs. More see above.
+ *
+ * @param inputs input buffers to read data from
+ * @param outputs output buffers to put the encoded data into, read to read
+ * after the call
*/
- public void encode(byte[][] inputs, byte[][] outputs);
+ void encode(byte[][] inputs, byte[][] outputs);
/**
- * Encode with inputs and generates outputs.
- *
- * Note, for both input and output ECChunks, no mixing of on-heap buffers and
- * direct buffers are allowed.
+ * Encode with inputs and generates outputs. More see above.
*
- * @param inputs inputs to read data from, contents may change after the call
- * @param outputs outputs to write into for data generated, ready for reading
- * the result data from after the call
+ * @param inputs input buffers to read data from
+ * @param outputs output buffers to put the encoded data into, read to read
+ * after the call
*/
- public void encode(ECChunk[] inputs, ECChunk[] outputs);
+ void encode(ECChunk[] inputs, ECChunk[] outputs);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e4f8a46/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
index 0d2b5ad..62a0b9c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
@@ -64,6 +64,8 @@ public abstract class TestCoderBase {
private static int FIXED_DATA_GENERATOR = 0;
protected byte[][] fixedData;
+ protected boolean allowChangeInputs;
+
protected int getChunkSize() {
return chunkSize;
}
@@ -253,6 +255,22 @@ public abstract class TestCoderBase {
}
}
+ protected void markChunks(ECChunk[] chunks) {
+ for (int i = 0; i < chunks.length; i++) {
+ if (chunks[i] != null) {
+ chunks[i].getBuffer().mark();
+ }
+ }
+ }
+
+ protected void restoreChunksFromMark(ECChunk[] chunks) {
+ for (int i = 0; i < chunks.length; i++) {
+ if (chunks[i] != null) {
+ chunks[i].getBuffer().reset();
+ }
+ }
+ }
+
/**
* Clone chunks along with copying the associated data. It respects how the
* chunk buffer is allocated, direct or non-direct. It avoids affecting the
@@ -277,6 +295,10 @@ public abstract class TestCoderBase {
* @return a new chunk
*/
protected ECChunk cloneChunkWithData(ECChunk chunk) {
+ if (chunk == null) {
+ return null;
+ }
+
ByteBuffer srcBuffer = chunk.getBuffer();
byte[] bytesArr = new byte[srcBuffer.remaining()];
@@ -453,14 +475,16 @@ public abstract class TestCoderBase {
byte[][] bytesArr = new byte[chunks.length][];
for (int i = 0; i < chunks.length; i++) {
- bytesArr[i] = chunks[i].toBytesArray();
+ if (chunks[i] != null) {
+ bytesArr[i] = chunks[i].toBytesArray();
+ }
}
return bytesArr;
}
/**
- * Dump all the settings used in the test case if allowDump is enabled.
+ * Dump all the settings used in the test case if isAllowingVerboseDump is enabled.
*/
protected void dumpSetting() {
if (allowDump) {
@@ -473,14 +497,16 @@ public abstract class TestCoderBase {
append(Arrays.toString(erasedDataIndexes));
sb.append(" erasedParityIndexes=").
append(Arrays.toString(erasedParityIndexes));
- sb.append(" usingDirectBuffer=").append(usingDirectBuffer).append("\n");
+ sb.append(" usingDirectBuffer=").append(usingDirectBuffer);
+ sb.append(" isAllowingChangeInputs=").append(allowChangeInputs);
+ sb.append("\n");
System.out.println(sb.toString());
}
}
/**
- * Dump chunks prefixed with a header if allowDump is enabled.
+ * Dump chunks prefixed with a header if isAllowingVerboseDump is enabled.
* @param header
* @param chunks
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e4f8a46/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
index 2b7a3c4..ec93d44 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
@@ -68,9 +68,9 @@ public abstract class TestRawCoderBase extends TestCoderBase {
* The following runs will use 3 different chunkSize for inputs and outputs,
* to verify the same encoder/decoder can process variable width of data.
*/
- performTestCoding(baseChunkSize, true, false, false);
- performTestCoding(baseChunkSize - 17, false, false, false);
- performTestCoding(baseChunkSize + 16, true, false, false);
+ performTestCoding(baseChunkSize, true, false, false, false);
+ performTestCoding(baseChunkSize - 17, false, false, false, true);
+ performTestCoding(baseChunkSize + 16, true, false, false, false);
}
/**
@@ -82,7 +82,7 @@ public abstract class TestRawCoderBase extends TestCoderBase {
prepareCoders();
try {
- performTestCoding(baseChunkSize, false, true, false);
+ performTestCoding(baseChunkSize, false, true, false, true);
Assert.fail("Encoding test with bad input should fail");
} catch (Exception e) {
// Expected
@@ -98,7 +98,7 @@ public abstract class TestRawCoderBase extends TestCoderBase {
prepareCoders();
try {
- performTestCoding(baseChunkSize, false, false, true);
+ performTestCoding(baseChunkSize, false, false, true, true);
Assert.fail("Decoding test with bad output should fail");
} catch (Exception e) {
// Expected
@@ -123,9 +123,11 @@ public abstract class TestRawCoderBase extends TestCoderBase {
}
private void performTestCoding(int chunkSize, boolean usingSlicedBuffer,
- boolean useBadInput, boolean useBadOutput) {
+ boolean useBadInput, boolean useBadOutput,
+ boolean allowChangeInputs) {
setChunkSize(chunkSize);
prepareBufferAllocator(usingSlicedBuffer);
+ setAllowChangeInputs(allowChangeInputs);
dumpSetting();
@@ -141,10 +143,16 @@ public abstract class TestRawCoderBase extends TestCoderBase {
// Backup all the source chunks for later recovering because some coders
// may affect the source data.
ECChunk[] clonedDataChunks = cloneChunksWithData(dataChunks);
+ markChunks(dataChunks);
encoder.encode(dataChunks, parityChunks);
dumpChunks("Encoded parity chunks", parityChunks);
+ if (!allowChangeInputs) {
+ restoreChunksFromMark(dataChunks);
+ compareAndVerify(clonedDataChunks, dataChunks);
+ }
+
// Backup and erase some chunks
ECChunk[] backupChunks = backupAndEraseChunks(clonedDataChunks, parityChunks);
@@ -160,14 +168,31 @@ public abstract class TestRawCoderBase extends TestCoderBase {
corruptSomeChunk(recoveredChunks);
}
+ ECChunk[] clonedInputChunks = null;
+ if (!allowChangeInputs) {
+ markChunks(inputChunks);
+ clonedInputChunks = cloneChunksWithData(inputChunks);
+ }
+
dumpChunks("Decoding input chunks", inputChunks);
decoder.decode(inputChunks, getErasedIndexesForDecoding(), recoveredChunks);
dumpChunks("Decoded/recovered chunks", recoveredChunks);
+ if (!allowChangeInputs) {
+ restoreChunksFromMark(inputChunks);
+ compareAndVerify(clonedInputChunks, inputChunks);
+ }
+
// Compare
compareAndVerify(backupChunks, recoveredChunks);
}
+ private void setAllowChangeInputs(boolean allowChangeInputs) {
+ this.allowChangeInputs = allowChangeInputs;
+ encoder.setCoderOption(CoderOption.ALLOW_CHANGE_INPUTS, allowChangeInputs);
+ decoder.setCoderOption(CoderOption.ALLOW_CHANGE_INPUTS, allowChangeInputs);
+ }
+
private void prepareCoders() {
if (encoder == null) {
encoder = createEncoder();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e4f8a46/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoder.java
index 48463ad..3b07b24 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoder.java
@@ -29,6 +29,7 @@ public class TestXORRawCoder extends TestRawCoderBase {
public void setup() {
this.encoderClass = XORRawEncoder.class;
this.decoderClass = XORRawDecoder.class;
+ setAllowDump(false);
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e4f8a46/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index 7856133..7433256 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -892,7 +892,6 @@ public class DFSStripedInputStream extends DFSInputStream {
@Override
void decode() {
- // TODO no copy for data chunks. this depends on HADOOP-12047
final int span = (int) alignedStripe.getSpanInBlock();
for (int i = 0; i < alignedStripe.chunks.length; i++) {
if (alignedStripe.chunks[i] != null &&
[15/50] hadoop git commit: YARN-4313. Race condition in
MiniMRYarnCluster when getting history server address. Contributed by Jian He
Posted by ji...@apache.org.
YARN-4313. Race condition in MiniMRYarnCluster when getting history
server address. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7412ff48
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7412ff48
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7412ff48
Branch: refs/heads/HDFS-8966
Commit: 7412ff48eeb967c972c19c1370c77a41c5b3b81f
Parents: e5b1733
Author: Xuan <xg...@apache.org>
Authored: Thu Oct 29 17:36:36 2015 -0700
Committer: Xuan <xg...@apache.org>
Committed: Thu Oct 29 17:36:36 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java | 5 ++++-
hadoop-yarn-project/CHANGES.txt | 3 +++
2 files changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7412ff48/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
index 1dd6fca..3521834 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
@@ -190,6 +190,7 @@ public class MiniMRYarnCluster extends MiniYARNCluster {
public JobHistoryServerWrapper() {
super(JobHistoryServerWrapper.class.getName());
}
+ private volatile boolean jhsStarted = false;
@Override
public synchronized void serviceStart() throws Exception {
@@ -211,9 +212,11 @@ public class MiniMRYarnCluster extends MiniYARNCluster {
new Thread() {
public void run() {
historyServer.start();
+ jhsStarted = true;
};
}.start();
- while (historyServer.getServiceState() == STATE.INITED) {
+
+ while (!jhsStarted) {
LOG.info("Waiting for HistoryServer to start...");
Thread.sleep(1500);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7412ff48/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0da15bd..d0fa27d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1144,6 +1144,9 @@ Release 2.7.2 - UNRELEASED
YARN-4183. Enabling generic application history forces every job to get a
timeline service delegation token (Mit Desai via jeagles)
+ YARN-4313. Race condition in MiniMRYarnCluster when getting history server
+ address. (Jian He via xgong)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
[45/50] hadoop git commit: HADOOP-12541. make re2j dependency
consistent (Matthew Paduano via aw)
Posted by ji...@apache.org.
HADOOP-12541. make re2j dependency consistent (Matthew Paduano via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6e0d3532
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6e0d3532
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6e0d3532
Branch: refs/heads/HDFS-8966
Commit: 6e0d35323505cc68dbd963b8628b89ee04af2f2b
Parents: d565480
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Nov 2 20:39:46 2015 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Nov 2 20:39:46 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
hadoop-common-project/hadoop-common/pom.xml | 1 -
hadoop-project/pom.xml | 5 +++++
3 files changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e0d3532/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 1a9c93c..b055069 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -251,6 +251,8 @@ Trunk (Unreleased)
HADOOP-12133. Add schemas to Maven Assembly XMLs (Gábor Lipták via aw)
+ HADOOP-12541. make re2j dependency consistent (Matthew Paduano via aw)
+
BUG FIXES
HADOOP-11473. test-patch says "-1 overall" even when all checks are +1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e0d3532/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 4e47a3f..4735c6b 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -194,7 +194,6 @@
<dependency>
<groupId>com.google.re2j</groupId>
<artifactId>re2j</artifactId>
- <version>${re2j.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e0d3532/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index c974a61..efc3a7d 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -800,6 +800,11 @@
<version>1.8.1</version>
</dependency>
<dependency>
+ <groupId>com.google.re2j</groupId>
+ <artifactId>re2j</artifactId>
+ <version>${re2j.version}</version>
+ </dependency>
+ <dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${protobuf.version}</version>
[41/50] hadoop git commit: HDFS-9312. Fix TestReplication to be
FsDataset-agnostic. (lei)
Posted by ji...@apache.org.
HDFS-9312. Fix TestReplication to be FsDataset-agnostic. (lei)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76324094
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76324094
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76324094
Branch: refs/heads/HDFS-8966
Commit: 7632409482aaf06ecc6fe370a9f519afb969ad30
Parents: 78d6890
Author: Lei Xu <le...@apache.org>
Authored: Mon Nov 2 17:09:39 2015 -0800
Committer: Lei Xu <le...@apache.org>
Committed: Mon Nov 2 17:09:39 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../org/apache/hadoop/hdfs/TestReplication.java | 115 +++++--------------
.../server/datanode/FsDatasetTestUtils.java | 7 ++
.../fsdataset/impl/FsDatasetImplTestUtils.java | 25 ++++
4 files changed, 64 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76324094/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index fea4106..c13a725 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1656,6 +1656,8 @@ Release 2.8.0 - UNRELEASED
HDFS-9168. Move client side unit test to hadoop-hdfs-client. (wheat9)
+ HDFS-9312. Fix TestReplication to be FsDataset-agnostic. (lei)
+
BUG FIXES
HDFS-7501. TransactionsSinceLastCheckpoint can be negative on SBNs.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76324094/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
index 6424bc3..d9c96ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
@@ -20,22 +20,14 @@ package org.apache.hadoop.hdfs;
import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import com.google.common.base.Supplier;
-import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
-import java.io.RandomAccessFile;
import java.net.InetSocketAddress;
-import java.nio.file.FileVisitResult;
-import java.nio.file.Files;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -50,7 +42,6 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -62,6 +53,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.FsDatasetTestUtils.MaterializedReplica;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
@@ -367,7 +359,7 @@ public class TestReplication {
for (int i=0; i<buffer.length; i++) {
buffer[i] = '1';
}
-
+
try {
Configuration conf = new HdfsConfiguration();
conf.set(DFSConfigKeys.DFS_REPLICATION_KEY, Integer.toString(numDataNodes));
@@ -387,41 +379,29 @@ public class TestReplication {
// get first block of the file.
ExtendedBlock block = dfsClient.getNamenode().getBlockLocations(testFile,
0, Long.MAX_VALUE).get(0).getBlock();
-
- cluster.shutdown();
-
- for (int i=0; i<25; i++) {
- buffer[i] = '0';
+
+ List<MaterializedReplica> replicas = new ArrayList<>();
+ for (int dnIndex=0; dnIndex<3; dnIndex++) {
+ replicas.add(cluster.getMaterializedReplica(dnIndex, block));
}
-
+ assertEquals(3, replicas.size());
+
+ cluster.shutdown();
+
int fileCount = 0;
// Choose 3 copies of block file - delete 1 and corrupt the remaining 2
- for (int dnIndex=0; dnIndex<3; dnIndex++) {
- File blockFile = cluster.getBlockFile(dnIndex, block);
- LOG.info("Checking for file " + blockFile);
-
- if (blockFile != null && blockFile.exists()) {
- if (fileCount == 0) {
- LOG.info("Deleting file " + blockFile);
- assertTrue(blockFile.delete());
- } else {
- // corrupt it.
- LOG.info("Corrupting file " + blockFile);
- long len = blockFile.length();
- assertTrue(len > 50);
- RandomAccessFile blockOut = new RandomAccessFile(blockFile, "rw");
- try {
- blockOut.seek(len/3);
- blockOut.write(buffer, 0, 25);
- } finally {
- blockOut.close();
- }
- }
- fileCount++;
+ for (MaterializedReplica replica : replicas) {
+ if (fileCount == 0) {
+ LOG.info("Deleting block " + replica);
+ replica.deleteData();
+ } else {
+ // corrupt it.
+ LOG.info("Corrupting file " + replica);
+ replica.corruptData();
}
+ fileCount++;
}
- assertEquals(3, fileCount);
-
+
/* Start the MiniDFSCluster with more datanodes since once a writeBlock
* to a datanode node fails, same block can not be written to it
* immediately. In our case some replication attempts will fail.
@@ -530,63 +510,28 @@ public class TestReplication {
DFSConfigKeys.DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY, 1);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
FileSystem fs = cluster.getFileSystem();
- FSDataOutputStream create = fs.create(new Path("/test"));
- fs.setReplication(new Path("/test"), (short) 1);
+ Path filePath = new Path("/test");
+ FSDataOutputStream create = fs.create(filePath);
+ fs.setReplication(filePath, (short) 1);
create.write(new byte[1024]);
create.close();
- List<File> nonParticipatedNodeDirs = new ArrayList<File>();
- File participatedNodeDirs = null;
- for (int i = 0; i < cluster.getDataNodes().size(); i++) {
- File storageDir = cluster.getInstanceStorageDir(i, 0);
- String bpid = cluster.getNamesystem().getBlockPoolId();
- File data_dir = MiniDFSCluster.getFinalizedDir(storageDir, bpid);
- if (data_dir.listFiles().length == 0) {
- nonParticipatedNodeDirs.add(data_dir);
- } else {
- assertNull("participatedNodeDirs has already been set.",
- participatedNodeDirs);
- participatedNodeDirs = data_dir;
- }
- }
- assertEquals(2, nonParticipatedNodeDirs.size());
-
- String blockFile = null;
- final List<File> listFiles = new ArrayList<>();
- Files.walkFileTree(participatedNodeDirs.toPath(),
- new SimpleFileVisitor<java.nio.file.Path>() {
- @Override
- public FileVisitResult visitFile(
- java.nio.file.Path file, BasicFileAttributes attrs)
- throws IOException {
- listFiles.add(file.toFile());
- return FileVisitResult.CONTINUE;
- }
- }
- );
- assertFalse(listFiles.isEmpty());
+ ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, filePath);
int numReplicaCreated = 0;
- for (File file : listFiles) {
- if (file.getName().startsWith(Block.BLOCK_FILE_PREFIX)
- && !file.getName().endsWith("meta")) {
- blockFile = file.getName();
- for (File file1 : nonParticipatedNodeDirs) {
- file1.mkdirs();
- new File(file1, blockFile).createNewFile();
- new File(file1, blockFile + "_1000.meta").createNewFile();
- numReplicaCreated++;
- }
- break;
+ for (final DataNode dn : cluster.getDataNodes()) {
+ if (!dn.getFSDataset().contains(block)) {
+ cluster.getFsDatasetTestUtils(dn).injectCorruptReplica(block);
+ numReplicaCreated++;
}
}
assertEquals(2, numReplicaCreated);
- fs.setReplication(new Path("/test"), (short) 3);
+ fs.setReplication(filePath, (short) 3);
cluster.restartDataNodes(); // Lets detect all DNs about dummy copied
// blocks
cluster.waitActive();
cluster.triggerBlockReports();
- DFSTestUtil.waitReplication(fs, new Path("/test"), (short) 3);
+ DFSTestUtil.waitReplication(fs, filePath, (short) 3);
} finally {
if (cluster != null) {
cluster.shutdown();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76324094/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java
index eb986ff..40c4438 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java
@@ -199,4 +199,11 @@ public interface FsDatasetTestUtils {
* @throws IOException
*/
void checkStoredReplica(final Replica replica) throws IOException;
+
+ /**
+ * Create dummy replicas for block data and metadata.
+ * @param block the block of which replica to be created.
+ * @throws IOException on I/O error.
+ */
+ void injectCorruptReplica(ExtendedBlock block) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76324094/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java
index ed32fae..e8e4532 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
import com.google.common.base.Preconditions;
+import org.apache.commons.io.FileExistsException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -292,4 +293,28 @@ public class FsDatasetImplTestUtils implements FsDatasetTestUtils {
ReplicaInfo r = (ReplicaInfo) replica;
FsDatasetImpl.checkReplicaFiles(r);
}
+
+ @Override
+ public void injectCorruptReplica(ExtendedBlock block) throws IOException {
+ Preconditions.checkState(!dataset.contains(block),
+ "Block " + block + " already exists on dataset.");
+ try (FsVolumeReferences volRef = dataset.getFsVolumeReferences()) {
+ FsVolumeImpl volume = (FsVolumeImpl) volRef.get(0);
+ FinalizedReplica finalized = new FinalizedReplica(
+ block.getLocalBlock(),
+ volume,
+ volume.getFinalizedDir(block.getBlockPoolId()));
+ File blockFile = finalized.getBlockFile();
+ if (!blockFile.createNewFile()) {
+ throw new FileExistsException(
+ "Block file " + blockFile + " already exists.");
+ }
+ File metaFile = FsDatasetUtil.getMetaFile(blockFile, 1000);
+ if (!metaFile.createNewFile()) {
+ throw new FileExistsException(
+ "Meta file " + metaFile + " already exists."
+ );
+ }
+ }
+ }
}
[46/50] hadoop git commit: HADOOP-12542. TestDNS fails on Windows
after HADOOP-12437. Contributed by Chris Nauroth.
Posted by ji...@apache.org.
HADOOP-12542. TestDNS fails on Windows after HADOOP-12437. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/957f0311
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/957f0311
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/957f0311
Branch: refs/heads/HDFS-8966
Commit: 957f0311a160afb40dbb0619f455445b4f5d1e32
Parents: 6e0d353
Author: cnauroth <cn...@apache.org>
Authored: Mon Nov 2 22:25:05 2015 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Mon Nov 2 22:25:05 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../src/test/java/org/apache/hadoop/net/TestDNS.java | 12 +++++++++---
2 files changed, 11 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/957f0311/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index b055069..0d1bce2 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1312,6 +1312,8 @@ Release 2.8.0 - UNRELEASED
HADOOP-12508. delete fails with exception when lease is held on blob.
(Gaurav Kanade via cnauroth)
+ HADOOP-12542. TestDNS fails on Windows after HADOOP-12437. (cnauroth)
+
OPTIMIZATIONS
HADOOP-12051. ProtobufRpcEngine.invoke() should use Exception.toString()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/957f0311/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestDNS.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestDNS.java
index b26c7ca..a0bfe73 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestDNS.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestDNS.java
@@ -30,6 +30,7 @@ import javax.naming.NameNotFoundException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.Time;
import org.junit.Test;
@@ -37,6 +38,7 @@ import org.junit.Test;
import static org.hamcrest.CoreMatchers.not;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.*;
+import static org.junit.Assume.assumeTrue;
/**
* Test host name and IP resolution and caching.
@@ -185,13 +187,17 @@ public class TestDNS {
*
* This test may fail on some misconfigured test machines that don't have
* an entry for "localhost" in their hosts file. This entry is correctly
- * configured out of the box on common Linux distributions, OS X and
- * Windows.
+ * configured out of the box on common Linux distributions and OS X.
+ *
+ * Windows refuses to resolve 127.0.0.1 to "localhost" despite the presence of
+ * this entry in the hosts file. We skip the test on Windows to avoid
+ * reporting a spurious failure.
*
* @throws Exception
*/
@Test (timeout=60000)
public void testLookupWithHostsFallback() throws Exception {
+ assumeTrue(!Shell.WINDOWS);
final String oldHostname = changeDnsCachedHostname(DUMMY_HOSTNAME);
try {
@@ -231,7 +237,7 @@ public class TestDNS {
private String getLoopbackInterface() throws SocketException {
return NetworkInterface.getByInetAddress(
- InetAddress.getLoopbackAddress()).getDisplayName();
+ InetAddress.getLoopbackAddress()).getName();
}
/**
[37/50] hadoop git commit: HDFS-9329.
TestBootstrapStandby#testRateThrottling is flaky because fsimage size is
smaller than IO buffer size. Contributed by Zhe Zhang.
Posted by ji...@apache.org.
HDFS-9329. TestBootstrapStandby#testRateThrottling is flaky because fsimage size is smaller than IO buffer size. Contributed by Zhe Zhang.
Change-Id: I09896c46e9ee0718b67c64fac5acfb3f7decf0b9
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/259bea3b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/259bea3b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/259bea3b
Branch: refs/heads/HDFS-8966
Commit: 259bea3b48de7469a500831efb3306e8464a2dc9
Parents: 04d97f8
Author: Zhe Zhang <zh...@apache.org>
Authored: Mon Nov 2 10:03:39 2015 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Mon Nov 2 10:03:39 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../namenode/ha/TestBootstrapStandby.java | 79 +++++++++++++++-----
2 files changed, 62 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/259bea3b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3b2d997..a2e4824 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2204,6 +2204,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9343. Empty caller context considered invalid. (Mingliang Liu via
Arpit Agarwal)
+ HDFS-9329. TestBootstrapStandby#testRateThrottling is flaky because fsimage
+ size is smaller than IO buffer size. (zhz)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/259bea3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java
index fd45816..9f0d95b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java
@@ -25,13 +25,16 @@ import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
import com.google.common.base.Supplier;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
@@ -109,12 +112,16 @@ public class TestBootstrapStandby {
"storage directory does not exist or is not accessible", ioe);
}
+ int expectedCheckpointTxId = (int)NameNodeAdapter.getNamesystem(nn0)
+ .getFSImage().getMostRecentCheckpointTxId();
+
int rc = BootstrapStandby.run(new String[] { "-nonInteractive" },
cluster.getConfiguration(index));
assertEquals(0, rc);
// Should have copied over the namespace from the active
- FSImageTestUtil.assertNNHasCheckpoints(cluster, index, ImmutableList.of(0));
+ FSImageTestUtil.assertNNHasCheckpoints(cluster, index,
+ ImmutableList.of(expectedCheckpointTxId));
}
// We should now be able to start the standbys successfully.
@@ -221,7 +228,7 @@ public class TestBootstrapStandby {
* {@link DFSConfigKeys#DFS_IMAGE_TRANSFER_BOOTSTRAP_STANDBY_RATE_KEY}
* created by HDFS-8808.
*/
- @Test
+ @Test(timeout=30000)
public void testRateThrottling() throws Exception {
cluster.getConfiguration(0).setLong(
DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_KEY, 1);
@@ -229,23 +236,46 @@ public class TestBootstrapStandby {
cluster.waitActive();
nn0 = cluster.getNameNode(0);
cluster.transitionToActive(0);
- // Each edit has at least 1 byte. So the lowRate definitely should cause
- // a timeout, if enforced. If lowRate is not enforced, any reasonable test
- // machine should at least download an image with 5 edits in 5 seconds.
- for (int i = 0; i < 5; i++) {
+ // Any reasonable test machine should be able to transfer 1 byte per MS
+ // (which is ~1K/s)
+ final int minXferRatePerMS = 1;
+ int imageXferBufferSize = DFSUtilClient.getIoFileBufferSize(
+ new Configuration());
+ File imageFile = null;
+ int dirIdx = 0;
+ while (imageFile == null || imageFile.length() < imageXferBufferSize) {
+ for (int i = 0; i < 5; i++) {
+ cluster.getFileSystem(0).mkdirs(new Path("/foo" + dirIdx++));
+ }
nn0.getRpcServer().rollEditLog();
+ NameNodeAdapter.enterSafeMode(nn0, false);
+ NameNodeAdapter.saveNamespace(nn0);
+ NameNodeAdapter.leaveSafeMode(nn0);
+ imageFile = FSImageTestUtil.findLatestImageFile(FSImageTestUtil
+ .getFSImage(nn0).getStorage().getStorageDir(0));
}
+
+ final int timeOut = (int)(imageFile.length() / minXferRatePerMS) + 1;
// A very low DFS_IMAGE_TRANSFER_RATE_KEY value won't affect bootstrapping
+ final AtomicBoolean bootStrapped = new AtomicBoolean(false);
+ new Thread(
+ new Runnable() {
+ @Override
+ public void run() {
+ try {
+ testSuccessfulBaseCase();
+ bootStrapped.set(true);
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+ }
+ }
+ ).start();
GenericTestUtils.waitFor(new Supplier<Boolean>() {
public Boolean get() {
- try {
- testSuccessfulBaseCase();
- return true;
- } catch (Exception e) {
- return false;
- }
+ return bootStrapped.get();
}
- }, 500, 5000);
+ }, 50, timeOut);
shutdownCluster();
setupCluster();
@@ -257,17 +287,26 @@ public class TestBootstrapStandby {
cluster.transitionToActive(0);
// A very low DFS_IMAGE_TRANSFER_BOOTSTRAP_STANDBY_RATE_KEY value should
// cause timeout
+ bootStrapped.set(false);
+ new Thread(
+ new Runnable() {
+ @Override
+ public void run() {
+ try {
+ testSuccessfulBaseCase();
+ bootStrapped.set(true);
+ } catch (Exception e) {
+ LOG.info(e.getMessage());
+ }
+ }
+ }
+ ).start();
try {
GenericTestUtils.waitFor(new Supplier<Boolean>() {
public Boolean get() {
- try {
- testSuccessfulBaseCase();
- return true;
- } catch (Exception e) {
- return false;
- }
+ return bootStrapped.get();
}
- }, 500, 5000);
+ }, 50, timeOut);
fail("Did not timeout");
} catch (TimeoutException e) {
LOG.info("Encountered expected timeout.");
[12/50] hadoop git commit: YARN-4310. FairScheduler: Log skipping
reservation messages at DEBUG level (asuresh)
Posted by ji...@apache.org.
YARN-4310. FairScheduler: Log skipping reservation messages at DEBUG level (asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/58d1df58
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/58d1df58
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/58d1df58
Branch: refs/heads/HDFS-8966
Commit: 58d1df585c2b8a4d2f78ae30726f07b4c6a94731
Parents: ecad125
Author: Arun Suresh <as...@apache.org>
Authored: Thu Oct 29 13:42:09 2015 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Thu Oct 29 13:42:09 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 ++
.../scheduler/fair/FSAppAttempt.java | 16 +++++++++-------
2 files changed, 11 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58d1df58/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4bdf6ec..baa66c1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -549,6 +549,8 @@ Release 2.8.0 - UNRELEASED
YARN-3216. Max-AM-Resource-Percentage should respect node labels.
(Sunil G via wangda)
+ YARN-4310. FairScheduler: Log skipping reservation messages at DEBUG level (asuresh)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58d1df58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 6da4ebf..abbf77a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -488,13 +488,15 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
if (existingReservations >= numAllowedReservations) {
DecimalFormat df = new DecimalFormat();
df.setMaximumFractionDigits(2);
- LOG.info("Reservation Exceeds Allowed number of nodes:" +
- " app_id=" + getApplicationId() +
- " existingReservations=" + existingReservations +
- " totalAvailableNodes=" + totalAvailNodes +
- " reservableNodesRatio=" + df.format(
- scheduler.getReservableNodesRatio()) +
- " numAllowedReservations=" + numAllowedReservations);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Reservation Exceeds Allowed number of nodes:" +
+ " app_id=" + getApplicationId() +
+ " existingReservations=" + existingReservations +
+ " totalAvailableNodes=" + totalAvailNodes +
+ " reservableNodesRatio=" + df.format(
+ scheduler.getReservableNodesRatio()) +
+ " numAllowedReservations=" + numAllowedReservations);
+ }
return true;
}
}
[26/50] hadoop git commit: HADOOP-12133 Add schemas to Maven Assembly
XMLs
Posted by ji...@apache.org.
HADOOP-12133 Add schemas to Maven Assembly XMLs
Signed-off-by: Allen Wittenauer <aw...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/18727c63
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/18727c63
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/18727c63
Branch: refs/heads/HDFS-8966
Commit: 18727c63da721da9d29932378818d8742f705808
Parents: 45d3967
Author: Gábor Lipták <gl...@gmail.com>
Authored: Sat Jun 27 11:11:20 2015 -0400
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Fri Oct 30 11:36:52 2015 -0700
----------------------------------------------------------------------
hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml | 4 +++-
.../src/main/resources/assemblies/hadoop-hdfs-nfs-dist.xml | 4 +++-
.../src/main/resources/assemblies/hadoop-httpfs-dist.xml | 4 +++-
.../src/main/resources/assemblies/hadoop-kms-dist.xml | 4 +++-
.../src/main/resources/assemblies/hadoop-mapreduce-dist.xml | 4 ++--
.../src/main/resources/assemblies/hadoop-nfs-dist.xml | 4 +++-
hadoop-assemblies/src/main/resources/assemblies/hadoop-sls.xml | 4 +++-
hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml | 4 ++--
hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml | 4 ++--
.../src/main/resources/assemblies/hadoop-yarn-dist.xml | 4 ++--
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
hadoop-tools/hadoop-sls/src/main/assemblies/sls.xml | 4 +++-
12 files changed, 31 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18727c63/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
index 1a5d7d0..85899e5 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml
@@ -14,7 +14,9 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<assembly>
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
<id>hadoop-distro</id>
<formats>
<format>dir</format>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18727c63/hadoop-assemblies/src/main/resources/assemblies/hadoop-hdfs-nfs-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-hdfs-nfs-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-hdfs-nfs-dist.xml
index 89e8771..0edfdeb 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-hdfs-nfs-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-hdfs-nfs-dist.xml
@@ -12,7 +12,9 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<assembly>
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
<id>hadoop-hdfs-nfs-dist</id>
<formats>
<format>dir</format>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18727c63/hadoop-assemblies/src/main/resources/assemblies/hadoop-httpfs-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-httpfs-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-httpfs-dist.xml
index 6468a8a..4d508ee 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-httpfs-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-httpfs-dist.xml
@@ -12,7 +12,9 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<assembly>
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
<id>hadoop-httpfs-dist</id>
<formats>
<format>dir</format>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18727c63/hadoop-assemblies/src/main/resources/assemblies/hadoop-kms-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-kms-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-kms-dist.xml
index 9bbd0eb..5830bba 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-kms-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-kms-dist.xml
@@ -12,7 +12,9 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<assembly>
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
<id>hadoop-kms-dist</id>
<formats>
<format>dir</format>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18727c63/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
index df08c6c..5157edc 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
@@ -14,9 +14,9 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
<id>hadoop-mapreduce-dist</id>
<formats>
<format>dir</format>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18727c63/hadoop-assemblies/src/main/resources/assemblies/hadoop-nfs-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-nfs-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-nfs-dist.xml
index 9271239..cb3d9cd 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-nfs-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-nfs-dist.xml
@@ -12,7 +12,9 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<assembly>
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
<id>hadoop-nfs-dist</id>
<formats>
<format>dir</format>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18727c63/hadoop-assemblies/src/main/resources/assemblies/hadoop-sls.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-sls.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-sls.xml
index 8bdab7b..95b263a 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-sls.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-sls.xml
@@ -15,7 +15,9 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<assembly>
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
<id>hadoop-sls</id>
<formats>
<format>dir</format>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18727c63/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml
index 41c4fb6..b1e039f 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml
@@ -14,9 +14,9 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
<id>hadoop-src</id>
<formats>
<format>tar.gz</format>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18727c63/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
index 3f646e6..15c2572 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml
@@ -14,9 +14,9 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
<id>hadoop-tools</id>
<formats>
<format>dir</format>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18727c63/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index 6d386f1..219ed81 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -14,9 +14,9 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
<id>hadoop-yarn-dist</id>
<formats>
<format>dir</format>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18727c63/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index ddd0796..a1409f8 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -249,6 +249,8 @@ Trunk (Unreleased)
HADOOP-12385. Include nested stack trace in SaslRpcClient.getServerToken()
(stevel)
+ HADOOP-12133. Add schemas to Maven Assembly XMLs (Gábor Lipták via aw)
+
BUG FIXES
HADOOP-11473. test-patch says "-1 overall" even when all checks are +1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/18727c63/hadoop-tools/hadoop-sls/src/main/assemblies/sls.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/assemblies/sls.xml b/hadoop-tools/hadoop-sls/src/main/assemblies/sls.xml
index 5b09c81..085a5d8 100644
--- a/hadoop-tools/hadoop-sls/src/main/assemblies/sls.xml
+++ b/hadoop-tools/hadoop-sls/src/main/assemblies/sls.xml
@@ -15,7 +15,9 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-<assembly>
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
<id>sls</id>
<formats>
<format>dir</format>
[20/50] hadoop git commit: HDFS-4937. ReplicationMonitor can
infinite-loop in BlockPlacementPolicyDefault#chooseRandom(). Contributed by
Kihwal Lee.
Posted by ji...@apache.org.
HDFS-4937. ReplicationMonitor can infinite-loop in BlockPlacementPolicyDefault#chooseRandom(). Contributed by Kihwal Lee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/43539b5f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/43539b5f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/43539b5f
Branch: refs/heads/HDFS-8966
Commit: 43539b5ff4ac0874a8a454dc93a2a782b0e0ea8f
Parents: ce31b22
Author: Kihwal Lee <ki...@apache.org>
Authored: Fri Oct 30 09:27:21 2015 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Fri Oct 30 09:29:13 2015 -0500
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../blockmanagement/BlockPlacementPolicyDefault.java | 12 ++++++++++++
2 files changed, 15 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43539b5f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 38b9e55..f6a22a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2201,6 +2201,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9332. Fix Precondition failures from NameNodeEditLogRoller while
saving namespace. (wang)
+ HDFS-4937. ReplicationMonitor can infinite-loop in
+ BlockPlacementPolicyDefault#chooseRandom() (kihwal)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43539b5f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index d9b8d60..f610574 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -659,6 +659,7 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
int numOfAvailableNodes = clusterMap.countNumOfAvailableNodes(
scope, excludedNodes);
+ int refreshCounter = numOfAvailableNodes;
StringBuilder builder = null;
if (LOG.isDebugEnabled()) {
builder = debugLoggingBuilder.get();
@@ -708,6 +709,17 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
// If no candidate storage was found on this DN then set badTarget.
badTarget = (storage == null);
}
+ // Refresh the node count. If the live node count became smaller,
+ // but it is not reflected in this loop, it may loop forever in case
+ // the replicas/rack cannot be satisfied.
+ if (--refreshCounter == 0) {
+ refreshCounter = clusterMap.countNumOfAvailableNodes(scope,
+ excludedNodes);
+ // It has already gone through enough number of nodes.
+ if (refreshCounter <= excludedNodes.size()) {
+ break;
+ }
+ }
}
if (numOfReplicas>0) {
[43/50] hadoop git commit: HDFS-9308. Add truncateMeta() and
deleteMeta() to MiniDFSCluster. (Tony Wu via lei)
Posted by ji...@apache.org.
HDFS-9308. Add truncateMeta() and deleteMeta() to MiniDFSCluster. (Tony Wu via lei)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8e05dbf2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8e05dbf2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8e05dbf2
Branch: refs/heads/HDFS-8966
Commit: 8e05dbf2bddce95d5f5a5bae5df61acabf0ba7c5
Parents: 5ba2b98
Author: Lei Xu <le...@apache.org>
Authored: Mon Nov 2 18:05:43 2015 -0800
Committer: Lei Xu <le...@apache.org>
Committed: Mon Nov 2 18:05:43 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 22 ++++
.../apache/hadoop/hdfs/TestCrcCorruption.java | 120 +++++--------------
.../apache/hadoop/hdfs/TestLeaseRecovery.java | 35 ++++--
4 files changed, 76 insertions(+), 103 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e05dbf2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3c60549..19ea5c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1661,6 +1661,8 @@ Release 2.8.0 - UNRELEASED
HDFS-9312. Fix TestReplication to be FsDataset-agnostic. (lei)
+ HDFS-9308. Add truncateMeta() and deleteMeta() to MiniDFSCluster. (Tony Wu via lei)
+
BUG FIXES
HDFS-7501. TransactionsSinceLastCheckpoint can be negative on SBNs.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e05dbf2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 7ebf333..c81f154 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -2117,6 +2117,28 @@ public class MiniDFSCluster {
getMaterializedReplica(i, blk).corruptMeta();
}
+ /**
+ * Corrupt the metadata of a block by deleting it.
+ * @param i index of the datanode
+ * @param blk name of the block.
+ */
+ public void deleteMeta(int i, ExtendedBlock blk)
+ throws IOException {
+ getMaterializedReplica(i, blk).deleteMeta();
+ }
+
+ /**
+ * Corrupt the metadata of a block by truncating it to a new size.
+ * @param i index of the datanode.
+ * @param blk name of the block.
+ * @param newSize the new size of the metadata file.
+ * @throws IOException if any I/O errors.
+ */
+ public void truncateMeta(int i, ExtendedBlock blk, int newSize)
+ throws IOException {
+ getMaterializedReplica(i, blk).truncateMeta(newSize);
+ }
+
public boolean changeGenStampOfBlock(int dnIndex, ExtendedBlock blk,
long newGenStamp) throws IOException {
File blockFile = getBlockFile(dnIndex, blk);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e05dbf2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestCrcCorruption.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestCrcCorruption.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestCrcCorruption.java
index 3850ff2..398bcc2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestCrcCorruption.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestCrcCorruption.java
@@ -22,11 +22,8 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.io.File;
import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
+import java.util.List;
import java.util.Random;
import org.apache.hadoop.conf.Configuration;
@@ -35,12 +32,15 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
-import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
import org.apache.hadoop.io.IOUtils;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A JUnit test for corrupted file handling.
@@ -70,6 +70,8 @@ import org.mockito.Mockito;
* replica was created from the non-corrupted replica.
*/
public class TestCrcCorruption {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(TestCrcCorruption.class);
private DFSClientFaultInjector faultInjector;
@@ -167,90 +169,26 @@ public class TestCrcCorruption {
// file disallows this Datanode to send data to another datanode.
// However, a client is alowed access to this block.
//
- File storageDir = cluster.getInstanceStorageDir(0, 1);
- String bpid = cluster.getNamesystem().getBlockPoolId();
- File data_dir = MiniDFSCluster.getFinalizedDir(storageDir, bpid);
- assertTrue("data directory does not exist", data_dir.exists());
- File[] blocks = data_dir.listFiles();
- assertTrue("Blocks do not exist in data-dir", (blocks != null) && (blocks.length > 0));
- int num = 0;
- for (int idx = 0; idx < blocks.length; idx++) {
- if (blocks[idx].getName().startsWith(Block.BLOCK_FILE_PREFIX) &&
- blocks[idx].getName().endsWith(".meta")) {
- num++;
- if (num % 3 == 0) {
- //
- // remove .meta file
- //
- System.out.println("Deliberately removing file " + blocks[idx].getName());
- assertTrue("Cannot remove file.", blocks[idx].delete());
- } else if (num % 3 == 1) {
- //
- // shorten .meta file
- //
- RandomAccessFile file = new RandomAccessFile(blocks[idx], "rw");
- FileChannel channel = file.getChannel();
- int newsize = random.nextInt((int)channel.size()/2);
- System.out.println("Deliberately truncating file " +
- blocks[idx].getName() +
- " to size " + newsize + " bytes.");
- channel.truncate(newsize);
- file.close();
- } else {
- //
- // corrupt a few bytes of the metafile
- //
- RandomAccessFile file = new RandomAccessFile(blocks[idx], "rw");
- FileChannel channel = file.getChannel();
- long position = 0;
- //
- // The very first time, corrupt the meta header at offset 0
- //
- if (num != 2) {
- position = (long)random.nextInt((int)channel.size());
- }
- int length = random.nextInt((int)(channel.size() - position + 1));
- byte[] buffer = new byte[length];
- random.nextBytes(buffer);
- channel.write(ByteBuffer.wrap(buffer), position);
- System.out.println("Deliberately corrupting file " +
- blocks[idx].getName() +
- " at offset " + position +
- " length " + length);
- file.close();
- }
- }
- }
-
- //
- // Now deliberately corrupt all meta blocks from the second
- // directory of the first datanode
- //
- storageDir = cluster.getInstanceStorageDir(0, 1);
- data_dir = MiniDFSCluster.getFinalizedDir(storageDir, bpid);
- assertTrue("data directory does not exist", data_dir.exists());
- blocks = data_dir.listFiles();
- assertTrue("Blocks do not exist in data-dir", (blocks != null) && (blocks.length > 0));
-
- int count = 0;
- File previous = null;
- for (int idx = 0; idx < blocks.length; idx++) {
- if (blocks[idx].getName().startsWith("blk_") &&
- blocks[idx].getName().endsWith(".meta")) {
- //
- // Move the previous metafile into the current one.
- //
- count++;
- if (count % 2 == 0) {
- System.out.println("Deliberately insertimg bad crc into files " +
- blocks[idx].getName() + " " + previous.getName());
- assertTrue("Cannot remove file.", blocks[idx].delete());
- assertTrue("Cannot corrupt meta file.", previous.renameTo(blocks[idx]));
- assertTrue("Cannot recreate empty meta file.", previous.createNewFile());
- previous = null;
- } else {
- previous = blocks[idx];
- }
+ final int dnIdx = 0;
+ final DataNode dn = cluster.getDataNodes().get(dnIdx);
+ final String bpid = cluster.getNamesystem().getBlockPoolId();
+ List<FinalizedReplica> replicas =
+ dn.getFSDataset().getFinalizedBlocks(bpid);
+ assertTrue("Replicas do not exist", !replicas.isEmpty());
+
+ for (int idx = 0; idx < replicas.size(); idx++) {
+ FinalizedReplica replica = replicas.get(idx);
+ ExtendedBlock eb = new ExtendedBlock(bpid, replica);
+ if (idx % 3 == 0) {
+ LOG.info("Deliberately removing meta for block " + eb);
+ cluster.deleteMeta(dnIdx, eb);
+ } else if (idx % 3 == 1) {
+ final int newSize = 2; // bytes
+ LOG.info("Deliberately truncating meta file for block " +
+ eb + " to size " + newSize + " bytes.");
+ cluster.truncateMeta(dnIdx, eb, newSize);
+ } else {
+ cluster.corruptMeta(dnIdx, eb);
}
}
@@ -260,7 +198,7 @@ public class TestCrcCorruption {
//
assertTrue("Corrupted replicas not handled properly.",
util.checkFiles(fs, "/srcdat"));
- System.out.println("All File still have a valid replica");
+ LOG.info("All File still have a valid replica");
//
// set replication factor back to 1. This causes only one replica of
@@ -273,7 +211,7 @@ public class TestCrcCorruption {
//System.out.println("All Files done with removing replicas");
//assertTrue("Excess replicas deleted. Corrupted replicas found.",
// util.checkFiles(fs, "/srcdat"));
- System.out.println("The excess-corrupted-replica test is disabled " +
+ LOG.info("The excess-corrupted-replica test is disabled " +
" pending HADOOP-1557");
util.cleanup(fs, "/srcdat");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e05dbf2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java
index c9f3842..4ae130d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery.java
@@ -20,9 +20,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.io.File;
import java.io.IOException;
-import java.io.RandomAccessFile;
import java.util.EnumSet;
import org.apache.hadoop.conf.Configuration;
@@ -32,7 +30,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -178,8 +175,9 @@ public class TestLeaseRecovery {
Path file = new Path("/testRecoveryFile");
DistributedFileSystem dfs = cluster.getFileSystem();
FSDataOutputStream out = dfs.create(file);
+ final int FILE_SIZE = 2 * 1024 * 1024;
int count = 0;
- while (count < 2 * 1024 * 1024) {
+ while (count < FILE_SIZE) {
out.writeBytes("Data");
count += 4;
}
@@ -190,15 +188,23 @@ public class TestLeaseRecovery {
LocatedBlocks locations = cluster.getNameNodeRpc().getBlockLocations(
file.toString(), 0, count);
ExtendedBlock block = locations.get(0).getBlock();
- DataNode dn = cluster.getDataNodes().get(0);
- BlockLocalPathInfo localPathInfo = dn.getBlockLocalPathInfo(block, null);
- File metafile = new File(localPathInfo.getMetaPath());
- assertTrue(metafile.exists());
- // reduce the block meta file size
- RandomAccessFile raf = new RandomAccessFile(metafile, "rw");
- raf.setLength(metafile.length() - 20);
- raf.close();
+ // Calculate meta file size
+ // From DataNode.java, checksum size is given by:
+ // (length of data + BYTE_PER_CHECKSUM - 1)/BYTES_PER_CHECKSUM *
+ // CHECKSUM_SIZE
+ final int CHECKSUM_SIZE = 4; // CRC32 & CRC32C
+ final int bytesPerChecksum = conf.getInt(
+ DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY,
+ DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT);
+ final int metaFileSize =
+ (FILE_SIZE + bytesPerChecksum - 1) / bytesPerChecksum * CHECKSUM_SIZE +
+ 8; // meta file header is 8 bytes
+ final int newMetaFileSize = metaFileSize - CHECKSUM_SIZE;
+
+ // Corrupt the block meta file by dropping checksum for bytesPerChecksum
+ // bytes. Lease recovery is expected to recover the uncorrupted file length.
+ cluster.truncateMeta(0, block, newMetaFileSize);
// restart DN to make replica to RWR
DataNodeProperties dnProp = cluster.stopDataNode(0);
@@ -213,6 +219,11 @@ public class TestLeaseRecovery {
}
assertTrue("File should be closed", newdfs.recoverLease(file));
+ // Verify file length after lease recovery. The new file length should not
+ // include the bytes with corrupted checksum.
+ final long expectedNewFileLen = FILE_SIZE - bytesPerChecksum;
+ final long newFileLen = newdfs.getFileStatus(file).getLen();
+ assertEquals(newFileLen, expectedNewFileLen);
}
/**
[36/50] hadoop git commit: MAPREDUCE-6525. Fix test failure of
TestMiniMRClientCluster.testRestart. Contributed by Masatake Iwasaki.
Posted by ji...@apache.org.
MAPREDUCE-6525. Fix test failure of TestMiniMRClientCluster.testRestart. Contributed by Masatake Iwasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/04d97f8a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/04d97f8a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/04d97f8a
Branch: refs/heads/HDFS-8966
Commit: 04d97f8abb7fcc7b635b9499a48ddaa1fe0ac7e3
Parents: 90e1405
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Nov 3 01:48:45 2015 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Nov 3 01:50:07 2015 +0900
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 ++
.../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 36 ++++++++++++--------
2 files changed, 24 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/04d97f8a/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 4d6dcb8..23bef37 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -287,6 +287,9 @@ Trunk (Unreleased)
MAPREDUCE-5801. Uber mode's log message is missing a vcore reason
(Steven Wong via aw)
+ MAPREDUCE-6525. Fix test failure of TestMiniMRClientCluster.testRestart.
+ (Masatake Iwasaki via aajisaka)
+
Release 2.8.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/04d97f8a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
index 3521834..cad6f3a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
@@ -186,6 +186,27 @@ public class MiniMRYarnCluster extends MiniYARNCluster {
super.serviceInit(conf);
}
+ @Override
+ protected void serviceStart() throws Exception {
+ super.serviceStart();
+
+ //need to do this because historyServer.init creates a new Configuration
+ getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
+ historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS));
+ MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(),
+ MRWebAppUtil.getJHSWebappURLWithoutScheme(historyServer.getConfig()));
+
+ LOG.info("MiniMRYARN ResourceManager address: " +
+ getConfig().get(YarnConfiguration.RM_ADDRESS));
+ LOG.info("MiniMRYARN ResourceManager web address: " +
+ WebAppUtils.getRMWebAppURLWithoutScheme(getConfig()));
+ LOG.info("MiniMRYARN HistoryServer address: " +
+ getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS));
+ LOG.info("MiniMRYARN HistoryServer web address: " +
+ getResolvedMRHistoryWebAppURLWithoutScheme(getConfig(),
+ MRWebAppUtil.getJHSHttpPolicy() == HttpConfig.Policy.HTTPS_ONLY));
+ }
+
private class JobHistoryServerWrapper extends AbstractService {
public JobHistoryServerWrapper() {
super(JobHistoryServerWrapper.class.getName());
@@ -228,21 +249,6 @@ public class MiniMRYarnCluster extends MiniYARNCluster {
} catch (Throwable t) {
throw new YarnRuntimeException(t);
}
- //need to do this because historyServer.init creates a new Configuration
- getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
- historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS));
- MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(),
- MRWebAppUtil.getJHSWebappURLWithoutScheme(historyServer.getConfig()));
-
- LOG.info("MiniMRYARN ResourceManager address: " +
- getConfig().get(YarnConfiguration.RM_ADDRESS));
- LOG.info("MiniMRYARN ResourceManager web address: " +
- WebAppUtils.getRMWebAppURLWithoutScheme(getConfig()));
- LOG.info("MiniMRYARN HistoryServer address: " +
- getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS));
- LOG.info("MiniMRYARN HistoryServer web address: "
- + getResolvedMRHistoryWebAppURLWithoutScheme(getConfig(),
- MRWebAppUtil.getJHSHttpPolicy() == HttpConfig.Policy.HTTPS_ONLY));
}
@Override
[50/50] hadoop git commit: Merge remote-tracking branch
'origin/trunk' into HDFS-8966
Posted by ji...@apache.org.
Merge remote-tracking branch 'origin/trunk' into HDFS-8966
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27009cf3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27009cf3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27009cf3
Branch: refs/heads/HDFS-8966
Commit: 27009cf32aa8044e38f8d0761d8193c2010c87ca
Parents: 08aafac 7e28296
Author: Jing Zhao <ji...@apache.org>
Authored: Tue Nov 3 13:07:04 2015 -0800
Committer: Jing Zhao <ji...@apache.org>
Committed: Tue Nov 3 13:07:04 2015 -0800
----------------------------------------------------------------------
.../main/resources/assemblies/hadoop-dist.xml | 4 +-
.../assemblies/hadoop-hdfs-nfs-dist.xml | 4 +-
.../resources/assemblies/hadoop-httpfs-dist.xml | 4 +-
.../resources/assemblies/hadoop-kms-dist.xml | 4 +-
.../assemblies/hadoop-mapreduce-dist.xml | 4 +-
.../resources/assemblies/hadoop-nfs-dist.xml | 4 +-
.../main/resources/assemblies/hadoop-sls.xml | 4 +-
.../main/resources/assemblies/hadoop-src.xml | 4 +-
.../main/resources/assemblies/hadoop-tools.xml | 4 +-
.../resources/assemblies/hadoop-yarn-dist.xml | 4 +-
hadoop-common-project/hadoop-common/CHANGES.txt | 32 +-
hadoop-common-project/hadoop-common/pom.xml | 1 -
.../java/org/apache/hadoop/fs/FileSystem.java | 19 +-
.../org/apache/hadoop/fs/FilterFileSystem.java | 8 +-
.../org/apache/hadoop/fs/HarFileSystem.java | 6 +
.../rawcoder/AbstractRawErasureCoder.java | 107 +-
.../rawcoder/AbstractRawErasureDecoder.java | 8 +-
.../rawcoder/AbstractRawErasureEncoder.java | 8 +-
.../io/erasurecode/rawcoder/CoderOption.java | 43 +
.../io/erasurecode/rawcoder/RSRawDecoder.java | 4 +-
.../io/erasurecode/rawcoder/RSRawEncoder.java | 48 +-
.../erasurecode/rawcoder/RawErasureCoder.java | 23 +-
.../erasurecode/rawcoder/RawErasureDecoder.java | 32 +-
.../erasurecode/rawcoder/RawErasureEncoder.java | 35 +-
.../io/erasurecode/rawcoder/XORRawDecoder.java | 1 -
.../io/erasurecode/rawcoder/XORRawEncoder.java | 1 -
.../apache/hadoop/io/retry/RetryPolicies.java | 44 +-
.../org/apache/hadoop/ipc/CallerContext.java | 13 +-
.../java/org/apache/hadoop/util/ProtoUtil.java | 2 +-
.../hadoop-common/src/site/markdown/Metrics.md | 1 +
.../hadoop/io/erasurecode/TestCoderBase.java | 34 +-
.../erasurecode/rawcoder/TestRawCoderBase.java | 37 +-
.../erasurecode/rawcoder/TestXORRawCoder.java | 1 +
.../apache/hadoop/io/retry/TestRetryProxy.java | 27 +-
.../io/retry/UnreliableImplementation.java | 17 +
.../hadoop/io/retry/UnreliableInterface.java | 3 +
.../java/org/apache/hadoop/net/TestDNS.java | 12 +-
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 21 +
.../hadoop/hdfs/DFSStripedInputStream.java | 1 -
.../hadoop/hdfs/DFSStripedOutputStream.java | 21 +-
.../datatransfer/ReplaceDatanodeOnFailure.java | 78 +-
.../hadoop/fs/TestUrlStreamHandlerFactory.java | 80 +
.../java/org/apache/hadoop/fs/TestXAttr.java | 94 +
.../org/apache/hadoop/hdfs/TestDFSPacket.java | 94 +
.../hadoop/hdfs/TestDefaultNameNodePort.java | 66 +
.../org/apache/hadoop/hdfs/TestPeerCache.java | 292 +++
.../hdfs/client/impl/TestLeaseRenewer.java | 207 +++
.../hadoop/hdfs/protocol/TestExtendedBlock.java | 77 +
.../hdfs/shortcircuit/TestShortCircuitShm.java | 109 ++
.../hadoop/hdfs/util/TestByteArrayManager.java | 644 +++++++
.../hdfs/util/TestExactSizeInputStream.java | 129 ++
.../hdfs/web/TestByteRangeInputStream.java | 292 +++
.../hdfs/web/TestOffsetUrlInputStream.java | 64 +
.../apache/hadoop/hdfs/web/TestTokenAspect.java | 316 ++++
.../hdfs/web/TestURLConnectionFactory.java | 50 +
.../hadoop/hdfs/web/TestWebHDFSOAuth2.java | 217 +++
.../hdfs/web/TestWebHdfsContentLength.java | 197 +++
.../hdfs/web/oauth2/TestAccessTokenTimer.java | 63 +
...ClientCredentialTimeBasedTokenRefresher.java | 139 ++
...TestRefreshTokenTimeBasedTokenRefresher.java | 138 ++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 69 +-
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 6 -
.../server/blockmanagement/BlockManager.java | 5 +
.../blockmanagement/BlockPlacementPolicy.java | 8 +-
.../BlockPlacementPolicyDefault.java | 46 +-
.../BlockPlacementPolicyWithNodeGroup.java | 78 +-
.../hadoop/hdfs/server/common/Storage.java | 15 +
.../erasurecode/ErasureCodingWorker.java | 7 +-
.../hadoop/hdfs/server/namenode/FSImage.java | 4 +
.../hdfs/server/namenode/FSNamesystem.java | 17 +-
.../hadoop/hdfs/server/namenode/NNStorage.java | 25 +
.../hdfs/server/namenode/NameNodeMXBean.java | 5 +
.../hdfs/server/namenode/ha/EditLogTailer.java | 4 +-
.../hadoop/fs/TestUrlStreamHandlerFactory.java | 80 -
.../java/org/apache/hadoop/fs/TestXAttr.java | 94 -
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 22 +
.../apache/hadoop/hdfs/StripedFileTestUtil.java | 8 +-
.../apache/hadoop/hdfs/TestAclsEndToEnd.java | 1657 ++++++++++++++++++
.../apache/hadoop/hdfs/TestCrcCorruption.java | 120 +-
.../org/apache/hadoop/hdfs/TestDFSPacket.java | 94 -
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 2 +
.../TestDFSStripedOutputStreamWithFailure.java | 33 +-
...estDFSStripedOutputStreamWithFailure020.java | 22 +
...estDFSStripedOutputStreamWithFailure030.java | 22 +
...estDFSStripedOutputStreamWithFailure040.java | 22 +
...estDFSStripedOutputStreamWithFailure050.java | 22 +
...estDFSStripedOutputStreamWithFailure060.java | 22 +
...estDFSStripedOutputStreamWithFailure070.java | 22 +
...estDFSStripedOutputStreamWithFailure080.java | 22 +
...estDFSStripedOutputStreamWithFailure090.java | 22 +
...estDFSStripedOutputStreamWithFailure100.java | 22 +
...estDFSStripedOutputStreamWithFailure110.java | 22 +
...estDFSStripedOutputStreamWithFailure120.java | 22 +
...estDFSStripedOutputStreamWithFailure130.java | 22 +
...estDFSStripedOutputStreamWithFailure140.java | 22 +
...estDFSStripedOutputStreamWithFailure150.java | 22 +
...estDFSStripedOutputStreamWithFailure160.java | 22 +
...estDFSStripedOutputStreamWithFailure170.java | 22 +
...estDFSStripedOutputStreamWithFailure180.java | 22 +
...estDFSStripedOutputStreamWithFailure190.java | 22 +
...estDFSStripedOutputStreamWithFailure200.java | 22 +
...estDFSStripedOutputStreamWithFailure210.java | 23 +
.../hadoop/hdfs/TestDefaultNameNodePort.java | 68 -
.../hadoop/hdfs/TestDistributedFileSystem.java | 28 +
.../TestErasureCodingPolicyWithSnapshot.java | 199 +++
.../apache/hadoop/hdfs/TestLeaseRecovery.java | 35 +-
.../org/apache/hadoop/hdfs/TestPeerCache.java | 293 ----
.../TestReadStripedFileWithMissingBlocks.java | 6 +-
.../hadoop/hdfs/TestRecoverStripedFile.java | 143 +-
.../org/apache/hadoop/hdfs/TestReplication.java | 115 +-
.../hdfs/TestSafeModeWithStripedFile.java | 5 +-
.../hadoop/hdfs/TestWriteReadStripedFile.java | 8 +-
.../hdfs/TestWriteStripedFileWithFailure.java | 6 +-
.../hdfs/client/impl/TestLeaseRenewer.java | 209 ---
.../hadoop/hdfs/protocol/TestExtendedBlock.java | 77 -
.../hdfs/server/balancer/TestBalancer.java | 572 +++---
.../TestBlockTokenWithDFSStriped.java | 4 +-
.../blockmanagement/TestReplicationPolicy.java | 31 +
.../TestReplicationPolicyWithNodeGroup.java | 58 +
.../server/datanode/FsDatasetTestUtils.java | 7 +
.../fsdataset/impl/FsDatasetImplTestUtils.java | 25 +
.../hdfs/server/namenode/TestAuditLogger.java | 41 +-
.../server/namenode/TestNameNodeMXBean.java | 62 +-
.../namenode/TestRecoverStripedBlocks.java | 70 +
.../namenode/ha/TestBootstrapStandby.java | 79 +-
.../hdfs/shortcircuit/TestShortCircuitShm.java | 109 --
.../hadoop/hdfs/util/TestByteArrayManager.java | 645 -------
.../hdfs/util/TestExactSizeInputStream.java | 129 --
.../hdfs/web/TestByteRangeInputStream.java | 292 ---
.../hdfs/web/TestOffsetUrlInputStream.java | 64 -
.../apache/hadoop/hdfs/web/TestTokenAspect.java | 316 ----
.../hdfs/web/TestURLConnectionFactory.java | 50 -
.../hadoop/hdfs/web/TestWebHDFSOAuth2.java | 216 ---
.../hdfs/web/TestWebHdfsContentLength.java | 197 ---
.../hdfs/web/oauth2/TestAccessTokenTimer.java | 63 -
...ClientCredentialTimeBasedTokenRefresher.java | 139 --
...TestRefreshTokenTimeBasedTokenRefresher.java | 138 --
hadoop-mapreduce-project/CHANGES.txt | 48 +-
.../apache/hadoop/mapreduce/v2/app/job/Job.java | 2 +
.../mapreduce/v2/app/job/impl/JobImpl.java | 13 +-
.../v2/app/rm/RMContainerAllocator.java | 9 +
.../jobhistory/TestJobHistoryEventHandler.java | 10 +-
.../hadoop/mapreduce/v2/app/MockJobs.java | 6 +
.../mapreduce/v2/app/TestRuntimeEstimators.java | 6 +
.../mapreduce/v2/app/job/impl/TestJobImpl.java | 35 +
.../v2/app/rm/TestRMContainerAllocator.java | 2 +
.../apache/hadoop/mapreduce/TypeConverter.java | 34 +-
.../mapreduce/v2/api/records/JobReport.java | 4 +
.../v2/api/records/impl/pb/JobReportPBImpl.java | 44 +-
.../mapreduce/v2/util/MRBuilderUtils.java | 15 +-
.../src/main/proto/mr_protos.proto | 1 +
.../hadoop/mapreduce/TestTypeConverter.java | 8 +-
.../org/apache/hadoop/mapred/JobPriority.java | 5 +-
.../apache/hadoop/mapreduce/JobPriority.java | 4 +-
.../hadoop/mapreduce/v2/hs/CompletedJob.java | 7 +
.../mapreduce/v2/hs/HistoryFileManager.java | 13 +-
.../hadoop/mapreduce/v2/hs/PartialJob.java | 7 +
.../v2/hs/webapp/TestHsWebServicesAcls.java | 5 +
.../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 41 +-
hadoop-project/pom.xml | 10 +
.../fs/azure/AzureNativeFileSystemStore.java | 32 +-
.../hadoop/fs/azure/NativeAzureFileSystem.java | 125 +-
.../hadoop/fs/azure/SelfRenewingLease.java | 5 +-
...estFileSystemOperationExceptionHandling.java | 131 ++
...perationsExceptionHandlingMultiThreaded.java | 185 ++
.../fs/azure/TestNativeAzureFileSystemLive.java | 86 +
.../tools/mapred/lib/DynamicInputChunk.java | 137 +-
.../mapred/lib/DynamicInputChunkContext.java | 113 ++
.../tools/mapred/lib/DynamicInputFormat.java | 31 +-
.../tools/mapred/lib/DynamicRecordReader.java | 13 +-
.../org/apache/hadoop/tools/StubContext.java | 4 +
.../mapred/lib/TestDynamicInputFormat.java | 33 +-
.../hadoop-sls/src/main/assemblies/sls.xml | 4 +-
hadoop-yarn-project/CHANGES.txt | 62 +-
.../distributedshell/TestDistributedShell.java | 9 +-
.../org/apache/hadoop/yarn/client/RMProxy.java | 6 +-
.../ApplicationHistoryServer.java | 2 +-
.../nodemanager/DefaultContainerExecutor.java | 9 +-
.../nodemanager/DockerContainerExecutor.java | 9 +-
.../localizer/LocalResourcesTrackerImpl.java | 10 +
.../localizer/ResourceLocalizationService.java | 13 +
.../impl/container-executor.c | 9 +-
.../test/test-container-executor.c | 11 +
.../TestLocalResourcesTrackerImpl.java | 6 +-
.../TestResourceLocalizationService.java | 293 +++-
.../server/resourcemanager/RMAppManager.java | 8 +-
.../metrics/SystemMetricsPublisher.java | 4 +-
.../recovery/ZKRMStateStore.java | 25 +-
.../scheduler/fair/FSAppAttempt.java | 16 +-
.../metrics/TestSystemMetricsPublisher.java | 2 +-
.../recovery/TestZKRMStateStore.java | 68 +
191 files changed, 8716 insertions(+), 4409 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27009cf3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27009cf3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27009cf3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
----------------------------------------------------------------------
[23/50] hadoop git commit: Update CHANGES.txt to reflect commit of
MR-6273 to branch-2.6
Posted by ji...@apache.org.
Update CHANGES.txt to reflect commit of MR-6273 to branch-2.6
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6ae9efaf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6ae9efaf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6ae9efaf
Branch: refs/heads/HDFS-8966
Commit: 6ae9efaf5949bd5a5f4fd99b5777ce8f6d7f3a2c
Parents: eadf7b3
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Oct 30 15:18:53 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Oct 30 15:18:53 2015 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ae9efaf/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 8594e1e..22f9e89 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -900,6 +900,10 @@ Release 2.6.3 - UNRELEASED
BUG FIXES
+ MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists to
+ avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED state.
+ (zhihai xu via devaraj)
+
Release 2.6.2 - 2015-10-21
INCOMPATIBLE CHANGES
[40/50] hadoop git commit: HDFS-9339. Extend full test of KMS ACLs.
Contributed by Daniel Templeton.
Posted by ji...@apache.org.
HDFS-9339. Extend full test of KMS ACLs. Contributed by Daniel Templeton.
Change-Id: I61eeee8fa5e85250eabc1eef3d8c11f32700d6fb
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/78d68908
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/78d68908
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/78d68908
Branch: refs/heads/HDFS-8966
Commit: 78d6890865424db850faecfc5c76f14c64925063
Parents: 9e7dcab
Author: Zhe Zhang <zh...@apache.org>
Authored: Mon Nov 2 13:51:45 2015 -0800
Committer: Zhe Zhang <zh...@apache.org>
Committed: Mon Nov 2 13:51:45 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../apache/hadoop/hdfs/TestAclsEndToEnd.java | 1042 +++++++++++++++++-
2 files changed, 1041 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78d68908/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index a2e4824..fea4106 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1613,6 +1613,8 @@ Release 2.8.0 - UNRELEASED
HDFS-9229. Expose size of NameNode directory as a metric.
(Surendra Singh Lilhore via zhz)
+ HDFS-9339. Extend full test of KMS ACLs. (Daniel Templeton via zhz)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78d68908/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAclsEndToEnd.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAclsEndToEnd.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAclsEndToEnd.java
index de0646a..2b515d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAclsEndToEnd.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAclsEndToEnd.java
@@ -233,15 +233,15 @@ public class TestAclsEndToEnd {
keyadminUgi.getUserName());
conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.ROLLOVER",
keyadminUgi.getUserName());
- conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GET", "");
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GET", " ");
conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GET_KEYS",
keyadminUgi.getUserName());
conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GET_METADATA",
hdfsUgi.getUserName());
- conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.SET_KEY_MATERIAL", "");
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.SET_KEY_MATERIAL", " ");
conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GENERATE_EEK",
hdfsUgi.getUserName());
- conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DECRYPT_EEK", "");
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DECRYPT_EEK", "*");
return conf;
}
@@ -478,6 +478,1042 @@ public class TestAclsEndToEnd {
}
/**
+ * Test that key creation is correctly governed by ACLs.
+ * @throws Exception thrown if setup fails
+ */
+ @Test
+ public void testCreateKey() throws Exception {
+ Configuration conf = new Configuration();
+
+ // Correct config with whitelist ACL
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.CREATE",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "MANAGEMENT",
+ realUgi.getUserName());
+
+ try {
+ setup(conf);
+
+ assertTrue("Exception during key creation with correct config"
+ + " using whitelist key ACLs", createKey(realUgi, KEY1, conf));
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Correct config with default ACL
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.CREATE",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + "MANAGEMENT",
+ realUgi.getUserName());
+
+ try {
+ setup(conf);
+
+ assertTrue("Exception during key creation with correct config"
+ + " using default key ACLs", createKey(realUgi, KEY2, conf));
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Denied because of blacklist
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.CREATE",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "blacklist.CREATE",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "MANAGEMENT",
+ realUgi.getUserName());
+
+ try {
+ setup(conf);
+
+ assertFalse("Allowed key creation with blacklist for CREATE",
+ createKey(realUgi, KEY3, conf));
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Missing KMS ACL
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.CREATE", " ");
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "MANAGEMENT",
+ realUgi.getUserName());
+
+ try {
+ setup(conf);
+
+ assertFalse("Allowed key creation without CREATE KMS ACL",
+ createKey(realUgi, KEY3, conf));
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Missing key ACL
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.CREATE",
+ realUgi.getUserName());
+
+ try {
+ setup(conf);
+
+ assertFalse("Allowed key creation without MANAGMENT key ACL",
+ createKey(realUgi, KEY3, conf));
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Denied because the key ACL set ignores the default ACL set for key3
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.CREATE",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + "MANAGEMENT",
+ realUgi.getUserName());
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + KEY3 + ".DECRYPT_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf);
+
+ assertFalse("Allowed key creation when default key ACL should have been"
+ + " overridden by key ACL", createKey(realUgi, KEY3, conf));
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Allowed because the default setting for KMS ACLs is fully permissive
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "MANAGEMENT",
+ realUgi.getUserName());
+
+ try {
+ setup(conf);
+
+ assertTrue("Exception during key creation with default KMS ACLs",
+ createKey(realUgi, KEY3, conf));
+ } finally {
+ teardown();
+ }
+ }
+
+ /**
+ * Test that zone creation is correctly governed by ACLs.
+ * @throws Exception thrown if setup fails
+ */
+ @Test
+ public void testCreateEncryptionZone() throws Exception {
+ Configuration conf = new Configuration();
+
+ // Create a test key
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.CREATE",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "MANAGEMENT",
+ realUgi.getUserName());
+
+ try {
+ setup(conf);
+
+ assertTrue("Exception during key creation",
+ createKey(realUgi, KEY1, conf));
+ } finally {
+ teardown();
+ }
+
+ // We tear everything down and then restart it with the ACLs we want to
+ // test so that there's no contamination from the ACLs needed for setup.
+ // To make that work, we have to tell the setup() method not to create a
+ // new KMS directory.
+ conf = new Configuration();
+
+ // Correct config with whitelist ACL
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GET_METADATA",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "READ",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "GENERATE_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false);
+
+ fs.mkdirs(ZONE1);
+
+ assertTrue("Exception during zone creation with correct config using"
+ + " whitelist key ACLs", createEncryptionZone(realUgi, KEY1, ZONE1));
+ } finally {
+ fs.delete(ZONE1, true);
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Correct config with default ACL
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GET_METADATA",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + "READ",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + "GENERATE_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false);
+
+ fs.mkdirs(ZONE2);
+
+ assertTrue("Exception during zone creation with correct config using"
+ + " default key ACLs", createEncryptionZone(realUgi, KEY1, ZONE2));
+ } finally {
+ fs.delete(ZONE2, true);
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Denied because the key ACL set ignores the default ACL set for key1
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GET_METADATA",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + "READ",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + "GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + KEY1 + ".DECRYPT_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false);
+
+ fs.mkdirs(ZONE3);
+
+ assertFalse("Allowed creation of zone when default key ACLs should have"
+ + " been overridden by key ACL",
+ createEncryptionZone(realUgi, KEY1, ZONE3));
+ } finally {
+ fs.delete(ZONE3, true);
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Correct config with blacklist
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GET_METADATA",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "blacklist.GET_METADATA",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "READ",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "GENERATE_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false);
+
+ fs.mkdirs(ZONE3);
+
+ assertFalse("Allowed zone creation of zone with blacklisted GET_METADATA",
+ createEncryptionZone(realUgi, KEY1, ZONE3));
+ } finally {
+ fs.delete(ZONE3, true);
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Correct config with blacklist
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GET_METADATA",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "blacklist.GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "READ",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "GENERATE_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false);
+
+ fs.mkdirs(ZONE3);
+
+ assertFalse("Allowed zone creation of zone with blacklisted GENERATE_EEK",
+ createEncryptionZone(realUgi, KEY1, ZONE3));
+ } finally {
+ fs.delete(ZONE3, true);
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Missing KMS ACL but works because defaults for KMS ACLs are fully
+ // permissive
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "READ",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "GENERATE_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false);
+
+ fs.mkdirs(ZONE3);
+
+ assertTrue("Exception during zone creation with default KMS ACLs",
+ createEncryptionZone(realUgi, KEY1, ZONE3));
+ } finally {
+ fs.delete(ZONE3, true);
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Missing GET_METADATA KMS ACL
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GET_METADATA", " ");
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "READ",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "GENERATE_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false);
+
+ fs.mkdirs(ZONE4);
+
+ assertFalse("Allowed zone creation without GET_METADATA KMS ACL",
+ createEncryptionZone(realUgi, KEY1, ZONE4));
+ } finally {
+ fs.delete(ZONE4, true);
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Missing GET_METADATA KMS ACL
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GET_METADATA",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GENERATE_EEK", " ");
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "READ",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "GENERATE_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false);
+
+ fs.mkdirs(ZONE4);
+
+ assertFalse("Allowed zone creation without GENERATE_EEK KMS ACL",
+ createEncryptionZone(realUgi, KEY1, ZONE4));
+ } finally {
+ fs.delete(ZONE4, true);
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Missing READ key ACL
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GET_METADATA",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "GENERATE_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false);
+
+ fs.mkdirs(ZONE4);
+
+ assertFalse("Allowed zone creation without READ ACL",
+ createEncryptionZone(realUgi, KEY1, ZONE4));
+ } finally {
+ fs.delete(ZONE4, true);
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Missing GENERATE_EEK key ACL
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GET_METADATA",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "READ",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false);
+
+ fs.mkdirs(ZONE4);
+
+ assertFalse("Allowed zone creation without GENERATE_EEK ACL",
+ createEncryptionZone(realUgi, KEY1, ZONE4));
+ } finally {
+ fs.delete(ZONE4, true);
+ teardown();
+ }
+ }
+
+ /**
+ * Test that in-zone file creation is correctly governed by ACLs.
+ * @throws Exception thrown if setup fails
+ */
+ @Test
+ public void testCreateFileInEncryptionZone() throws Exception {
+ Configuration conf = new Configuration();
+
+ // Create a test key
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.CREATE",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GET_METADATA",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "MANAGEMENT",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "READ",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "GENERATE_EEK",
+ realUgi.getUserName());
+
+ assertTrue(new File(kmsDir, "kms.keystore").length() == 0);
+
+ try {
+ setup(conf);
+
+ assertTrue("Exception during key creation",
+ createKey(realUgi, KEY1, conf));
+ fs.mkdirs(ZONE1);
+ assertTrue("Exception during zone creation",
+ createEncryptionZone(realUgi, KEY1, ZONE1));
+ fs.mkdirs(ZONE2);
+ assertTrue("Exception during zone creation",
+ createEncryptionZone(realUgi, KEY1, ZONE2));
+ fs.mkdirs(ZONE3);
+ assertTrue("Exception during zone creation",
+ createEncryptionZone(realUgi, KEY1, ZONE3));
+ fs.mkdirs(ZONE4);
+ assertTrue("Exception during zone creation",
+ createEncryptionZone(realUgi, KEY1, ZONE4));
+ } catch (Throwable ex) {
+ fs.delete(ZONE1, true);
+ fs.delete(ZONE2, true);
+ fs.delete(ZONE3, true);
+ fs.delete(ZONE4, true);
+
+ throw ex;
+ } finally {
+ teardown();
+ }
+
+ // We tear everything down and then restart it with the ACLs we want to
+ // test so that there's no contamination from the ACLs needed for setup.
+ // To make that work, we have to tell the setup() method not to create a
+ // new KMS directory or DFS dierctory.
+
+ conf = new Configuration();
+
+ // Correct config with whitelist ACLs
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DECRYPT_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "DECRYPT_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false, false);
+
+ assertTrue("Exception during file creation with correct config"
+ + " using whitelist ACL", createFile(realUgi, FILE1, TEXT));
+ } finally {
+ fs.delete(ZONE1, true);
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Correct config with default ACLs
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DECRYPT_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + "GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + "DECRYPT_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false, false);
+
+ assertTrue("Exception during file creation with correct config"
+ + " using whitelist ACL", createFile(realUgi, FILE2, TEXT));
+ } finally {
+ fs.delete(ZONE2, true);
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Denied because the key ACL set ignores the default ACL set for key1
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DECRYPT_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + "GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + "DECRYPT_EEK",
+ realUgi.getUserName());
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + KEY1 + ".READ",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false, false);
+
+ assertFalse("Allowed file creation when default key ACLs should have been"
+ + " overridden by key ACL", createFile(realUgi, FILE3, TEXT));
+ } catch (Exception ex) {
+ fs.delete(ZONE3, true);
+
+ throw ex;
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Denied by blacklist
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DECRYPT_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "blacklist.GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "DECRYPT_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false, false);
+
+ assertFalse("Allowed file creation with blacklist for GENERATE_EEK",
+ createFile(realUgi, FILE3, TEXT));
+ } catch (Exception ex) {
+ fs.delete(ZONE3, true);
+
+ throw ex;
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Denied by blacklist
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DECRYPT_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "blacklist.DECRYPT_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "DECRYPT_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false, false);
+
+ assertFalse("Allowed file creation with blacklist for DECRYPT_EEK",
+ createFile(realUgi, FILE3, TEXT));
+ } catch (Exception ex) {
+ fs.delete(ZONE3, true);
+
+ throw ex;
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Allowed because default KMS ACLs are fully permissive
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "DECRYPT_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false, false);
+
+ assertTrue("Exception during file creation with default KMS ACLs",
+ createFile(realUgi, FILE3, TEXT));
+ } catch (Exception ex) {
+ fs.delete(ZONE3, true);
+
+ throw ex;
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Denied because of missing GENERATE_EEK KMS ACL
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GENERATE_EEK", " ");
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DECRYPT_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "DECRYPT_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false, false);
+
+ assertFalse("Allowed file creation without GENERATE_EEK KMS ACL",
+ createFile(realUgi, FILE4, TEXT));
+ } catch (Exception ex) {
+ fs.delete(ZONE3, true);
+
+ throw ex;
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Denied because of missing DECRYPT_EEK KMS ACL
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DECRYPT_EEK", " ");
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "DECRYPT_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false, false);
+
+ assertFalse("Allowed file creation without DECRYPT_EEK KMS ACL",
+ createFile(realUgi, FILE3, TEXT));
+ } catch (Exception ex) {
+ fs.delete(ZONE3, true);
+
+ throw ex;
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Denied because of missing GENERATE_EEK key ACL
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DECRYPT_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "DECRYPT_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false, false);
+
+ assertFalse("Allowed file creation without GENERATE_EEK key ACL",
+ createFile(realUgi, FILE3, TEXT));
+ } catch (Exception ex) {
+ fs.delete(ZONE3, true);
+
+ throw ex;
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Denied because of missing DECRYPT_EEK key ACL
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DECRYPT_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "GENERATE_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false, false);
+
+ assertFalse("Allowed file creation without DECRYPT_EEK key ACL",
+ createFile(realUgi, FILE3, TEXT));
+ } catch (Exception ex) {
+ fs.delete(ZONE3, true);
+
+ throw ex;
+ } finally {
+ teardown();
+ }
+ }
+
+ /**
+ * Test that in-zone file read is correctly governed by ACLs.
+ * @throws Exception thrown if setup fails
+ */
+ @Test
+ public void testReadFileInEncryptionZone() throws Exception {
+ Configuration conf = new Configuration();
+
+ // Create a test key
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.CREATE",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GET_METADATA",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DECRYPT_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "MANAGEMENT",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "READ",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "GENERATE_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "DECRYPT_EEK",
+ realUgi.getUserName());
+
+ assertTrue(new File(kmsDir, "kms.keystore").length() == 0);
+
+ try {
+ setup(conf);
+
+ assertTrue("Exception during key creation",
+ createKey(realUgi, KEY1, conf));
+ fs.mkdirs(ZONE1);
+ assertTrue("Exception during zone creation",
+ createEncryptionZone(realUgi, KEY1, ZONE1));
+ assertTrue("Exception during file creation",
+ createFile(realUgi, FILE1, TEXT));
+ } catch (Throwable ex) {
+ fs.delete(ZONE1, true);
+
+ throw ex;
+ } finally {
+ teardown();
+ }
+
+ // We tear everything down and then restart it with the ACLs we want to
+ // test so that there's no contamination from the ACLs needed for setup.
+ // To make that work, we have to tell the setup() method not to create a
+ // new KMS directory or DFS dierctory.
+
+ conf = new Configuration();
+
+ // Correct config with whitelist ACLs
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DECRYPT_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "DECRYPT_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false, false);
+
+ assertTrue("Exception while reading file with correct config with"
+ + " whitelist ACLs", compareFile(realUgi, FILE1, TEXT));
+ } catch (Throwable ex) {
+ fs.delete(ZONE1, true);
+
+ throw ex;
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Correct config with default ACLs
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DECRYPT_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + "DECRYPT_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false, false);
+
+ assertTrue("Exception while reading file with correct config"
+ + " with default ACLs", compareFile(realUgi, FILE1, TEXT));
+ } catch (Throwable ex) {
+ fs.delete(ZONE1, true);
+
+ throw ex;
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Denied because the key ACL set ignores the default ACL set for key1
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DECRYPT_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + "DECRYPT_EEK",
+ realUgi.getUserName());
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + KEY1 + ".READ",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false, false);
+
+ assertFalse("Allowed file read when default key ACLs should have been"
+ + " overridden by key ACL", compareFile(realUgi, FILE1, TEXT));
+ } catch (Throwable ex) {
+ fs.delete(ZONE1, true);
+
+ throw ex;
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Denied by blacklist
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DECRYPT_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "blacklist.DECRYPT_EEK",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "DECRYPT_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false, false);
+
+ assertFalse("Allowed file read with blacklist for DECRYPT_EEK",
+ compareFile(realUgi, FILE1, TEXT));
+ } catch (Throwable ex) {
+ fs.delete(ZONE1, true);
+
+ throw ex;
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Allowed because default KMS ACLs are fully permissive
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "DECRYPT_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false, false);
+
+ assertTrue("Exception while reading file with default KMS ACLs",
+ compareFile(realUgi, FILE1, TEXT));
+ } catch (Throwable ex) {
+ fs.delete(ZONE1, true);
+
+ throw ex;
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Denied because of missing DECRYPT_EEK KMS ACL
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DECRYPT_EEK", " ");
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "DECRYPT_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false, false);
+
+ assertFalse("Allowed file read without DECRYPT_EEK KMS ACL",
+ compareFile(realUgi, FILE1, TEXT));
+ } catch (Throwable ex) {
+ fs.delete(ZONE1, true);
+
+ throw ex;
+ } finally {
+ teardown();
+ }
+
+ // Denied because of missing DECRYPT_EEK key ACL
+ conf = new Configuration();
+
+ try {
+ setup(conf, false, false);
+
+ assertFalse("Allowed file read without DECRYPT_EEK key ACL",
+ compareFile(realUgi, FILE1, TEXT));
+ } catch (Throwable ex) {
+ fs.delete(ZONE1, true);
+
+ throw ex;
+ } finally {
+ teardown();
+ }
+ }
+
+ /**
+ * Test that key deletion is correctly governed by ACLs.
+ * @throws Exception thrown if setup fails
+ */
+ @Test
+ public void testDeleteKey() throws Exception {
+ Configuration conf = new Configuration();
+
+ // Create a test key
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.CREATE",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "MANAGEMENT",
+ realUgi.getUserName());
+
+ try {
+ setup(conf);
+
+ assertTrue("Exception during key creation",
+ createKey(realUgi, KEY1, conf));
+ assertTrue("Exception during key creation",
+ createKey(realUgi, KEY2, conf));
+ assertTrue("Exception during key creation",
+ createKey(realUgi, KEY3, conf));
+ } finally {
+ teardown();
+ }
+
+ // We tear everything down and then restart it with the ACLs we want to
+ // test so that there's no contamination from the ACLs needed for setup.
+ // To make that work, we have to tell the setup() method not to create a
+ // new KMS directory.
+
+ conf = new Configuration();
+
+ // Correct config with whitelist ACL
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DELETE",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "MANAGEMENT",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false);
+
+ assertTrue("Exception during key deletion with correct config"
+ + " using whitelist key ACLs", deleteKey(realUgi, KEY1));
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Correct config with default ACL
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DELETE",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + "MANAGEMENT",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false);
+
+ assertTrue("Exception during key deletion with correct config"
+ + " using default key ACLs", deleteKey(realUgi, KEY2));
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Denied because of blacklist
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DELETE",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "blacklist.DELETE",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "MANAGEMENT",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false);
+
+ assertFalse("Allowed key deletion with blacklist for DELETE",
+ deleteKey(realUgi, KEY3));
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Missing KMS ACL
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DELETE", " ");
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "MANAGEMENT",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false);
+
+ assertFalse("Allowed key deletion without DELETE KMS ACL",
+ deleteKey(realUgi, KEY3));
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+
+ // Missing key ACL
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DELETE",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false);
+
+ assertFalse("Allowed key deletion without MANAGMENT key ACL",
+ deleteKey(realUgi, KEY3));
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Denied because the key ACL set ignores the default ACL set for key3
+ conf.set(KMSConfiguration.CONFIG_PREFIX + "acl.DELETE",
+ realUgi.getUserName());
+ conf.set(KMSConfiguration.DEFAULT_KEY_ACL_PREFIX + "MANAGEMENT",
+ realUgi.getUserName());
+ conf.set(KeyAuthorizationKeyProvider.KEY_ACL + KEY3 + ".DECRYPT_EEK",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false);
+
+ assertFalse("Allowed key deletion when default key ACL should have been"
+ + " overridden by key ACL", deleteKey(realUgi, KEY3));
+ } finally {
+ teardown();
+ }
+
+ conf = new Configuration();
+
+ // Allowed because the default setting for KMS ACLs is fully permissive
+ conf.set(KMSConfiguration.WHITELIST_KEY_ACL_PREFIX + "MANAGEMENT",
+ realUgi.getUserName());
+
+ try {
+ setup(conf, false);
+
+ assertTrue("Exception during key deletion with default KMS ACLs",
+ deleteKey(realUgi, KEY3));
+ } finally {
+ teardown();
+ }
+ }
+
+ /**
* Create a key as the specified user.
*
* @param ugi the target user
[39/50] hadoop git commit: HADOOP-12508. delete fails with exception
when lease is held on blob. Contributed by Gaurav Kanade.
Posted by ji...@apache.org.
HADOOP-12508. delete fails with exception when lease is held on blob. Contributed by Gaurav Kanade.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e7dcab1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e7dcab1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e7dcab1
Branch: refs/heads/HDFS-8966
Commit: 9e7dcab185abf2fdabb28f2799b9952b5664a4b0
Parents: 3ce0a65
Author: cnauroth <cn...@apache.org>
Authored: Mon Nov 2 10:21:39 2015 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Mon Nov 2 10:21:39 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../fs/azure/AzureNativeFileSystemStore.java | 32 +++++++-
.../hadoop/fs/azure/SelfRenewingLease.java | 5 +-
.../fs/azure/TestNativeAzureFileSystemLive.java | 86 ++++++++++++++++++++
4 files changed, 124 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e7dcab1/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index c8d60b0..1a9c93c 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1307,6 +1307,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12533. Introduce FileNotFoundException in WASB for read and seek API.
(Dushyanth via cnauroth)
+ HADOOP-12508. delete fails with exception when lease is held on blob.
+ (Gaurav Kanade via cnauroth)
+
OPTIMIZATIONS
HADOOP-12051. ProtobufRpcEngine.invoke() should use Exception.toString()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e7dcab1/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index 6412714..69ece4a 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -2370,7 +2370,37 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
@Override
public void delete(String key) throws IOException {
- delete(key, null);
+ try {
+ delete(key, null);
+ } catch (IOException e) {
+ Throwable t = e.getCause();
+ if(t != null && t instanceof StorageException) {
+ StorageException se = (StorageException) t;
+ if(se.getErrorCode().equals(("LeaseIdMissing"))){
+ SelfRenewingLease lease = null;
+ try {
+ lease = acquireLease(key);
+ delete(key, lease);
+ } catch (AzureException e3) {
+ LOG.warn("Got unexpected exception trying to acquire lease on "
+ + key + "." + e3.getMessage());
+ throw e3;
+ } finally {
+ try {
+ if(lease != null){
+ lease.free();
+ }
+ } catch (Exception e4){
+ LOG.error("Unable to free lease on " + key, e4);
+ }
+ }
+ } else {
+ throw e;
+ }
+ } else {
+ throw e;
+ }
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e7dcab1/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java
index 06f32ce..900d730 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/SelfRenewingLease.java
@@ -22,6 +22,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.azure.StorageInterface.CloudBlobWrapper;
+import com.google.common.annotations.VisibleForTesting;
+
import com.microsoft.azure.storage.AccessCondition;
import com.microsoft.azure.storage.StorageException;
import com.microsoft.azure.storage.blob.CloudBlob;
@@ -61,7 +63,8 @@ public class SelfRenewingLease {
// Time to wait to retry getting the lease in milliseconds
- private static final int LEASE_ACQUIRE_RETRY_INTERVAL = 2000;
+ @VisibleForTesting
+ static final int LEASE_ACQUIRE_RETRY_INTERVAL = 2000;
public SelfRenewingLease(CloudBlobWrapper blobWrapper)
throws StorageException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e7dcab1/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java
index b033460..721cb5f 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemLive.java
@@ -21,10 +21,16 @@ package org.apache.hadoop.fs.azure;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+
+import java.util.concurrent.CountDownLatch;
+
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+
import org.junit.Test;
+import com.microsoft.azure.storage.StorageException;
+
/*
* Tests the Native Azure file system (WASB) against an actual blob store if
* provided in the environment.
@@ -38,6 +44,86 @@ public class TestNativeAzureFileSystemLive extends
}
/**
+ * Tests fs.delete() function to delete a blob when another blob is holding a
+ * lease on it. Delete if called without a lease should fail if another process
+ * is holding a lease and throw appropriate exception
+ * This is a scenario that would happen in HMaster startup when it tries to
+ * clean up the temp dirs while the HMaster process which was killed earlier
+ * held lease on the blob when doing some DDL operation
+ */
+ @Test
+ public void testDeleteThrowsExceptionWithLeaseExistsErrorMessage()
+ throws Exception {
+ LOG.info("Starting test");
+ final String FILE_KEY = "fileWithLease";
+ // Create the file
+ Path path = new Path(FILE_KEY);
+ fs.create(path);
+ assertTrue(fs.exists(path));
+ NativeAzureFileSystem nfs = (NativeAzureFileSystem)fs;
+ final String fullKey = nfs.pathToKey(nfs.makeAbsolute(path));
+ final AzureNativeFileSystemStore store = nfs.getStore();
+
+ // Acquire the lease on the file in a background thread
+ final CountDownLatch leaseAttemptComplete = new CountDownLatch(1);
+ final CountDownLatch beginningDeleteAttempt = new CountDownLatch(1);
+ Thread t = new Thread() {
+ @Override
+ public void run() {
+ // Acquire the lease and then signal the main test thread.
+ SelfRenewingLease lease = null;
+ try {
+ lease = store.acquireLease(fullKey);
+ LOG.info("Lease acquired: " + lease.getLeaseID());
+ } catch (AzureException e) {
+ LOG.warn("Lease acqusition thread unable to acquire lease", e);
+ } finally {
+ leaseAttemptComplete.countDown();
+ }
+
+ // Wait for the main test thread to signal it will attempt the delete.
+ try {
+ beginningDeleteAttempt.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+
+ // Keep holding the lease past the lease acquisition retry interval, so
+ // the test covers the case of delete retrying to acquire the lease.
+ try {
+ Thread.sleep(SelfRenewingLease.LEASE_ACQUIRE_RETRY_INTERVAL * 3);
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ }
+
+ try {
+ if (lease != null){
+ LOG.info("Freeing lease");
+ lease.free();
+ }
+ } catch (StorageException se) {
+ LOG.warn("Unable to free lease.", se);
+ }
+ }
+ };
+
+ // Start the background thread and wait for it to signal the lease is held.
+ t.start();
+ try {
+ leaseAttemptComplete.await();
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ }
+
+ // Try to delete the same file
+ beginningDeleteAttempt.countDown();
+ store.delete(fullKey);
+
+ // At this point file SHOULD BE DELETED
+ assertFalse(fs.exists(path));
+ }
+
+ /**
* Check that isPageBlobKey works as expected. This assumes that
* in the test configuration, the list of supported page blob directories
* only includes "pageBlobs". That's why this test is made specific
[05/50] hadoop git commit: YARN-4288. Fixed RMProxy to retry on
IOException from local host. Contributed by Junping Du
Posted by ji...@apache.org.
YARN-4288. Fixed RMProxy to retry on IOException from local host. Contributed by Junping Du
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c4169996
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c4169996
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c4169996
Branch: refs/heads/HDFS-8966
Commit: c41699965e78ce5e87669d17923ab84e494c4188
Parents: 5eca6de
Author: Jian He <ji...@apache.org>
Authored: Thu Oct 29 00:00:48 2015 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Oct 29 00:00:48 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/io/retry/RetryPolicies.java | 44 +++++++++++++++++++-
.../apache/hadoop/io/retry/TestRetryProxy.java | 27 +++++++++++-
.../io/retry/UnreliableImplementation.java | 17 ++++++++
.../hadoop/io/retry/UnreliableInterface.java | 3 ++
hadoop-yarn-project/CHANGES.txt | 3 ++
.../org/apache/hadoop/yarn/client/RMProxy.java | 6 ++-
6 files changed, 94 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4169996/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
index f978ae7..171d52a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
@@ -139,7 +139,17 @@ public class RetryPolicies {
Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap) {
return new RemoteExceptionDependentRetry(defaultPolicy, exceptionToPolicyMap);
}
-
+
+ /**
+ * A retry policy for exceptions other than RemoteException.
+ */
+ public static final RetryPolicy retryOtherThanRemoteException(
+ RetryPolicy defaultPolicy,
+ Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap) {
+ return new OtherThanRemoteExceptionDependentRetry(defaultPolicy,
+ exceptionToPolicyMap);
+ }
+
public static final RetryPolicy failoverOnNetworkException(int maxFailovers) {
return failoverOnNetworkException(TRY_ONCE_THEN_FAIL, maxFailovers);
}
@@ -489,7 +499,37 @@ public class RetryPolicies {
return policy.shouldRetry(e, retries, failovers, isIdempotentOrAtMostOnce);
}
}
-
+
+ static class OtherThanRemoteExceptionDependentRetry implements RetryPolicy {
+
+ private RetryPolicy defaultPolicy;
+ private Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap;
+
+ public OtherThanRemoteExceptionDependentRetry(RetryPolicy defaultPolicy,
+ Map<Class<? extends Exception>,
+ RetryPolicy> exceptionToPolicyMap) {
+ this.defaultPolicy = defaultPolicy;
+ this.exceptionToPolicyMap = exceptionToPolicyMap;
+ }
+
+ @Override
+ public RetryAction shouldRetry(Exception e, int retries, int failovers,
+ boolean isIdempotentOrAtMostOnce) throws Exception {
+ RetryPolicy policy = null;
+ // ignore Remote Exception
+ if (e instanceof RemoteException) {
+ // do nothing
+ } else {
+ policy = exceptionToPolicyMap.get(e.getClass());
+ }
+ if (policy == null) {
+ policy = defaultPolicy;
+ }
+ return policy.shouldRetry(
+ e, retries, failovers, isIdempotentOrAtMostOnce);
+ }
+ }
+
static class ExponentialBackoffRetry extends RetryLimited {
public ExponentialBackoffRetry(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4169996/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
index 81f3a9b..35a45b4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
@@ -22,6 +22,7 @@ import static org.apache.hadoop.io.retry.RetryPolicies.RETRY_FOREVER;
import static org.apache.hadoop.io.retry.RetryPolicies.TRY_ONCE_THEN_FAIL;
import static org.apache.hadoop.io.retry.RetryPolicies.retryByException;
import static org.apache.hadoop.io.retry.RetryPolicies.retryByRemoteException;
+import static org.apache.hadoop.io.retry.RetryPolicies.retryOtherThanRemoteException;
import static org.apache.hadoop.io.retry.RetryPolicies.retryUpToMaximumCountWithFixedSleep;
import static org.apache.hadoop.io.retry.RetryPolicies.retryUpToMaximumCountWithProportionalSleep;
import static org.apache.hadoop.io.retry.RetryPolicies.retryUpToMaximumTimeWithFixedSleep;
@@ -29,6 +30,7 @@ import static org.apache.hadoop.io.retry.RetryPolicies.retryForeverWithFixedSlee
import static org.apache.hadoop.io.retry.RetryPolicies.exponentialBackoffRetry;
import static org.junit.Assert.*;
+import java.io.IOException;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.Callable;
@@ -213,8 +215,29 @@ public class TestRetryProxy {
} catch (RemoteException e) {
// expected
}
- }
-
+ }
+
+ @Test
+ public void testRetryOtherThanRemoteException() throws Throwable {
+ Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
+ Collections.<Class<? extends Exception>, RetryPolicy>singletonMap(
+ IOException.class, RETRY_FOREVER);
+
+ UnreliableInterface unreliable = (UnreliableInterface)
+ RetryProxy.create(UnreliableInterface.class, unreliableImpl,
+ retryOtherThanRemoteException(TRY_ONCE_THEN_FAIL,
+ exceptionToPolicyMap));
+ // should retry with local IOException.
+ unreliable.failsOnceWithIOException();
+ try {
+ // won't get retry on remote exception
+ unreliable.failsOnceWithRemoteException();
+ fail("Should fail");
+ } catch (RemoteException e) {
+ // expected
+ }
+ }
+
@Test
public void testRetryInterruptible() throws Throwable {
final UnreliableInterface unreliable = (UnreliableInterface)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4169996/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java
index ce9c16e..9387772 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java
@@ -26,6 +26,8 @@ class UnreliableImplementation implements UnreliableInterface {
private int failsOnceInvocationCount,
failsOnceWithValueInvocationCount,
+ failsOnceIOExceptionInvocationCount,
+ failsOnceRemoteExceptionInvocationCount,
failsTenTimesInvocationCount,
succeedsOnceThenFailsCount,
succeedsOnceThenFailsIdempotentCount,
@@ -90,6 +92,21 @@ class UnreliableImplementation implements UnreliableInterface {
}
@Override
+ public void failsOnceWithIOException() throws IOException {
+ if (failsOnceIOExceptionInvocationCount++ == 0) {
+ throw new IOException("test exception for failsOnceWithIOException");
+ }
+ }
+
+ @Override
+ public void failsOnceWithRemoteException() throws RemoteException {
+ if (failsOnceRemoteExceptionInvocationCount++ == 0) {
+ throw new RemoteException(IOException.class.getName(),
+ "test exception for failsOnceWithRemoteException");
+ }
+ }
+
+ @Override
public void failsTenTimesThenSucceeds() throws UnreliableException {
if (failsTenTimesInvocationCount++ < 10) {
throw new UnreliableException();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4169996/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java
index 3fbe11a..6c9c153 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java
@@ -54,6 +54,9 @@ public interface UnreliableInterface {
void alwaysFailsWithFatalException() throws FatalException;
void alwaysFailsWithRemoteFatalException() throws RemoteException;
+ void failsOnceWithIOException() throws IOException;
+ void failsOnceWithRemoteException() throws RemoteException;
+
void failsOnceThenSucceeds() throws UnreliableException;
boolean failsOnceThenSucceedsWithReturnValue() throws UnreliableException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4169996/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 57cdb15..8dc1fd0 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1034,6 +1034,9 @@ Release 2.8.0 - UNRELEASED
YARN-4130. Duplicate declaration of ApplicationId in RMAppManager#submitApplication method.
(Kai Sasaki via rohithsharmaks)
+ YARN-4288. Fixed RMProxy to retry on IOException from local host.
+ (Junping Du via jianhe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4169996/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
index be08f2f..23e1691 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
@@ -250,8 +250,10 @@ public class RMProxy<T> {
exceptionToPolicyMap.put(ConnectTimeoutException.class, retryPolicy);
exceptionToPolicyMap.put(RetriableException.class, retryPolicy);
exceptionToPolicyMap.put(SocketException.class, retryPolicy);
-
- return RetryPolicies.retryByException(
+ // YARN-4288: local IOException is also possible.
+ exceptionToPolicyMap.put(IOException.class, retryPolicy);
+ // Not retry on remote IO exception.
+ return RetryPolicies.retryOtherThanRemoteException(
RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
}
}
[04/50] hadoop git commit: YARN-4130. Duplicate declaration of
ApplicationId in RMAppManager#submitApplication method. (Kai Sasaki via
rohithsharmaks)
Posted by ji...@apache.org.
YARN-4130. Duplicate declaration of ApplicationId in RMAppManager#submitApplication method. (Kai Sasaki via rohithsharmaks)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/656c8f95
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/656c8f95
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/656c8f95
Branch: refs/heads/HDFS-8966
Commit: 656c8f952738f469766ede729dc0f25a3ce4214e
Parents: 07ecdb8
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Thu Oct 29 12:22:44 2015 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Thu Oct 29 12:22:44 2015 +0530
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/server/resourcemanager/RMAppManager.java | 8 ++++----
2 files changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/656c8f95/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1533e2b..57cdb15 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1031,6 +1031,9 @@ Release 2.8.0 - UNRELEASED
YARN-4251. TestAMRMClientOnRMRestart#testAMRMClientOnAMRMTokenRollOverOnRMRestart
is failing. (Brahma Reddy Battula via ozawa)
+ YARN-4130. Duplicate declaration of ApplicationId in RMAppManager#submitApplication method.
+ (Kai Sasaki via rohithsharmaks)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/656c8f95/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 0b7083c..711a7a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -281,14 +281,14 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
RMAppImpl application =
createAndPopulateNewRMApp(submissionContext, submitTime, user, false);
- ApplicationId appId = submissionContext.getApplicationId();
Credentials credentials = null;
try {
credentials = parseCredentials(submissionContext);
if (UserGroupInformation.isSecurityEnabled()) {
- this.rmContext.getDelegationTokenRenewer().addApplicationAsync(appId,
- credentials, submissionContext.getCancelTokensWhenComplete(),
- application.getUser());
+ this.rmContext.getDelegationTokenRenewer()
+ .addApplicationAsync(applicationId, credentials,
+ submissionContext.getCancelTokensWhenComplete(),
+ application.getUser());
} else {
// Dispatcher is not yet started at this time, so these START events
// enqueued should be guaranteed to be first processed when dispatcher
[34/50] hadoop git commit: Fix CHANGES.txt
Posted by ji...@apache.org.
Fix CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2529464f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2529464f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2529464f
Branch: refs/heads/HDFS-8966
Commit: 2529464f0841732792343d515cd1be1dccb3c453
Parents: 6e4f8a4
Author: Kihwal Lee <ki...@apache.org>
Authored: Mon Nov 2 09:09:33 2015 -0600
Committer: Kihwal Lee <ki...@apache.org>
Committed: Mon Nov 2 09:09:33 2015 -0600
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2529464f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8e6634a..0bbc60d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1529,9 +1529,6 @@ Release 2.8.0 - UNRELEASED
HDFS-9110. Use Files.walkFileTree in NNUpgradeUtil#doPreUpgrade for
better efficiency. (Charlie Helin via wang)
- HDFS-9221. HdfsServerConstants#ReplicaState#getState should avoid calling
- values() since it creates a temporary array. (Staffan Friberg via yliu)
-
HDFS-8988. Use LightWeightHashSet instead of LightWeightLinkedSet in
BlockManager#excessReplicateMap. (yliu)
@@ -2220,6 +2217,9 @@ Release 2.7.2 - UNRELEASED
HDFS-8099. Change "DFSInputStream has been closed already" message to
debug log level (Charles Lamb via Colin P. McCabe)
+ HDFS-9221. HdfsServerConstants#ReplicaState#getState should avoid calling
+ values() since it creates a temporary array. (Staffan Friberg via yliu)
+
OPTIMIZATIONS
HDFS-8722. Optimize datanode writes for small writes and flushes (kihwal)
[16/50] hadoop git commit: Move YARN-3580 in CHANGES.txt from 2.8 to
2.7.2.
Posted by ji...@apache.org.
Move YARN-3580 in CHANGES.txt from 2.8 to 2.7.2.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d2e01f4e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d2e01f4e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d2e01f4e
Branch: refs/heads/HDFS-8966
Commit: d2e01f4ed87c3c41156ec9a68855f923f8c0adf9
Parents: 7412ff4
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Fri Oct 30 15:49:06 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Fri Oct 30 15:49:06 2015 +0900
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2e01f4e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index d0fa27d..78c18d5 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -689,9 +689,6 @@ Release 2.8.0 - UNRELEASED
YARN-3582. NPE in WebAppProxyServlet. (jian he via xgong)
- YARN-3580. [JDK8] TestClientRMService.testGetLabelsToNodes fails. (Robert Kanter
- via junping_du)
-
YARN-3577. Misspelling of threshold in log4j.properties for tests.
(Brahma Reddy Battula via aajisaka)
@@ -1147,6 +1144,9 @@ Release 2.7.2 - UNRELEASED
YARN-4313. Race condition in MiniMRYarnCluster when getting history server
address. (Jian He via xgong)
+ YARN-3580. [JDK8] TestClientRMService.testGetLabelsToNodes fails. (Robert Kanter
+ via junping_du)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
[22/50] hadoop git commit: Creating 2.6.3 entries in CHANGES.txt
files.
Posted by ji...@apache.org.
Creating 2.6.3 entries in CHANGES.txt files.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eadf7b30
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eadf7b30
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eadf7b30
Branch: refs/heads/HDFS-8966
Commit: eadf7b3096cb010eb7f0afd9afd4ae0d67b2645f
Parents: 3c0204a
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Oct 30 14:50:50 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Oct 30 14:50:50 2015 +0000
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 12 ++++++++++++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 12 ++++++++++++
hadoop-mapreduce-project/CHANGES.txt | 12 ++++++++++++
hadoop-yarn-project/CHANGES.txt | 12 ++++++++++++
4 files changed, 48 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eadf7b30/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 6b33a2c..ddd0796 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -2156,6 +2156,18 @@ Release 2.7.0 - 2015-04-20
HADOOP-11837. AuthenticationFilter should destroy SignerSecretProvider in
Tomcat deployments. (Bowen Zhang via wheat9)
+Release 2.6.3 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Release 2.6.2 - 2015-10-21
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eadf7b30/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index c5846b3..17df171 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -3298,6 +3298,18 @@ Release 2.7.0 - 2015-04-20
HDFS-7700. Document quota support for storage types. (Xiaoyu Yao via
Arpit Agarwal)
+Release 2.6.3 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Release 2.6.2 - 2015-10-21
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eadf7b30/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index af5c3f6..8594e1e 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -888,6 +888,18 @@ Release 2.7.0 - 2015-04-20
MAPREDUCE-6285. ClientServiceDelegate should not retry upon
AuthenticationException. (Jonathan Eagles via ozawa)
+Release 2.6.3 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Release 2.6.2 - 2015-10-21
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eadf7b30/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 874397d..cc8f5f3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1875,6 +1875,18 @@ Release 2.7.0 - 2015-04-20
YARN-3466. Fix RM nodes web page to sort by node HTTP-address, #containers
and node-label column (Jason Lowe via wangda)
+Release 2.6.3 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Release 2.6.2 - 2015-10-21
INCOMPATIBLE CHANGES
[02/50] hadoop git commit: HDFS-9044. Give Priority to FavouredNodes ,
before selecting nodes from FavouredNode's Node Group (Contributed by
J.Andreina)
Posted by ji...@apache.org.
HDFS-9044. Give Priority to FavouredNodes , before selecting nodes from FavouredNode's Node Group (Contributed by J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/588baab1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/588baab1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/588baab1
Branch: refs/heads/HDFS-8966
Commit: 588baab160e7c328dca1c45cf3541e79218406e8
Parents: 7d2d16f
Author: Vinayakumar B <vi...@apache.org>
Authored: Thu Oct 29 11:44:01 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu Oct 29 11:44:01 2015 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../BlockPlacementPolicyDefault.java | 40 ++++++----
.../BlockPlacementPolicyWithNodeGroup.java | 78 ++++++++++++++++++--
.../TestReplicationPolicyWithNodeGroup.java | 58 +++++++++++++++
4 files changed, 157 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/588baab1/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f8f7452..aa26088 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -2187,6 +2187,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9297. Decomissioned capacity should not be considered for
configured/used capacity (Contributed by Kuhu Shukla)
+ HDFS-9044. Give Priority to FavouredNodes , before selecting
+ nodes from FavouredNode's Node Group (J.Andreina via vinayakumarb)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/588baab1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index ad1a739..d9b8d60 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -138,20 +138,9 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
numOfReplicas = maxNodesAndReplicas[0];
int maxNodesPerRack = maxNodesAndReplicas[1];
- for (int i = 0; i < favoredNodes.size() && results.size() < numOfReplicas; i++) {
- DatanodeDescriptor favoredNode = favoredNodes.get(i);
- // Choose a single node which is local to favoredNode.
- // 'results' is updated within chooseLocalNode
- final DatanodeStorageInfo target = chooseLocalStorage(favoredNode,
- favoriteAndExcludedNodes, blocksize, maxNodesPerRack,
- results, avoidStaleNodes, storageTypes, false);
- if (target == null) {
- LOG.warn("Could not find a target for file " + src
- + " with favored node " + favoredNode);
- continue;
- }
- favoriteAndExcludedNodes.add(target.getDatanodeDescriptor());
- }
+ chooseFavouredNodes(src, numOfReplicas, favoredNodes,
+ favoriteAndExcludedNodes, blocksize, maxNodesPerRack, results,
+ avoidStaleNodes, storageTypes);
if (results.size() < numOfReplicas) {
// Not enough favored nodes, choose other nodes.
@@ -177,6 +166,29 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
}
}
+ protected void chooseFavouredNodes(String src, int numOfReplicas,
+ List<DatanodeDescriptor> favoredNodes,
+ Set<Node> favoriteAndExcludedNodes, long blocksize, int maxNodesPerRack,
+ List<DatanodeStorageInfo> results, boolean avoidStaleNodes,
+ EnumMap<StorageType, Integer> storageTypes)
+ throws NotEnoughReplicasException {
+ for (int i = 0; i < favoredNodes.size() && results.size() < numOfReplicas;
+ i++) {
+ DatanodeDescriptor favoredNode = favoredNodes.get(i);
+ // Choose a single node which is local to favoredNode.
+ // 'results' is updated within chooseLocalNode
+ final DatanodeStorageInfo target =
+ chooseLocalStorage(favoredNode, favoriteAndExcludedNodes, blocksize,
+ maxNodesPerRack, results, avoidStaleNodes, storageTypes, false);
+ if (target == null) {
+ LOG.warn("Could not find a target for file " + src
+ + " with favored node " + favoredNode);
+ continue;
+ }
+ favoriteAndExcludedNodes.add(target.getDatanodeDescriptor());
+ }
+ }
+
/** This is the implementation. */
private DatanodeStorageInfo[] chooseTarget(int numOfReplicas,
Node writer,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/588baab1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java
index 89f47ad..187d8d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyWithNodeGroup.java
@@ -54,16 +54,79 @@ public class BlockPlacementPolicyWithNodeGroup extends BlockPlacementPolicyDefau
super.initialize(conf, stats, clusterMap, host2datanodeMap);
}
- /** choose local node of localMachine as the target.
- * if localMachine is not available, choose a node on the same nodegroup or
- * rack instead.
+ /**
+ * choose all good favored nodes as target.
+ * If no enough targets, then choose one replica from
+ * each bad favored node's node group.
+ * @throws NotEnoughReplicasException
+ */
+ @Override
+ protected void chooseFavouredNodes(String src, int numOfReplicas,
+ List<DatanodeDescriptor> favoredNodes,
+ Set<Node> favoriteAndExcludedNodes, long blocksize,
+ int maxNodesPerRack, List<DatanodeStorageInfo> results,
+ boolean avoidStaleNodes, EnumMap<StorageType, Integer> storageTypes)
+ throws NotEnoughReplicasException {
+ super.chooseFavouredNodes(src, numOfReplicas, favoredNodes,
+ favoriteAndExcludedNodes, blocksize, maxNodesPerRack, results,
+ avoidStaleNodes, storageTypes);
+ if (results.size() < numOfReplicas) {
+ // Not enough replicas, choose from unselected Favorednode's Nodegroup
+ for (int i = 0;
+ i < favoredNodes.size() && results.size() < numOfReplicas; i++) {
+ DatanodeDescriptor favoredNode = favoredNodes.get(i);
+ boolean chosenNode =
+ isNodeChosen(results, favoredNode);
+ if (chosenNode) {
+ continue;
+ }
+ NetworkTopologyWithNodeGroup clusterMapNodeGroup =
+ (NetworkTopologyWithNodeGroup) clusterMap;
+ // try a node on FavouredNode's node group
+ DatanodeStorageInfo target = null;
+ String scope =
+ clusterMapNodeGroup.getNodeGroup(favoredNode.getNetworkLocation());
+ try {
+ target =
+ chooseRandom(scope, favoriteAndExcludedNodes, blocksize,
+ maxNodesPerRack, results, avoidStaleNodes, storageTypes);
+ } catch (NotEnoughReplicasException e) {
+ // catch Exception and continue with other favored nodes
+ continue;
+ }
+ if (target == null) {
+ LOG.warn("Could not find a target for file "
+ + src + " within nodegroup of favored node " + favoredNode);
+ continue;
+ }
+ favoriteAndExcludedNodes.add(target.getDatanodeDescriptor());
+ }
+ }
+ }
+
+ private boolean isNodeChosen(
+ List<DatanodeStorageInfo> results, DatanodeDescriptor favoredNode) {
+ boolean chosenNode = false;
+ for (int j = 0; j < results.size(); j++) {
+ if (results.get(j).getDatanodeDescriptor().equals(favoredNode)) {
+ chosenNode = true;
+ break;
+ }
+ }
+ return chosenNode;
+ }
+
+ /** choose local node of <i>localMachine</i> as the target.
+ * If localMachine is not available, will fallback to nodegroup/rack
+ * when flag <i>fallbackToNodeGroupAndLocalRack</i> is set.
* @return the chosen node
*/
@Override
protected DatanodeStorageInfo chooseLocalStorage(Node localMachine,
Set<Node> excludedNodes, long blocksize, int maxNodesPerRack,
List<DatanodeStorageInfo> results, boolean avoidStaleNodes,
- EnumMap<StorageType, Integer> storageTypes, boolean fallbackToLocalRack)
+ EnumMap<StorageType, Integer> storageTypes,
+ boolean fallbackToNodeGroupAndLocalRack)
throws NotEnoughReplicasException {
DatanodeStorageInfo localStorage = chooseLocalStorage(localMachine,
excludedNodes, blocksize, maxNodesPerRack, results,
@@ -72,6 +135,9 @@ public class BlockPlacementPolicyWithNodeGroup extends BlockPlacementPolicyDefau
return localStorage;
}
+ if (!fallbackToNodeGroupAndLocalRack) {
+ return null;
+ }
// try a node on local node group
DatanodeStorageInfo chosenStorage = chooseLocalNodeGroup(
(NetworkTopologyWithNodeGroup)clusterMap, localMachine, excludedNodes,
@@ -79,10 +145,6 @@ public class BlockPlacementPolicyWithNodeGroup extends BlockPlacementPolicyDefau
if (chosenStorage != null) {
return chosenStorage;
}
-
- if (!fallbackToLocalRack) {
- return null;
- }
// try a node on local rack
return chooseLocalRack(localMachine, excludedNodes,
blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/588baab1/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
index 528021d..0ff7770 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
@@ -162,6 +162,16 @@ public class TestReplicationPolicyWithNodeGroup extends BaseReplicationPolicyTes
return cluster.isOnSameNodeGroup(left, right.getDatanodeDescriptor());
}
+ private DatanodeStorageInfo[] chooseTarget(
+ int numOfReplicas,
+ DatanodeDescriptor writer,
+ Set<Node> excludedNodes,
+ List<DatanodeDescriptor> favoredNodes) {
+ return replicator.chooseTarget(filename, numOfReplicas, writer,
+ excludedNodes, BLOCK_SIZE, favoredNodes,
+ TestBlockStoragePolicy.DEFAULT_STORAGE_POLICY);
+ }
+
/**
* In this testcase, client is dataNodes[0]. So the 1st replica should be
* placed on dataNodes[0], the 2nd replica should be placed on
@@ -723,4 +733,52 @@ public class TestReplicationPolicyWithNodeGroup extends BaseReplicationPolicyTes
assertTrue(excludedNodes.contains(dataNodesForDependencies[i]));
}
}
+
+ /**
+ * In this testcase, favored node is dataNodes[6].
+ * 1st replica should be placed on favored node.
+ * @throws Exception
+ */
+ @Test
+ public void testChooseTargetAsFavouredNodes() throws Exception {
+ DatanodeStorageInfo[] targets;
+ List<DatanodeDescriptor> favoredNodes =
+ new ArrayList<DatanodeDescriptor>();
+ favoredNodes.add(dataNodes[6]);
+ favoredNodes.add(dataNodes[0]);
+ favoredNodes.add(dataNodes[1]);
+ targets = chooseTarget(1, dataNodes[7], null, favoredNodes);
+ assertEquals(targets.length, 1);
+ assertTrue(favoredNodes.contains(targets[0].getDatanodeDescriptor()));
+ }
+
+ /**
+ * In this testcase, passed 2 favored nodes
+ * dataNodes[0](Good Node), dataNodes[3](Bad node).
+ * 1st replica should be placed on good favored node dataNodes[0].
+ * 2nd replica should be on bad favored node's nodegroup dataNodes[4].
+ * @throws Exception
+ */
+ @Test
+ public void testChooseFavoredNodesNodeGroup() throws Exception {
+ updateHeartbeatWithUsage(dataNodes[3],
+ 2* HdfsServerConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
+ (HdfsServerConstants.MIN_BLOCKS_FOR_WRITE-1)*BLOCK_SIZE, 0L,
+ 0L, 0L, 0, 0); // no space
+
+ DatanodeStorageInfo[] targets;
+ List<DatanodeDescriptor> expectedTargets =
+ new ArrayList<DatanodeDescriptor>();
+ expectedTargets.add(dataNodes[0]);
+ expectedTargets.add(dataNodes[4]);
+ List<DatanodeDescriptor> favouredNodes =
+ new ArrayList<DatanodeDescriptor>();
+ favouredNodes.add(dataNodes[3]);
+ favouredNodes.add(dataNodes[0]);
+ targets = chooseTarget(2, dataNodes[7], null, favouredNodes);
+ assertTrue("1st Replica is incorrect",
+ expectedTargets.contains(targets[0].getDatanodeDescriptor()));
+ assertTrue("2nd Replica is incorrect",
+ expectedTargets.contains(targets[1].getDatanodeDescriptor()));
+ }
}
[03/50] hadoop git commit: HDFS-9261. Erasure Coding: Skip encoding
the data cells if all the parity data streamers are failed for the current
block group. (Rakesh R via umamahesh)
Posted by ji...@apache.org.
HDFS-9261. Erasure Coding: Skip encoding the data cells if all the parity data streamers are failed for the current block group. (Rakesh R via umamahesh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/07ecdb87
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/07ecdb87
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/07ecdb87
Branch: refs/heads/HDFS-8966
Commit: 07ecdb877dee0aa76a3269e37ac4783a55ab6bf6
Parents: 588baab
Author: Uma Mahesh <um...@apache.org>
Authored: Wed Oct 28 23:33:11 2015 -0700
Committer: Uma Mahesh <um...@apache.org>
Committed: Wed Oct 28 23:33:11 2015 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/DFSStripedOutputStream.java | 18 ++++++++++++++++++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
2 files changed, 21 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07ecdb87/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index 884fa42..c595026 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -844,6 +844,11 @@ public class DFSStripedOutputStream extends DFSOutputStream {
void writeParityCells() throws IOException {
final ByteBuffer[] buffers = cellBuffers.getBuffers();
+ // Skips encoding and writing parity cells if there are no healthy parity
+ // data streamers
+ if (!checkAnyParityStreamerIsHealthy()) {
+ return;
+ }
//encode the data cells
encode(encoder, numDataBlocks, buffers);
for (int i = numDataBlocks; i < numAllBlocks; i++) {
@@ -852,6 +857,19 @@ public class DFSStripedOutputStream extends DFSOutputStream {
cellBuffers.clear();
}
+ private boolean checkAnyParityStreamerIsHealthy() {
+ for (int i = numDataBlocks; i < numAllBlocks; i++) {
+ if (streamers.get(i).isHealthy()) {
+ return true;
+ }
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skips encoding and writing parity cells as there are "
+ + "no healthy parity data streamers: " + streamers);
+ }
+ return false;
+ }
+
void writeParity(int index, ByteBuffer buffer, byte[] checksumBuf)
throws IOException {
final StripedDataStreamer current = setCurrentStreamer(index);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07ecdb87/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index aa26088..a87fe76 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -184,6 +184,9 @@ Trunk (Unreleased)
HDFS-9070. Allow fsck display pending replica location information for
being-written blocks. (GAO Rui via jing9)
+ HDFS-9261. Erasure Coding: Skip encoding the data cells if all the parity data
+ streamers are failed for the current block group. (Rakesh R via umamahesh)
+
OPTIMIZATIONS
BUG FIXES
[42/50] hadoop git commit: HDFS-9275. Wait previous ErasureCodingWork
to finish before schedule another one. (Walter Su via yliu)
Posted by ji...@apache.org.
HDFS-9275. Wait previous ErasureCodingWork to finish before schedule another one. (Walter Su via yliu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ba2b98d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ba2b98d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ba2b98d
Branch: refs/heads/HDFS-8966
Commit: 5ba2b98d0fe29603e136fc43a14f853e820cf7e2
Parents: 7632409
Author: yliu <yl...@apache.org>
Authored: Tue Nov 3 09:14:32 2015 +0800
Committer: yliu <yl...@apache.org>
Committed: Tue Nov 3 09:14:32 2015 +0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../server/blockmanagement/BlockManager.java | 5 +
.../apache/hadoop/hdfs/StripedFileTestUtil.java | 8 +-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 2 +
.../TestReadStripedFileWithMissingBlocks.java | 6 +-
.../hadoop/hdfs/TestRecoverStripedFile.java | 143 ++++++-------------
.../hdfs/TestSafeModeWithStripedFile.java | 5 +-
.../hadoop/hdfs/TestWriteReadStripedFile.java | 8 +-
.../hdfs/TestWriteStripedFileWithFailure.java | 6 +-
.../TestBlockTokenWithDFSStriped.java | 4 +-
.../namenode/TestRecoverStripedBlocks.java | 70 +++++++++
11 files changed, 145 insertions(+), 115 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ba2b98d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index c13a725..3c60549 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -849,6 +849,9 @@ Trunk (Unreleased)
HDFS-8438. Erasure Coding: Allow concat striped files if they have the same
ErasureCodingPolicy. (Walter Su via jing9)
+ HDFS-9275. Wait previous ErasureCodingWork to finish before schedule
+ another one. (Walter Su via yliu)
+
Release 2.8.0 - UNRELEASED
NEW FEATURES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ba2b98d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 897df1e..dbe0726 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -1586,6 +1586,10 @@ public class BlockManager implements BlockStatsMXBean {
}
if (block.isStriped()) {
+ if (pendingNum > 0) {
+ // Wait the previous recovery to finish.
+ return null;
+ }
short[] indices = new short[liveBlockIndices.size()];
for (int i = 0 ; i < liveBlockIndices.size(); i++) {
indices[i] = liveBlockIndices.get(i);
@@ -1641,6 +1645,7 @@ public class BlockManager implements BlockStatsMXBean {
if (block.isStriped()) {
assert rw instanceof ErasureCodingWork;
assert rw.getTargets().length > 0;
+ assert pendingNum == 0: "Should wait the previous recovery to finish";
String src = getBlockCollection(block).getName();
ErasureCodingPolicy ecPolicy = null;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ba2b98d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
index cc6e7d3..9942a2d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
@@ -61,10 +61,10 @@ public class StripedFileTestUtil {
public static final int BLOCK_STRIPED_CELL_SIZE = 64 * 1024;
public static final int BLOCK_STRIPE_SIZE = BLOCK_STRIPED_CELL_SIZE * NUM_DATA_BLOCKS;
- static final int stripesPerBlock = 4;
- static final int blockSize = BLOCK_STRIPED_CELL_SIZE * stripesPerBlock;
- static final int numDNs = NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS + 2;
- static final int BLOCK_GROUP_SIZE = blockSize * NUM_DATA_BLOCKS;
+ public static final int stripesPerBlock = 4;
+ public static final int blockSize = BLOCK_STRIPED_CELL_SIZE * stripesPerBlock;
+ public static final int numDNs = NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS + 2;
+ public static final int BLOCK_GROUP_SIZE = blockSize * NUM_DATA_BLOCKS;
static byte[] generateBytes(int cnt) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ba2b98d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
index f1ce8ff..76b471a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
@@ -57,6 +57,8 @@ public class TestDFSStripedOutputStream {
int numDNs = dataBlocks + parityBlocks + 2;
conf = new Configuration();
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
+ false);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ba2b98d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
index f521d8e..b5ffb38 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
@@ -41,9 +41,9 @@ import static org.apache.hadoop.hdfs.StripedFileTestUtil.numDNs;
public class TestReadStripedFileWithMissingBlocks {
public static final Log LOG = LogFactory
.getLog(TestReadStripedFileWithMissingBlocks.class);
- private static MiniDFSCluster cluster;
- private static DistributedFileSystem fs;
- private static Configuration conf = new HdfsConfiguration();
+ private MiniDFSCluster cluster;
+ private DistributedFileSystem fs;
+ private Configuration conf = new HdfsConfiguration();
private final short dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS;
private final int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
private final int fileLength = blockSize * dataBlocks + 123;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ba2b98d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java
index ec7594f..5b9245b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java
@@ -20,9 +20,7 @@ package org.apache.hadoop.hdfs;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import java.io.ByteArrayOutputStream;
import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.BitSet;
@@ -41,11 +39,9 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
-import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
-import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.BlockECRecoveryCommand.BlockECRecoveryInfo;
import org.apache.hadoop.hdfs.util.StripedBlockUtil;
@@ -80,9 +76,12 @@ public class TestRecoverStripedFile {
public void setup() throws IOException {
conf = new Configuration();
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
- conf.setInt(DFSConfigKeys.DFS_DATANODE_STRIPED_READ_BUFFER_SIZE_KEY, cellSize - 1);
+ conf.setInt(DFSConfigKeys.DFS_DATANODE_STRIPED_READ_BUFFER_SIZE_KEY,
+ cellSize - 1);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(dnNum).build();;
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
+ false);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(dnNum).build();
cluster.waitActive();
fs = cluster.getFileSystem();
@@ -251,82 +250,56 @@ public class TestRecoverStripedFile {
lastBlock.getBlockSize(), cellSize, dataBlkNum, indices[toDead[i]]));
assertTrue(metadatas[i].getName().
endsWith(blocks[i].getGenerationStamp() + ".meta"));
- replicaContents[i] = readReplica(replicas[i]);
+ replicaContents[i] = DFSTestUtil.readFileAsBytes(replicas[i]);
}
int cellsNum = (fileLen - 1) / cellSize + 1;
int groupSize = Math.min(cellsNum, dataBlkNum) + parityBlkNum;
- try {
- DatanodeID[] dnIDs = new DatanodeID[toRecoverBlockNum];
- for (int i = 0; i < toRecoverBlockNum; i++) {
- /*
- * Kill the datanode which contains one replica
- * We need to make sure it dead in namenode: clear its update time and
- * trigger NN to check heartbeat.
- */
- DataNode dn = cluster.getDataNodes().get(deadDnIndices[i]);
- dn.shutdown();
- dnIDs[i] = dn.getDatanodeId();
- }
- setDataNodesDead(dnIDs);
-
- // Check the locatedBlocks of the file again
- locatedBlocks = getLocatedBlocks(file);
- lastBlock = (LocatedStripedBlock)locatedBlocks.getLastLocatedBlock();
- storageInfos = lastBlock.getLocations();
- assertEquals(storageInfos.length, groupSize - toRecoverBlockNum);
-
- int[] targetDNs = new int[dnNum - groupSize];
- n = 0;
- for (int i = 0; i < dnNum; i++) {
- if (!bitset.get(i)) { // not contain replica of the block.
- targetDNs[n++] = i;
- }
- }
-
- waitForRecoveryFinished(file, groupSize);
-
- targetDNs = sortTargetsByReplicas(blocks, targetDNs);
-
- // Check the replica on the new target node.
- for (int i = 0; i < toRecoverBlockNum; i++) {
- File replicaAfterRecovery = cluster.getBlockFile(targetDNs[i], blocks[i]);
- File metadataAfterRecovery =
- cluster.getBlockMetadataFile(targetDNs[i], blocks[i]);
- assertEquals(replicaAfterRecovery.length(), replicas[i].length());
- assertTrue(metadataAfterRecovery.getName().
- endsWith(blocks[i].getGenerationStamp() + ".meta"));
- byte[] replicaContentAfterRecovery = readReplica(replicaAfterRecovery);
-
- Assert.assertArrayEquals(replicaContents[i], replicaContentAfterRecovery);
- }
- } finally {
- for (int i = 0; i < toRecoverBlockNum; i++) {
- restartDataNode(toDead[i]);
- }
- cluster.waitActive();
+ for (int i = 0; i < toRecoverBlockNum; i++) {
+ /*
+ * Kill the datanode which contains one replica
+ * We need to make sure it dead in namenode: clear its update time and
+ * trigger NN to check heartbeat.
+ */
+ DataNode dn = cluster.getDataNodes().get(deadDnIndices[i]);
+ dn.shutdown();
+ cluster.setDataNodeDead(dn.getDatanodeId());
}
- fs.delete(file, true);
- }
-
- private void setDataNodesDead(DatanodeID[] dnIDs) throws IOException {
- for (DatanodeID dn : dnIDs) {
- DatanodeDescriptor dnd =
- NameNodeAdapter.getDatanode(cluster.getNamesystem(), dn);
- DFSTestUtil.setDatanodeDead(dnd);
+
+ // Check the locatedBlocks of the file again
+ locatedBlocks = getLocatedBlocks(file);
+ lastBlock = (LocatedStripedBlock)locatedBlocks.getLastLocatedBlock();
+ storageInfos = lastBlock.getLocations();
+ assertEquals(storageInfos.length, groupSize - toRecoverBlockNum);
+
+ int[] targetDNs = new int[dnNum - groupSize];
+ n = 0;
+ for (int i = 0; i < dnNum; i++) {
+ if (!bitset.get(i)) { // not contain replica of the block.
+ targetDNs[n++] = i;
+ }
}
- BlockManagerTestUtil.checkHeartbeat(cluster.getNamesystem().getBlockManager());
- }
-
- private void restartDataNode(int dn) {
- try {
- cluster.restartDataNode(dn, true, true);
- } catch (IOException e) {
+ waitForRecoveryFinished(file, groupSize);
+
+ targetDNs = sortTargetsByReplicas(blocks, targetDNs);
+
+ // Check the replica on the new target node.
+ for (int i = 0; i < toRecoverBlockNum; i++) {
+ File replicaAfterRecovery = cluster.getBlockFile(targetDNs[i], blocks[i]);
+ File metadataAfterRecovery =
+ cluster.getBlockMetadataFile(targetDNs[i], blocks[i]);
+ assertEquals(replicaAfterRecovery.length(), replicas[i].length());
+ assertTrue(metadataAfterRecovery.getName().
+ endsWith(blocks[i].getGenerationStamp() + ".meta"));
+ byte[] replicaContentAfterRecovery =
+ DFSTestUtil.readFileAsBytes(replicaAfterRecovery);
+
+ Assert.assertArrayEquals(replicaContents[i], replicaContentAfterRecovery);
}
}
-
+
private int[] sortTargetsByReplicas(ExtendedBlock[] blocks, int[] targetDNs) {
int[] result = new int[blocks.length];
for (int i = 0; i < blocks.length; i++) {
@@ -347,31 +320,7 @@ public class TestRecoverStripedFile {
}
return result;
}
-
- private byte[] readReplica(File replica) throws IOException {
- int length = (int)replica.length();
- ByteArrayOutputStream content = new ByteArrayOutputStream(length);
- FileInputStream in = new FileInputStream(replica);
- try {
- byte[] buffer = new byte[1024];
- int total = 0;
- while (total < length) {
- int n = in.read(buffer);
- if (n <= 0) {
- break;
- }
- content.write(buffer, 0, n);
- total += n;
- }
- if (total < length) {
- Assert.fail("Failed to read all content of replica");
- }
- return content.toByteArray();
- } finally {
- in.close();
- }
- }
-
+
private LocatedBlocks waitForRecoveryFinished(Path file, int groupSize)
throws Exception {
final int ATTEMPTS = 60;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ba2b98d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
index 9853b8a..6d711d1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
@@ -46,8 +46,8 @@ public class TestSafeModeWithStripedFile {
static final int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
static final int blockSize = cellSize * 2;
- static MiniDFSCluster cluster;
- static Configuration conf;
+ private MiniDFSCluster cluster;
+ private Configuration conf;
@Before
public void setup() throws IOException {
@@ -57,7 +57,6 @@ public class TestSafeModeWithStripedFile {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null);
cluster.waitActive();
-
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ba2b98d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
index 4beb01f..3ea6eea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
@@ -47,11 +47,11 @@ import static org.apache.hadoop.hdfs.StripedFileTestUtil.stripesPerBlock;
public class TestWriteReadStripedFile {
public static final Log LOG = LogFactory.getLog(TestWriteReadStripedFile.class);
- private static MiniDFSCluster cluster;
- private static DistributedFileSystem fs;
private static int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
private static short dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS;
- private static Configuration conf = new HdfsConfiguration();
+ private MiniDFSCluster cluster;
+ private DistributedFileSystem fs;
+ private Configuration conf = new HdfsConfiguration();
static {
GenericTestUtils.setLogLevel(DFSOutputStream.LOG, Level.ALL);
@@ -64,6 +64,8 @@ public class TestWriteReadStripedFile {
@Before
public void setup() throws IOException {
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
+ false);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
fs = cluster.getFileSystem();
fs.mkdirs(new Path("/ec"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ba2b98d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
index 764527d..6dcff69 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
@@ -38,9 +38,9 @@ import static org.apache.hadoop.hdfs.StripedFileTestUtil.numDNs;
public class TestWriteStripedFileWithFailure {
public static final Log LOG = LogFactory
.getLog(TestWriteStripedFileWithFailure.class);
- private static MiniDFSCluster cluster;
- private static FileSystem fs;
- private static Configuration conf = new HdfsConfiguration();
+ private MiniDFSCluster cluster;
+ private FileSystem fs;
+ private Configuration conf = new HdfsConfiguration();
static {
GenericTestUtils.setLogLevel(DFSOutputStream.LOG, Level.ALL);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ba2b98d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
index ae33ffe..a9b2aaa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
@@ -36,8 +36,8 @@ public class TestBlockTokenWithDFSStriped extends TestBlockTokenWithDFS {
private final static int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
private final static int stripesPerBlock = 4;
private final static int numDNs = dataBlocks + parityBlocks + 2;
- private static MiniDFSCluster cluster;
- private static Configuration conf;
+ private MiniDFSCluster cluster;
+ private Configuration conf;
{
BLOCK_SIZE = cellSize * stripesPerBlock;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ba2b98d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRecoverStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRecoverStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRecoverStripedBlocks.java
index 6774aed..101601e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRecoverStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRecoverStripedBlocks.java
@@ -21,30 +21,41 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.StripedFileTestUtil;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.protocol.BlockECRecoveryCommand.BlockECRecoveryInfo;
+
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
import org.junit.Test;
import java.util.List;
import static org.apache.hadoop.hdfs.StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_DATA_BLOCKS;
import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_PARITY_BLOCKS;
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.blockSize;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class TestRecoverStripedBlocks {
+ private static final int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
private final short GROUP_SIZE =
(short) (NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS);
+
private MiniDFSCluster cluster;
private final Path dirPath = new Path("/dir");
private Path filePath = new Path(dirPath, "file");
@@ -166,4 +177,63 @@ public class TestRecoverStripedBlocks {
cluster.shutdown();
}
}
+
+ @Test
+ public void test2RecoveryTasksForSameBlockGroup() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1000);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1000);
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE + 2)
+ .build();
+ try {
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ BlockManager bm = cluster.getNamesystem().getBlockManager();
+ fs.getClient().setErasureCodingPolicy("/", null);
+ int fileLen = NUM_DATA_BLOCKS * blockSize;
+ Path p = new Path("/test2RecoveryTasksForSameBlockGroup");
+ final byte[] data = new byte[fileLen];
+ DFSTestUtil.writeFile(fs, p, data);
+
+ LocatedStripedBlock lb = (LocatedStripedBlock)fs.getClient()
+ .getLocatedBlocks(p.toString(), 0).get(0);
+ LocatedBlock[] lbs = StripedBlockUtil.parseStripedBlockGroup(lb,
+ cellSize, NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS);
+
+ assertEquals(0, getNumberOfBlocksToBeErasureCoded(cluster));
+ assertEquals(0, bm.getPendingReplicationBlocksCount());
+
+ // missing 1 block, so 1 task should be scheduled
+ DatanodeInfo dn0 = lbs[0].getLocations()[0];
+ cluster.stopDataNode(dn0.getName());
+ cluster.setDataNodeDead(dn0);
+ BlockManagerTestUtil.getComputedDatanodeWork(bm);
+ assertEquals(1, getNumberOfBlocksToBeErasureCoded(cluster));
+ assertEquals(1, bm.getPendingReplicationBlocksCount());
+
+ // missing another block, but no new task should be scheduled because
+ // previous task isn't finished.
+ DatanodeInfo dn1 = lbs[1].getLocations()[0];
+ cluster.stopDataNode(dn1.getName());
+ cluster.setDataNodeDead(dn1);
+ BlockManagerTestUtil.getComputedDatanodeWork(bm);
+ assertEquals(1, getNumberOfBlocksToBeErasureCoded(cluster));
+ assertEquals(1, bm.getPendingReplicationBlocksCount());
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
+ private static int getNumberOfBlocksToBeErasureCoded(MiniDFSCluster cluster)
+ throws Exception {
+ DatanodeManager dm =
+ cluster.getNamesystem().getBlockManager().getDatanodeManager();
+ int count = 0;
+ for( DataNode dn : cluster.getDataNodes()){
+ DatanodeDescriptor dd = dm.getDatanode(dn.getDatanodeId());
+ count += dd.getNumberOfBlocksToBeErasureCoded();
+ }
+ return count;
+ }
}
[18/50] hadoop git commit: Add an entry of YARN-4312 to CHANGES.txt
Posted by ji...@apache.org.
Add an entry of YARN-4312 to CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d21214ce
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d21214ce
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d21214ce
Branch: refs/heads/HDFS-8966
Commit: d21214ce33cb176926aa3ae5a9f4efe00f66480b
Parents: f072eb5
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Fri Oct 30 17:56:59 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Fri Oct 30 17:56:59 2015 +0900
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d21214ce/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 78c18d5..2151136 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1147,6 +1147,9 @@ Release 2.7.2 - UNRELEASED
YARN-3580. [JDK8] TestClientRMService.testGetLabelsToNodes fails. (Robert Kanter
via junping_du)
+ YARN-4312. TestSubmitApplicationWithRMHA fails on branch-2.7 and branch-2.6
+ as some of the test cases time out. (Varun Saxena via ozawa)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES