You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2014/09/11 22:50:52 UTC
[01/11] git commit: MAPREDUCE-6070.
yarn.app.am.resource.mb/cpu-vcores affects uber mode but is not documented.
Contributed by Tsuyoshi OZAWA
Repository: hadoop
Updated Branches:
refs/heads/HDFS-6584 70dfe9cfa -> 8de20a198
MAPREDUCE-6070. yarn.app.am.resource.mb/cpu-vcores affects uber mode but is not documented. Contributed by Tsuyoshi OZAWA
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e28e089
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e28e089
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e28e089
Branch: refs/heads/HDFS-6584
Commit: 9e28e089c6bcb0b264751bdfa1f994ce002ef2de
Parents: 4be9517
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Sep 11 16:19:23 2014 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Sep 11 16:19:23 2014 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../src/main/resources/mapred-default.xml | 7 ++++++-
2 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e28e089/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 490c0a1..0c065c5 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -279,6 +279,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6075. HistoryServerFileSystemStateStore can create zero-length
files (jlowe)
+ MAPREDUCE-6070. yarn.app.am.resource.mb/cpu-vcores affects uber mode but
+ is not documented (Tsuyoshi OZAWA via jlowe)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e28e089/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 703a103..6cefdc9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -508,7 +508,12 @@
<description>Whether to enable the small-jobs "ubertask" optimization,
which runs "sufficiently small" jobs sequentially within a single JVM.
"Small" is defined by the following maxmaps, maxreduces, and maxbytes
- settings. Users may override this value.
+ settings. Note that configurations for application masters also affect
+ the "Small" definition - yarn.app.mapreduce.am.resource.mb must be
+ larger than both mapreduce.map.memory.mb and mapreduce.reduce.memory.mb,
+ and yarn.app.mapreduce.am.resource.cpu-vcores must be larger than
+ both mapreduce.map.cpu.vcores and mapreduce.reduce.cpu.vcores to enable
+ ubertask. Users may override this value.
</description>
</property>
[02/11] git commit: HDFS-6966. Add additional unit tests for
encryption zones. Contributed by Stephen Chu.
Posted by ji...@apache.org.
HDFS-6966. Add additional unit tests for encryption zones. Contributed by Stephen Chu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bf64fce7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bf64fce7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bf64fce7
Branch: refs/heads/HDFS-6584
Commit: bf64fce78c5474f4d6ead839c4de18c8401a54d6
Parents: 9e28e08
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Sep 11 10:26:13 2014 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu Sep 11 10:26:13 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 20 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 187 +++++++++++++++++++
.../hadoop/hdfs/TestEncryptionZonesWithHA.java | 121 ++++++++++++
4 files changed, 329 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf64fce7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index fa00d44..266937d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -635,6 +635,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6506. Newly moved block replica been invalidated and deleted in
TestBalancer. (Binglin Chang via cnauroth)
+ HDFS-6966. Add additional unit tests for encryption zones.
+ (Stephen Chu via wang)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf64fce7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index e3a265f..9e442ef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -1359,7 +1359,8 @@ public class DFSTestUtil {
}
/**
- * Helper function to create a key in the Key Provider.
+ * Helper function to create a key in the Key Provider. Defaults
+ * to the first indexed NameNode's Key Provider.
*
* @param keyName The name of the key to create
* @param cluster The cluster to create it in
@@ -1368,7 +1369,22 @@ public class DFSTestUtil {
public static void createKey(String keyName, MiniDFSCluster cluster,
Configuration conf)
throws NoSuchAlgorithmException, IOException {
- KeyProvider provider = cluster.getNameNode().getNamesystem().getProvider();
+ createKey(keyName, cluster, 0, conf);
+ }
+
+ /**
+ * Helper function to create a key in the Key Provider.
+ *
+ * @param keyName The name of the key to create
+ * @param cluster The cluster to create it in
+ * @param idx The NameNode index
+ * @param conf Configuration to use
+ */
+ public static void createKey(String keyName, MiniDFSCluster cluster,
+ int idx, Configuration conf)
+ throws NoSuchAlgorithmException, IOException {
+ NameNode nn = cluster.getNameNode(idx);
+ KeyProvider provider = nn.getNamesystem().getProvider();
final KeyProvider.Options options = KeyProvider.options(conf);
options.setDescription(keyName);
options.setBitLength(128);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf64fce7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index d0d29ea..db3c085 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -17,8 +17,14 @@
*/
package org.apache.hadoop.hdfs;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.io.RandomAccessFile;
+import java.io.StringReader;
+import java.io.StringWriter;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
import java.util.List;
@@ -52,10 +58,15 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.namenode.EncryptionFaultInjector;
import org.apache.hadoop.hdfs.server.namenode.EncryptionZoneManager;
+import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
+import org.apache.hadoop.hdfs.server.namenode.NamenodeFsck;
+import org.apache.hadoop.hdfs.tools.DFSck;
+import org.apache.hadoop.hdfs.tools.offlineImageViewer.PBImageXmlWriter;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.crypto.key.KeyProviderDelegationTokenExtension.DelegationTokenExtension;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.CryptoExtension;
import org.apache.hadoop.io.Text;
@@ -78,6 +89,11 @@ import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import org.xml.sax.InputSource;
+import org.xml.sax.helpers.DefaultHandler;
+
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
public class TestEncryptionZones {
@@ -374,6 +390,7 @@ public class TestEncryptionZones {
final Path allPathFile = new Path(allPath, "file1");
final Path nonEZDir = new Path(testRoot, "nonEZDir");
final Path nonEZFile = new Path(nonEZDir, "file1");
+ final Path nonexistent = new Path("/nonexistent");
final int len = 8192;
fsWrapper.mkdir(testRoot, new FsPermission((short) 0777), true);
@@ -421,6 +438,9 @@ public class TestEncryptionZones {
assertExceptionContains("Permission denied:", e);
}
+ assertNull("expected null for nonexistent path",
+ userAdmin.getEncryptionZoneForPath(nonexistent));
+
// Check operation with non-ez paths
assertNull("expected null for non-ez path",
userAdmin.getEncryptionZoneForPath(nonEZDir));
@@ -463,9 +483,15 @@ public class TestEncryptionZones {
final Path testRoot = new Path("/tmp/TestEncryptionZones");
final Path pathFoo = new Path(testRoot, "foo");
final Path pathFooBaz = new Path(pathFoo, "baz");
+ final Path pathFooBazFile = new Path(pathFooBaz, "file");
+ final Path pathFooBar = new Path(pathFoo, "bar");
+ final Path pathFooBarFile = new Path(pathFooBar, "file");
+ final int len = 8192;
wrapper.mkdir(pathFoo, FsPermission.getDirDefault(), true);
dfsAdmin.createEncryptionZone(pathFoo, TEST_KEY);
wrapper.mkdir(pathFooBaz, FsPermission.getDirDefault(), true);
+ DFSTestUtil.createFile(fs, pathFooBazFile, len, (short) 1, 0xFEED);
+ String contents = DFSTestUtil.readFile(fs, pathFooBazFile);
try {
wrapper.rename(pathFooBaz, testRoot);
} catch (IOException e) {
@@ -473,6 +499,13 @@ public class TestEncryptionZones {
" an encryption zone.", e
);
}
+
+ // Verify that we can rename dir and files within an encryption zone.
+ assertTrue(fs.rename(pathFooBaz, pathFooBar));
+ assertTrue("Rename of dir and file within ez failed",
+ !wrapper.exists(pathFooBaz) && wrapper.exists(pathFooBar));
+ assertEquals("Renamed file contents not the same",
+ contents, DFSTestUtil.readFile(fs, pathFooBarFile));
}
@Test(timeout = 60000)
@@ -806,4 +839,158 @@ public class TestEncryptionZones {
Assert.assertEquals(tokens[1], testToken);
Assert.assertEquals(1, creds.numberOfTokens());
}
+
+ /**
+ * Test running fsck on a system with encryption zones.
+ */
+ @Test(timeout = 60000)
+ public void testFsckOnEncryptionZones() throws Exception {
+ final int len = 8196;
+ final Path zoneParent = new Path("/zones");
+ final Path zone1 = new Path(zoneParent, "zone1");
+ final Path zone1File = new Path(zone1, "file");
+ fsWrapper.mkdir(zone1, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone1, TEST_KEY);
+ DFSTestUtil.createFile(fs, zone1File, len, (short) 1, 0xFEED);
+ ByteArrayOutputStream bStream = new ByteArrayOutputStream();
+ PrintStream out = new PrintStream(bStream, true);
+ int errCode = ToolRunner.run(new DFSck(conf, out),
+ new String[]{ "/" });
+ assertEquals("Fsck ran with non-zero error code", 0, errCode);
+ String result = bStream.toString();
+ assertTrue("Fsck did not return HEALTHY status",
+ result.contains(NamenodeFsck.HEALTHY_STATUS));
+
+ // Run fsck directly on the encryption zone instead of root
+ errCode = ToolRunner.run(new DFSck(conf, out),
+ new String[]{ zoneParent.toString() });
+ assertEquals("Fsck ran with non-zero error code", 0, errCode);
+ result = bStream.toString();
+ assertTrue("Fsck did not return HEALTHY status",
+ result.contains(NamenodeFsck.HEALTHY_STATUS));
+ }
+
+ /**
+ * Test correctness of successive snapshot creation and deletion
+ * on a system with encryption zones.
+ */
+ @Test(timeout = 60000)
+ public void testSnapshotsOnEncryptionZones() throws Exception {
+ final int len = 8196;
+ final Path zoneParent = new Path("/zones");
+ final Path zone = new Path(zoneParent, "zone");
+ final Path zoneFile = new Path(zone, "zoneFile");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.allowSnapshot(zoneParent);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY);
+ DFSTestUtil.createFile(fs, zoneFile, len, (short) 1, 0xFEED);
+ String contents = DFSTestUtil.readFile(fs, zoneFile);
+ final Path snap1 = fs.createSnapshot(zoneParent);
+ final Path snap1Zone = new Path(snap1, zone.getName());
+ assertEquals("Got unexpected ez path", zone.toString(),
+ dfsAdmin.getEncryptionZoneForPath(snap1Zone).getPath().toString());
+
+ // Now delete the encryption zone, recreate the dir, and take another snapshot
+ fsWrapper.delete(zone, true);
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ final Path snap2 = fs.createSnapshot(zoneParent);
+ final Path snap2Zone = new Path(snap2, zone.getName());
+ assertNull("Expected null ez path",
+ dfsAdmin.getEncryptionZoneForPath(snap2Zone));
+
+ // Create the encryption zone again
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY);
+ final Path snap3 = fs.createSnapshot(zoneParent);
+ final Path snap3Zone = new Path(snap3, zone.getName());
+ assertEquals("Got unexpected ez path", zone.toString(),
+ dfsAdmin.getEncryptionZoneForPath(snap3Zone).getPath().toString());
+
+ // Verify contents of the snapshotted file
+ final Path snapshottedZoneFile = new Path(
+ snap1.toString() + "/" + zone.getName() + "/" + zoneFile.getName());
+ assertEquals("Contents of snapshotted file have changed unexpectedly",
+ contents, DFSTestUtil.readFile(fs, snapshottedZoneFile));
+
+ // Now delete the snapshots out of order and verify the zones are still correct
+ fs.deleteSnapshot(zoneParent, snap2.getName());
+ assertEquals("Got unexpected ez path", zone.toString(),
+ dfsAdmin.getEncryptionZoneForPath(snap1Zone).getPath().toString());
+ assertEquals("Got unexpected ez path", zone.toString(),
+ dfsAdmin.getEncryptionZoneForPath(snap3Zone).getPath().toString());
+ fs.deleteSnapshot(zoneParent, snap1.getName());
+ assertEquals("Got unexpected ez path", zone.toString(),
+ dfsAdmin.getEncryptionZoneForPath(snap3Zone).getPath().toString());
+ }
+
+ /**
+ * Verify symlinks can be created in encryption zones and that
+ * they function properly when the target is in the same
+ * or different ez.
+ */
+ @Test(timeout = 60000)
+ public void testEncryptionZonesWithSymlinks() throws Exception {
+ // Verify we can create an encryption zone over both link and target
+ final int len = 8192;
+ final Path parent = new Path("/parent");
+ final Path linkParent = new Path(parent, "symdir1");
+ final Path targetParent = new Path(parent, "symdir2");
+ final Path link = new Path(linkParent, "link");
+ final Path target = new Path(targetParent, "target");
+ fs.mkdirs(parent);
+ dfsAdmin.createEncryptionZone(parent, TEST_KEY);
+ fs.mkdirs(linkParent);
+ fs.mkdirs(targetParent);
+ DFSTestUtil.createFile(fs, target, len, (short)1, 0xFEED);
+ String content = DFSTestUtil.readFile(fs, target);
+ fs.createSymlink(target, link, false);
+ assertEquals("Contents read from link are not the same as target",
+ content, DFSTestUtil.readFile(fs, link));
+ fs.delete(parent, true);
+
+ // Now let's test when the symlink and target are in different
+ // encryption zones
+ fs.mkdirs(linkParent);
+ fs.mkdirs(targetParent);
+ dfsAdmin.createEncryptionZone(linkParent, TEST_KEY);
+ dfsAdmin.createEncryptionZone(targetParent, TEST_KEY);
+ DFSTestUtil.createFile(fs, target, len, (short)1, 0xFEED);
+ content = DFSTestUtil.readFile(fs, target);
+ fs.createSymlink(target, link, false);
+ assertEquals("Contents read from link are not the same as target",
+ content, DFSTestUtil.readFile(fs, link));
+ fs.delete(link, true);
+ fs.delete(target, true);
+ }
+
+ /**
+ * Test running the OfflineImageViewer on a system with encryption zones.
+ */
+ @Test(timeout = 60000)
+ public void testOfflineImageViewerOnEncryptionZones() throws Exception {
+ final int len = 8196;
+ final Path zoneParent = new Path("/zones");
+ final Path zone1 = new Path(zoneParent, "zone1");
+ final Path zone1File = new Path(zone1, "file");
+ fsWrapper.mkdir(zone1, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone1, TEST_KEY);
+ DFSTestUtil.createFile(fs, zone1File, len, (short) 1, 0xFEED);
+ fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER, false);
+ fs.saveNamespace();
+
+ File originalFsimage = FSImageTestUtil.findLatestImageFile(FSImageTestUtil
+ .getFSImage(cluster.getNameNode()).getStorage().getStorageDir(0));
+ if (originalFsimage == null) {
+ throw new RuntimeException("Didn't generate or can't find fsimage");
+ }
+
+ // Run the XML OIV processor
+ StringWriter output = new StringWriter();
+ PrintWriter pw = new PrintWriter(output);
+ PBImageXmlWriter v = new PBImageXmlWriter(new Configuration(), pw);
+ v.visit(new RandomAccessFile(originalFsimage, "r"));
+ final String xml = output.getBuffer().toString();
+ SAXParser parser = SAXParserFactory.newInstance().newSAXParser();
+ parser.parse(new InputSource(new StringReader(xml)), new DefaultHandler());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf64fce7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithHA.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithHA.java
new file mode 100644
index 0000000..b604004
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithHA.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
+import org.apache.hadoop.crypto.key.KeyProviderFactory;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.FileSystemTestHelper;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
+import org.apache.hadoop.hdfs.client.HdfsAdmin;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Tests interaction of encryption zones with HA failover.
+ */
+public class TestEncryptionZonesWithHA {
+ private Configuration conf;
+ private MiniDFSCluster cluster;
+ private NameNode nn0;
+ private NameNode nn1;
+ private DistributedFileSystem fs;
+ private HdfsAdmin dfsAdmin0;
+ private HdfsAdmin dfsAdmin1;
+ private FileSystemTestHelper fsHelper;
+ private File testRootDir;
+
+ private final String TEST_KEY = "testKey";
+
+
+ @Before
+ public void setupCluster() throws Exception {
+ conf = new Configuration();
+ conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1);
+ HAUtil.setAllowStandbyReads(conf, true);
+ fsHelper = new FileSystemTestHelper();
+ String testRoot = fsHelper.getTestRootDir();
+ testRootDir = new File(testRoot).getAbsoluteFile();
+ conf.set(KeyProviderFactory.KEY_PROVIDER_PATH,
+ JavaKeyStoreProvider.SCHEME_NAME + "://file" + testRootDir + "/test.jks"
+ );
+
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nnTopology(MiniDFSNNTopology.simpleHATopology())
+ .numDataNodes(1)
+ .build();
+ cluster.waitActive();
+ cluster.transitionToActive(0);
+
+ fs = (DistributedFileSystem)HATestUtil.configureFailoverFs(cluster, conf);
+ DFSTestUtil.createKey(TEST_KEY, cluster, 0, conf);
+ DFSTestUtil.createKey(TEST_KEY, cluster, 1, conf);
+ nn0 = cluster.getNameNode(0);
+ nn1 = cluster.getNameNode(1);
+ dfsAdmin0 = new HdfsAdmin(cluster.getURI(0), conf);
+ dfsAdmin1 = new HdfsAdmin(cluster.getURI(1), conf);
+ KeyProviderCryptoExtension nn0Provider =
+ cluster.getNameNode(0).getNamesystem().getProvider();
+ fs.getClient().provider = nn0Provider;
+ }
+
+ @After
+ public void shutdownCluster() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ /**
+ * Test that encryption zones are properly tracked by the standby.
+ */
+ @Test(timeout = 60000)
+ public void testEncryptionZonesTrackedOnStandby() throws Exception {
+ final int len = 8196;
+ final Path dir = new Path("/enc");
+ final Path dirChild = new Path(dir, "child");
+ final Path dirFile = new Path(dir, "file");
+ fs.mkdir(dir, FsPermission.getDirDefault());
+ dfsAdmin0.createEncryptionZone(dir, TEST_KEY);
+ fs.mkdir(dirChild, FsPermission.getDirDefault());
+ DFSTestUtil.createFile(fs, dirFile, len, (short) 1, 0xFEED);
+ String contents = DFSTestUtil.readFile(fs, dirFile);
+
+ // Failover the current standby to active.
+ HATestUtil.waitForStandbyToCatchUp(nn0, nn1);
+ cluster.shutdownNameNode(0);
+ cluster.transitionToActive(1);
+
+ Assert.assertEquals("Got unexpected ez path", dir.toString(),
+ dfsAdmin1.getEncryptionZoneForPath(dir).getPath().toString());
+ Assert.assertEquals("Got unexpected ez path", dir.toString(),
+ dfsAdmin1.getEncryptionZoneForPath(dirChild).getPath().toString());
+ Assert.assertEquals("File contents after failover were changed",
+ contents, DFSTestUtil.readFile(fs, dirFile));
+ }
+
+}
[10/11] git commit: HDFS-7034. Archival Storage: Fix
TestBlockPlacement and TestStorageMover. Contributed by Jing Zhao.
Posted by ji...@apache.org.
HDFS-7034. Archival Storage: Fix TestBlockPlacement and TestStorageMover. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d85f7e5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d85f7e5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d85f7e5
Branch: refs/heads/HDFS-6584
Commit: 0d85f7e59146cc3e9a040c2203995f3efd8ed4eb
Parents: 70dfe9c
Author: Jing Zhao <ji...@apache.org>
Authored: Thu Sep 11 13:00:43 2014 -0700
Committer: Jing Zhao <ji...@apache.org>
Committed: Thu Sep 11 13:00:43 2014 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +-
.../hadoop/hdfs/server/balancer/Dispatcher.java | 27 +-
.../hdfs/server/balancer/NameNodeConnector.java | 20 +-
.../server/blockmanagement/BlockManager.java | 4 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 4 +-
.../apache/hadoop/hdfs/server/mover/Mover.java | 10 +-
.../hadoop/hdfs/TestBlockStoragePolicy.java | 8 +-
.../hdfs/server/mover/TestStorageMover.java | 286 ++++++++++---------
8 files changed, 208 insertions(+), 153 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d85f7e5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index f3edbf5..641fb52 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -372,7 +372,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final int DFS_BALANCER_MOVERTHREADS_DEFAULT = 1000;
public static final String DFS_BALANCER_DISPATCHERTHREADS_KEY = "dfs.balancer.dispatcherThreads";
public static final int DFS_BALANCER_DISPATCHERTHREADS_DEFAULT = 200;
-
+
public static final String DFS_MOVER_MOVEDWINWIDTH_KEY = "dfs.mover.movedWinWidth";
public static final long DFS_MOVER_MOVEDWINWIDTH_DEFAULT = 5400*1000L;
public static final String DFS_MOVER_MOVERTHREADS_KEY = "dfs.mover.moverThreads";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d85f7e5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
index 98bd58e..f2a1299 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
@@ -49,6 +49,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.StorageType;
@@ -88,7 +89,11 @@ public class Dispatcher {
private static final long MAX_BLOCKS_SIZE_TO_FETCH = 2 * GB;
private static final int MAX_NO_PENDING_MOVE_ITERATIONS = 5;
- private static final long DELAY_AFTER_ERROR = 10 * 1000L; // 10 seconds
+ /**
+ * the period of time to delay the usage of a DataNode after hitting
+ * errors when using it for migrating data
+ */
+ private static long delayAfterErrors = 10 * 1000;
private final NameNodeConnector nnc;
private final SaslDataTransferClient saslClient;
@@ -112,6 +117,7 @@ public class Dispatcher {
private final ExecutorService moveExecutor;
private final ExecutorService dispatchExecutor;
+
/** The maximum number of concurrent blocks moves at a datanode */
private final int maxConcurrentMovesPerNode;
@@ -187,10 +193,12 @@ public class Dispatcher {
@Override
public String toString() {
- final Block b = block.getBlock();
- return b + " with size=" + b.getNumBytes() + " from "
- + source.getDisplayName() + " to " + target.getDisplayName()
- + " through " + proxySource.datanode;
+ final Block b = block != null ? block.getBlock() : null;
+ String bStr = b != null ? (b + " with size=" + b.getNumBytes() + " ")
+ : " ";
+ return bStr + "from " + source.getDisplayName() + " to " + target
+ .getDisplayName() + " through " + (proxySource != null ? proxySource
+ .datanode : "");
}
/**
@@ -316,8 +324,8 @@ public class Dispatcher {
// further in order to avoid a potential storm of "threads quota
// exceeded" warnings when the dispatcher gets out of sync with work
// going on in datanodes.
- proxySource.activateDelay(DELAY_AFTER_ERROR);
- target.getDDatanode().activateDelay(DELAY_AFTER_ERROR);
+ proxySource.activateDelay(delayAfterErrors);
+ target.getDDatanode().activateDelay(delayAfterErrors);
} finally {
IOUtils.closeStream(out);
IOUtils.closeStream(in);
@@ -1043,6 +1051,11 @@ public class Dispatcher {
blockMoveWaitTime = time;
}
+ @VisibleForTesting
+ public static void setDelayAfterErrors(long time) {
+ delayAfterErrors = time;
+ }
+
/** shutdown thread pools */
public void shutdownNow() {
if (dispatchExecutor != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d85f7e5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
index 79815c0..9e08d51 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
@@ -25,13 +25,16 @@ import java.net.InetAddress;
import java.net.URI;
import java.util.*;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.NameNodeProxies;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
@@ -53,6 +56,7 @@ public class NameNodeConnector implements Closeable {
private static final Log LOG = LogFactory.getLog(NameNodeConnector.class);
private static final int MAX_NOT_CHANGED_ITERATIONS = 5;
+ private static boolean createIdFile = true;
/** Create {@link NameNodeConnector} for the given namenodes. */
public static List<NameNodeConnector> newNameNodeConnectors(
@@ -83,6 +87,11 @@ public class NameNodeConnector implements Closeable {
return connectors;
}
+ @VisibleForTesting
+ public static void setCreateIdFile(boolean create) {
+ createIdFile = create;
+ }
+
private final URI nameNodeUri;
private final String blockpoolID;
@@ -117,9 +126,10 @@ public class NameNodeConnector implements Closeable {
final FsServerDefaults defaults = fs.getServerDefaults(new Path("/"));
this.keyManager = new KeyManager(blockpoolID, namenode,
defaults.getEncryptDataTransfer(), conf);
- // Exit if there is another one running.
- out = checkAndMarkRunning();
- if (out == null) {
+ // if it is for test, we do not create the id file
+ out = createIdFile ? checkAndMarkRunning() : null;
+ if (createIdFile && out == null) {
+ // Exit if there is another one running.
throw new IOException("Another " + name + " is running.");
}
}
@@ -188,9 +198,9 @@ public class NameNodeConnector implements Closeable {
*/
private OutputStream checkAndMarkRunning() throws IOException {
try {
- final DataOutputStream out = fs.create(idPath);
+ final FSDataOutputStream out = fs.create(idPath);
out.writeBytes(InetAddress.getLocalHost().getHostName());
- out.flush();
+ out.hflush();
return out;
} catch(RemoteException e) {
if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d85f7e5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 956900d..cb303a7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -1502,7 +1502,7 @@ public class BlockManager {
* @throws IOException
* if the number of targets < minimum replication.
* @see BlockPlacementPolicy#chooseTarget(String, int, Node,
- * List, boolean, Set, long, StorageType)
+ * Set, long, List, BlockStoragePolicy)
*/
public DatanodeStorageInfo[] chooseTarget4NewBlock(final String src,
final int numOfReplicas, final DatanodeDescriptor client,
@@ -2811,7 +2811,7 @@ public class BlockManager {
return false; // only consider delHint for the first case
} else if (delHint == null) {
return false; // no delHint
- } else if (!excessTypes.remove(delHint.getStorageType())) {
+ } else if (!excessTypes.contains(delHint.getStorageType())) {
return false; // delHint storage type is not an excess type
} else {
// check if removing delHint reduces the number of racks
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d85f7e5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 3810621..9b57262 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1744,7 +1744,9 @@ public class DataNode extends Configured
+ b + " (numBytes=" + b.getNumBytes() + ")"
+ ", stage=" + stage
+ ", clientname=" + clientname
- + ", targets=" + Arrays.asList(targets));
+ + ", targets=" + Arrays.asList(targets)
+ + ", target storage types=" + (targetStorageTypes == null ? "[]" :
+ Arrays.asList(targetStorageTypes)));
}
this.targets = targets;
this.targetStorageTypes = targetStorageTypes;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d85f7e5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
index c5d6dab..96588ff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
@@ -328,8 +328,6 @@ public class Mover {
if (scheduleMoves4Block(diff, lb)) {
hasRemaining |= (diff.existing.size() > 1 &&
diff.expected.size() > 1);
- } else {
- hasRemaining = false; // not able to schedule any move
}
}
}
@@ -453,9 +451,11 @@ public class Mover {
static int run(Map<URI, List<Path>> namenodes, Configuration conf)
throws IOException, InterruptedException {
- final long sleeptime = 2000 * conf.getLong(
- DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
- DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT);
+ final long sleeptime =
+ conf.getLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
+ DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT) * 2000 +
+ conf.getLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT) * 1000;
LOG.info("namenodes = " + namenodes);
List<NameNodeConnector> connectors = Collections.emptyList();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d85f7e5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index da7306c..158c225 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -256,7 +256,7 @@ public class TestBlockStoragePolicy {
final short replication = 3;
{
- final List<StorageType> chosen = Arrays.asList();
+ final List<StorageType> chosen = Lists.newArrayList();
method.checkChooseStorageTypes(hot, replication, chosen,
StorageType.DISK, StorageType.DISK, StorageType.DISK);
method.checkChooseStorageTypes(warm, replication, chosen,
@@ -393,7 +393,7 @@ public class TestBlockStoragePolicy {
final EnumSet<StorageType> unavailables = disk;
final boolean isNewBlock = true;
{
- final List<StorageType> chosen = Arrays.asList();
+ final List<StorageType> chosen = Lists.newArrayList();
checkChooseStorageTypes(hot, replication, chosen, unavailables, isNewBlock);
checkChooseStorageTypes(warm, replication, chosen, unavailables, isNewBlock,
StorageType.ARCHIVE, StorageType.ARCHIVE, StorageType.ARCHIVE);
@@ -500,7 +500,7 @@ public class TestBlockStoragePolicy {
final short replication = 3;
{
- final List<StorageType> chosen = Arrays.asList();
+ final List<StorageType> chosen = Lists.newArrayList();
method.checkChooseStorageTypes(hot, replication, chosen,
StorageType.DISK, StorageType.DISK, StorageType.DISK);
method.checkChooseStorageTypes(warm, replication, chosen,
@@ -603,7 +603,7 @@ public class TestBlockStoragePolicy {
final EnumSet<StorageType> unavailables = disk;
final boolean isNewBlock = false;
{
- final List<StorageType> chosen = Arrays.asList();
+ final List<StorageType> chosen = Lists.newArrayList();
checkChooseStorageTypes(hot, replication, chosen, unavailables, isNewBlock,
StorageType.ARCHIVE, StorageType.ARCHIVE, StorageType.ARCHIVE);
checkChooseStorageTypes(warm, replication, chosen, unavailables, isNewBlock,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d85f7e5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
index fda744f..d5d5cab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
@@ -44,9 +44,13 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtocol;
import org.apache.hadoop.hdfs.server.balancer.Dispatcher;
import org.apache.hadoop.hdfs.server.balancer.ExitStatus;
+import org.apache.hadoop.hdfs.server.balancer.NameNodeConnector;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper;
import org.apache.hadoop.io.IOUtils;
import org.apache.log4j.Level;
@@ -66,6 +70,8 @@ public class TestStorageMover {
).getLogger().setLevel(Level.ALL);
((Log4JLogger)LogFactory.getLog(Dispatcher.class)
).getLogger().setLevel(Level.ALL);
+ ((Log4JLogger)LogFactory.getLog(DataTransferProtocol.class)).getLogger()
+ .setLevel(Level.ALL);
}
private static final int BLOCK_SIZE = 1024;
@@ -80,6 +86,8 @@ public class TestStorageMover {
static {
DEFAULT_CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
DEFAULT_CONF.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
+ DEFAULT_CONF.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
+ 2L);
DEFAULT_CONF.setLong(DFSConfigKeys.DFS_MOVER_MOVEDWINWIDTH_KEY, 2000L);
DEFAULT_POLICIES = BlockStoragePolicy.readBlockStorageSuite(DEFAULT_CONF);
@@ -87,6 +95,9 @@ public class TestStorageMover {
WARM = DEFAULT_POLICIES.getPolicy("WARM");
COLD = DEFAULT_POLICIES.getPolicy("COLD");
Dispatcher.setBlockMoveWaitTime(1000L);
+ Dispatcher.setDelayAfterErrors(1000L);
+ // do not create id file since we will eat up all the disk space
+ NameNodeConnector.setCreateIdFile(false);
}
/**
@@ -151,7 +162,7 @@ public class TestStorageMover {
ClusterScheme() {
this(DEFAULT_CONF, NUM_DATANODES, REPL,
- genStorageTypes(NUM_DATANODES, 1, 1), null);
+ genStorageTypes(NUM_DATANODES), null);
}
ClusterScheme(Configuration conf, int numDataNodes, short repl,
@@ -195,7 +206,7 @@ public class TestStorageMover {
dfs = cluster.getFileSystem();
}
- private void runBasicTest(boolean shotdown) throws Exception {
+ private void runBasicTest(boolean shutdown) throws Exception {
setupCluster();
try {
prepareNamespace();
@@ -205,7 +216,7 @@ public class TestStorageMover {
migrate();
verify(true);
} finally {
- if (shotdown) {
+ if (shutdown) {
shutdownCluster();
}
}
@@ -233,7 +244,7 @@ public class TestStorageMover {
/**
* Run the migration tool.
*/
- void migrate(String... args) throws Exception {
+ void migrate() throws Exception {
runMover();
Thread.sleep(5000); // let the NN finish deletion
}
@@ -242,6 +253,9 @@ public class TestStorageMover {
* Verify block locations after running the migration tool.
*/
void verify(boolean verifyAll) throws Exception {
+ for (DataNode dn : cluster.getDataNodes()) {
+ DataNodeTestUtils.triggerBlockReport(dn);
+ }
if (verifyAll) {
verifyNamespace();
} else {
@@ -308,7 +322,8 @@ public class TestStorageMover {
final Mover.StorageTypeDiff diff = new Mover.StorageTypeDiff(types,
lb.getStorageTypes());
Assert.assertTrue(fileStatus.getFullName(parent.toString())
- + " with policy " + policy + " has non-empty overlap: " + diff,
+ + " with policy " + policy + " has non-empty overlap: " + diff
+ + ", the corresponding block is " + lb.getBlock().getLocalBlock(),
diff.removeOverlap());
}
}
@@ -378,6 +393,7 @@ public class TestStorageMover {
return "[disk=" + disk + ", archive=" + archive + "]";
}
}
+
private static StorageType[][] genStorageTypes(int numDataNodes) {
return genStorageTypes(numDataNodes, 0, 0);
}
@@ -414,21 +430,6 @@ public class TestStorageMover {
return capacities;
}
- /**
- * A normal case for Mover: move a file into archival storage
- */
- @Test
- public void testMigrateFileToArchival() throws Exception {
- final Path foo = new Path("/foo");
- Map<Path, BlockStoragePolicy> policyMap = Maps.newHashMap();
- policyMap.put(foo, COLD);
- NamespaceScheme nsScheme = new NamespaceScheme(null, Arrays.asList(foo),
- 2*BLOCK_SIZE, null, policyMap);
- ClusterScheme clusterScheme = new ClusterScheme(DEFAULT_CONF,
- NUM_DATANODES, REPL, genStorageTypes(NUM_DATANODES), null);
- new MigrationTest(clusterScheme, nsScheme).runBasicTest(true);
- }
-
private static class PathPolicyMap {
final Map<Path, BlockStoragePolicy> map = Maps.newHashMap();
final Path hot = new Path("/hot");
@@ -447,13 +448,13 @@ public class TestStorageMover {
}
}
}
-
+
NamespaceScheme newNamespaceScheme() {
return new NamespaceScheme(Arrays.asList(hot, warm, cold),
files, BLOCK_SIZE/2, null, map);
}
-
- /**
+
+ /**
* Move hot files to warm and cold, warm files to hot and cold,
* and cold files to hot and warm.
*/
@@ -473,21 +474,41 @@ public class TestStorageMover {
}
/**
+ * A normal case for Mover: move a file into archival storage
+ */
+ @Test
+ public void testMigrateFileToArchival() throws Exception {
+ LOG.info("testMigrateFileToArchival");
+ final Path foo = new Path("/foo");
+ Map<Path, BlockStoragePolicy> policyMap = Maps.newHashMap();
+ policyMap.put(foo, COLD);
+ NamespaceScheme nsScheme = new NamespaceScheme(null, Arrays.asList(foo),
+ 2*BLOCK_SIZE, null, policyMap);
+ ClusterScheme clusterScheme = new ClusterScheme(DEFAULT_CONF,
+ NUM_DATANODES, REPL, genStorageTypes(NUM_DATANODES), null);
+ new MigrationTest(clusterScheme, nsScheme).runBasicTest(true);
+ }
+
+ /**
* Test directories with Hot, Warm and Cold polices.
*/
@Test
public void testHotWarmColdDirs() throws Exception {
+ LOG.info("testHotWarmColdDirs");
PathPolicyMap pathPolicyMap = new PathPolicyMap(3);
NamespaceScheme nsScheme = pathPolicyMap.newNamespaceScheme();
ClusterScheme clusterScheme = new ClusterScheme();
MigrationTest test = new MigrationTest(clusterScheme, nsScheme);
- test.runBasicTest(false);
+ try {
+ test.runBasicTest(false);
+ pathPolicyMap.moveAround(test.dfs);
+ test.migrate();
- pathPolicyMap.moveAround(test.dfs);
- test.migrate();
- test.verify(true);
- test.shutdownCluster();
+ test.verify(true);
+ } finally {
+ test.shutdownCluster();
+ }
}
/**
@@ -495,76 +516,81 @@ public class TestStorageMover {
*/
@Test
public void testNoSpaceDisk() throws Exception {
+ LOG.info("testNoSpaceDisk");
final PathPolicyMap pathPolicyMap = new PathPolicyMap(0);
final NamespaceScheme nsScheme = pathPolicyMap.newNamespaceScheme();
- final long diskCapacity = (10 + HdfsConstants.MIN_BLOCKS_FOR_WRITE)*BLOCK_SIZE;
- final long archiveCapacity = 100*BLOCK_SIZE;
+ final long diskCapacity = (6 + HdfsConstants.MIN_BLOCKS_FOR_WRITE)
+ * BLOCK_SIZE;
+ final long archiveCapacity = 100 * BLOCK_SIZE;
final long[][] capacities = genCapacities(NUM_DATANODES, 1, 1,
diskCapacity, archiveCapacity);
- final ClusterScheme clusterScheme = new ClusterScheme(DEFAULT_CONF,
+ Configuration conf = new Configuration(DEFAULT_CONF);
+ final ClusterScheme clusterScheme = new ClusterScheme(conf,
NUM_DATANODES, REPL, genStorageTypes(NUM_DATANODES, 1, 1), capacities);
final MigrationTest test = new MigrationTest(clusterScheme, nsScheme);
- test.runBasicTest(false);
-
- // create hot files with replication 3 until not more spaces.
- final short replication = 3;
- {
- int hotFileCount = 0;
- try {
- for(; ; hotFileCount++) {
- final Path p = new Path(pathPolicyMap.hot, "file" + hotFileCount);
- DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, replication, 0L);
+ try {
+ test.runBasicTest(false);
+
+ // create hot files with replication 3 until not more spaces.
+ final short replication = 3;
+ {
+ int hotFileCount = 0;
+ try {
+ for (; ; hotFileCount++) {
+ final Path p = new Path(pathPolicyMap.hot, "file" + hotFileCount);
+ DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, replication, 0L);
+ }
+ } catch (IOException e) {
+ LOG.info("Expected: hotFileCount=" + hotFileCount, e);
}
- } catch(IOException e) {
- LOG.info("Expected: hotFileCount=" + hotFileCount, e);
+ Assert.assertTrue(hotFileCount >= 1);
}
- Assert.assertTrue(hotFileCount >= 2);
- }
- // create hot files with replication 1 to use up all remaining spaces.
- {
- int hotFileCount_r1 = 0;
- try {
- for(; ; hotFileCount_r1++) {
- final Path p = new Path(pathPolicyMap.hot, "file_r1_" + hotFileCount_r1);
- DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, (short)1, 0L);
+ // create hot files with replication 1 to use up all remaining spaces.
+ {
+ int hotFileCount_r1 = 0;
+ try {
+ for (; ; hotFileCount_r1++) {
+ final Path p = new Path(pathPolicyMap.hot, "file_r1_" + hotFileCount_r1);
+ DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, (short) 1, 0L);
+ }
+ } catch (IOException e) {
+ LOG.info("Expected: hotFileCount_r1=" + hotFileCount_r1, e);
}
- } catch(IOException e) {
- LOG.info("Expected: hotFileCount_r1=" + hotFileCount_r1, e);
}
- }
- { // test increasing replication. Since DISK is full,
- // new replicas should be stored in ARCHIVE as a fallback storage.
- final Path file0 = new Path(pathPolicyMap.hot, "file0");
- final Replication r = test.getReplication(file0);
- final short newReplication = (short)5;
- test.dfs.setReplication(file0, newReplication);
- Thread.sleep(10000);
- test.verifyReplication(file0, r.disk, newReplication - r.disk);
- }
+ { // test increasing replication. Since DISK is full,
+ // new replicas should be stored in ARCHIVE as a fallback storage.
+ final Path file0 = new Path(pathPolicyMap.hot, "file0");
+ final Replication r = test.getReplication(file0);
+ final short newReplication = (short) 5;
+ test.dfs.setReplication(file0, newReplication);
+ Thread.sleep(10000);
+ test.verifyReplication(file0, r.disk, newReplication - r.disk);
+ }
- { // test creating a cold file and then increase replication
- final Path p = new Path(pathPolicyMap.cold, "foo");
- DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, replication, 0L);
- test.verifyReplication(p, 0, replication);
+ { // test creating a cold file and then increase replication
+ final Path p = new Path(pathPolicyMap.cold, "foo");
+ DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, replication, 0L);
+ test.verifyReplication(p, 0, replication);
- final short newReplication = 5;
- test.dfs.setReplication(p, newReplication);
- Thread.sleep(10000);
- test.verifyReplication(p, 0, newReplication);
- }
+ final short newReplication = 5;
+ test.dfs.setReplication(p, newReplication);
+ Thread.sleep(10000);
+ test.verifyReplication(p, 0, newReplication);
+ }
- { //test move a hot file to warm
- final Path file1 = new Path(pathPolicyMap.hot, "file1");
- test.dfs.rename(file1, pathPolicyMap.warm);
- test.migrate();
- test.verifyFile(new Path(pathPolicyMap.warm, "file1"), WARM.getId());;
+ { //test move a hot file to warm
+ final Path file1 = new Path(pathPolicyMap.hot, "file1");
+ test.dfs.rename(file1, pathPolicyMap.warm);
+ test.migrate();
+ test.verifyFile(new Path(pathPolicyMap.warm, "file1"), WARM.getId());
+ }
+ } finally {
+ test.shutdownCluster();
}
-
- test.shutdownCluster();
}
/**
@@ -572,73 +598,77 @@ public class TestStorageMover {
*/
@Test
public void testNoSpaceArchive() throws Exception {
+ LOG.info("testNoSpaceArchive");
final PathPolicyMap pathPolicyMap = new PathPolicyMap(0);
final NamespaceScheme nsScheme = pathPolicyMap.newNamespaceScheme();
- final long diskCapacity = 100*BLOCK_SIZE;
- final long archiveCapacity = (10 + HdfsConstants.MIN_BLOCKS_FOR_WRITE)*BLOCK_SIZE;
+ final long diskCapacity = 100 * BLOCK_SIZE;
+ final long archiveCapacity = (6 + HdfsConstants.MIN_BLOCKS_FOR_WRITE)
+ * BLOCK_SIZE;
final long[][] capacities = genCapacities(NUM_DATANODES, 1, 1,
diskCapacity, archiveCapacity);
final ClusterScheme clusterScheme = new ClusterScheme(DEFAULT_CONF,
NUM_DATANODES, REPL, genStorageTypes(NUM_DATANODES, 1, 1), capacities);
final MigrationTest test = new MigrationTest(clusterScheme, nsScheme);
- test.runBasicTest(false);
-
- // create cold files with replication 3 until not more spaces.
- final short replication = 3;
- {
- int coldFileCount = 0;
- try {
- for(; ; coldFileCount++) {
- final Path p = new Path(pathPolicyMap.cold, "file" + coldFileCount);
- DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, replication, 0L);
+ try {
+ test.runBasicTest(false);
+
+ // create cold files with replication 3 until not more spaces.
+ final short replication = 3;
+ {
+ int coldFileCount = 0;
+ try {
+ for (; ; coldFileCount++) {
+ final Path p = new Path(pathPolicyMap.cold, "file" + coldFileCount);
+ DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, replication, 0L);
+ }
+ } catch (IOException e) {
+ LOG.info("Expected: coldFileCount=" + coldFileCount, e);
}
- } catch(IOException e) {
- LOG.info("Expected: coldFileCount=" + coldFileCount, e);
+ Assert.assertTrue(coldFileCount >= 1);
}
- Assert.assertTrue(coldFileCount >= 2);
- }
- // create cold files with replication 1 to use up all remaining spaces.
- {
- int coldFileCount_r1 = 0;
- try {
- for(; ; coldFileCount_r1++) {
- final Path p = new Path(pathPolicyMap.cold, "file_r1_" + coldFileCount_r1);
- DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, (short)1, 0L);
+ // create cold files with replication 1 to use up all remaining spaces.
+ {
+ int coldFileCount_r1 = 0;
+ try {
+ for (; ; coldFileCount_r1++) {
+ final Path p = new Path(pathPolicyMap.cold, "file_r1_" + coldFileCount_r1);
+ DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, (short) 1, 0L);
+ }
+ } catch (IOException e) {
+ LOG.info("Expected: coldFileCount_r1=" + coldFileCount_r1, e);
}
- } catch(IOException e) {
- LOG.info("Expected: coldFileCount_r1=" + coldFileCount_r1, e);
}
- }
- { // test increasing replication but new replicas cannot be created
- // since no more ARCHIVE space.
- final Path file0 = new Path(pathPolicyMap.cold, "file0");
- final Replication r = test.getReplication(file0);
- LOG.info("XXX " + file0 + ": replication=" + r);
- Assert.assertEquals(0, r.disk);
+ { // test increasing replication but new replicas cannot be created
+ // since no more ARCHIVE space.
+ final Path file0 = new Path(pathPolicyMap.cold, "file0");
+ final Replication r = test.getReplication(file0);
+ LOG.info("XXX " + file0 + ": replication=" + r);
+ Assert.assertEquals(0, r.disk);
- final short newReplication = (short)5;
- test.dfs.setReplication(file0, newReplication);
- Thread.sleep(10000);
+ final short newReplication = (short) 5;
+ test.dfs.setReplication(file0, newReplication);
+ Thread.sleep(10000);
- test.verifyReplication(file0, 0, r.archive);
- }
+ test.verifyReplication(file0, 0, r.archive);
+ }
- { // test creating a hot file
- final Path p = new Path(pathPolicyMap.hot, "foo");
- DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, (short)3, 0L);
- }
+ { // test creating a hot file
+ final Path p = new Path(pathPolicyMap.hot, "foo");
+ DFSTestUtil.createFile(test.dfs, p, BLOCK_SIZE, (short) 3, 0L);
+ }
- { //test move a cold file to warm
- final Path file1 = new Path(pathPolicyMap.hot, "file1");
- test.dfs.rename(file1, pathPolicyMap.warm);
- test.migrate();
- test.verify(true);
+ { //test move a cold file to warm
+ final Path file1 = new Path(pathPolicyMap.cold, "file1");
+ test.dfs.rename(file1, pathPolicyMap.warm);
+ test.migrate();
+ test.verify(true);
+ }
+ } finally {
+ test.shutdownCluster();
}
-
- test.shutdownCluster();
}
}
[09/11] git commit: HADOOP-11085. Excessive logging by
org.apache.hadoop.util.Progress when value is NaN. Contributed by Mit Desai
Posted by ji...@apache.org.
HADOOP-11085. Excessive logging by org.apache.hadoop.util.Progress when value is NaN. Contributed by Mit Desai
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/581176cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/581176cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/581176cd
Branch: refs/heads/HDFS-6584
Commit: 581176cdc868b8701a761b5a0d52f771cbba709a
Parents: 80ac6aa
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Sep 11 19:53:25 2014 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Sep 11 19:53:25 2014 +0000
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/util/Progress.java | 10 +++++-----
2 files changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/581176cd/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index ffb8ede..3bf92ec 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -784,6 +784,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-9989. Bug introduced in HADOOP-9374, which parses the -tokenCacheFile
as binary file but set it to the configuration as JSON file. (zxu via tucu)
+ HADOOP-11085. Excessive logging by org.apache.hadoop.util.Progress when
+ value is NaN (Mit Desai via jlowe)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/581176cd/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
index e3445b5..3fbc935 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Progress.java
@@ -162,27 +162,27 @@ public class Progress {
public synchronized void set(float progress) {
if (Float.isNaN(progress)) {
progress = 0;
- LOG.warn("Illegal progress value found, progress is Float.NaN. " +
+ LOG.debug("Illegal progress value found, progress is Float.NaN. " +
"Progress will be changed to 0");
}
else if (progress == Float.NEGATIVE_INFINITY) {
progress = 0;
- LOG.warn("Illegal progress value found, progress is " +
+ LOG.debug("Illegal progress value found, progress is " +
"Float.NEGATIVE_INFINITY. Progress will be changed to 0");
}
else if (progress < 0) {
progress = 0;
- LOG.warn("Illegal progress value found, progress is less than 0." +
+ LOG.debug("Illegal progress value found, progress is less than 0." +
" Progress will be changed to 0");
}
else if (progress > 1) {
progress = 1;
- LOG.warn("Illegal progress value found, progress is larger than 1." +
+ LOG.debug("Illegal progress value found, progress is larger than 1." +
" Progress will be changed to 1");
}
else if (progress == Float.POSITIVE_INFINITY) {
progress = 1;
- LOG.warn("Illegal progress value found, progress is " +
+ LOG.debug("Illegal progress value found, progress is " +
"Float.POSITIVE_INFINITY. Progress will be changed to 1");
}
this.progress = progress;
[05/11] git commit: HADOOP-9540: move CHANGES.txt entry to 2.6
Posted by ji...@apache.org.
HADOOP-9540: move CHANGES.txt entry to 2.6
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a123d02f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a123d02f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a123d02f
Branch: refs/heads/HDFS-6584
Commit: a123d02f4190ceadf94a66bc37a89c7f4e2ede51
Parents: 1e68499
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Thu Sep 11 11:31:58 2014 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Thu Sep 11 11:31:58 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a123d02f/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 3cea14a..2ae642c 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -102,9 +102,6 @@ Trunk (Unreleased)
HADOOP-9486. Promoted Windows and Shell related utils from YARN to Hadoop
Common. (Chris Nauroth via vinodkv)
- HADOOP-9540. Expose the InMemoryS3 and S3N FilesystemStores implementations
- for Unit testing. (Hari via stevel)
-
HADOOP-8844. Add a plaintext fs -text test-case.
(Akira AJISAKA via harsh)
@@ -511,6 +508,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10758. KMS: add ACLs on per key basis. (tucu)
+ HADOOP-9540. Expose the InMemoryS3 and S3N FilesystemStores implementations
+ for Unit testing. (Hari via stevel)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
[08/11] git commit: HDFS-7042. Upgrade fails for Windows HA cluster
due to file locks held during rename in JournalNode. Contributed by Chris
Nauroth.
Posted by ji...@apache.org.
HDFS-7042. Upgrade fails for Windows HA cluster due to file locks held during rename in JournalNode. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/80ac6aab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/80ac6aab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/80ac6aab
Branch: refs/heads/HDFS-6584
Commit: 80ac6aabcea9f808fd55504cdaef2da7b50da7f1
Parents: c11ada5
Author: cnauroth <cn...@apache.org>
Authored: Thu Sep 11 12:16:19 2014 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Thu Sep 11 12:16:19 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/qjournal/server/Journal.java | 8 +++++++-
.../java/org/apache/hadoop/hdfs/util/BestEffortLongFile.java | 1 +
3 files changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80ac6aab/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4c9dbaf..a11371a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -751,6 +751,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6776. Using distcp to copy data between insecure and secure cluster via webdhfs
doesn't work. (yzhangal via tucu)
+ HDFS-7042. Upgrade fails for Windows HA cluster due to file locks held during
+ rename in JournalNode. (cnauroth)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80ac6aab/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
index b36e547..c79e269 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
@@ -988,6 +988,9 @@ public class Journal implements Closeable {
}
public synchronized void doPreUpgrade() throws IOException {
+ // Do not hold file lock on committedTxnId, because the containing
+ // directory will be renamed. It will be reopened lazily on next access.
+ committedTxnId.close();
storage.getJournalManager().doPreUpgrade();
}
@@ -1037,7 +1040,10 @@ public class Journal implements Closeable {
targetLayoutVersion);
}
- public void doRollback() throws IOException {
+ public synchronized void doRollback() throws IOException {
+ // Do not hold file lock on committedTxnId, because the containing
+ // directory will be renamed. It will be reopened lazily on next access.
+ committedTxnId.close();
storage.getJournalManager().doRollback();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80ac6aab/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/BestEffortLongFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/BestEffortLongFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/BestEffortLongFile.java
index b2626e9..2fdf547 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/BestEffortLongFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/BestEffortLongFile.java
@@ -112,6 +112,7 @@ public class BestEffortLongFile implements Closeable {
public void close() throws IOException {
if (ch != null) {
ch.close();
+ ch = null;
}
}
}
[03/11] git commit: HDFS-6621. Hadoop Balancer prematurely exits
iterations. Contributed by Rafal Wodjdyla and Benjamin Bowman.
Posted by ji...@apache.org.
HDFS-6621. Hadoop Balancer prematurely exits iterations. Contributed by Rafal Wodjdyla and Benjamin Bowman.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d1dab6cc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d1dab6cc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d1dab6cc
Branch: refs/heads/HDFS-6584
Commit: d1dab6cc63abf5d44533ad9b39e32c582327627b
Parents: bf64fce
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Sep 11 10:32:20 2014 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu Sep 11 10:32:20 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java | 2 ++
2 files changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1dab6cc/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 266937d..cfa9344 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -638,6 +638,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6966. Add additional unit tests for encryption zones.
(Stephen Chu via wang)
+ HDFS-6621. Hadoop Balancer prematurely exits iterations.
+ (Rafal Wojdyla and Benjamin Bowman via wang)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1dab6cc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
index b7dceb6..14d5989 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
@@ -654,6 +654,8 @@ public class Dispatcher {
&& (!srcBlocks.isEmpty() || blocksToReceive > 0)) {
final PendingMove p = chooseNextMove();
if (p != null) {
+ // Reset no pending move counter
+ noPendingMoveIteration=0;
// move the block
moveExecutor.execute(new Runnable() {
@Override
[06/11] git commit: HADOOP-10373: move CHANGES.txt entry to branch-2.6
Posted by ji...@apache.org.
HADOOP-10373: move CHANGES.txt entry to branch-2.6
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ac35e346
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ac35e346
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ac35e346
Branch: refs/heads/HDFS-6584
Commit: ac35e3464ff1b5bacf416fb78112681f0a9e7ed8
Parents: a123d02
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Thu Sep 11 11:46:35 2014 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Thu Sep 11 11:46:35 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac35e346/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2ae642c..ffb8ede 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -126,8 +126,6 @@ Trunk (Unreleased)
HADOOP-11041. VersionInfo specifies subversion (Tsuyoshi OZAWA via aw)
- HADOOP-10373 create tools/hadoop-amazon for aws/EMR support (stevel)
-
BUG FIXES
HADOOP-9451. Fault single-layer config if node group topology is enabled.
@@ -511,6 +509,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-9540. Expose the InMemoryS3 and S3N FilesystemStores implementations
for Unit testing. (Hari via stevel)
+ HADOOP-10373 create tools/hadoop-amazon for aws/EMR support (stevel)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
[11/11] git commit: Merge trunk into HDFS-6584
Posted by ji...@apache.org.
Merge trunk into HDFS-6584
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8de20a19
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8de20a19
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8de20a19
Branch: refs/heads/HDFS-6584
Commit: 8de20a1987a3ca26eb006cd261d70155b3307db9
Parents: 0d85f7e 581176c
Author: Jing Zhao <ji...@apache.org>
Authored: Thu Sep 11 13:04:35 2014 -0700
Committer: Jing Zhao <ji...@apache.org>
Committed: Thu Sep 11 13:04:35 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 13 +-
.../java/org/apache/hadoop/util/Progress.java | 10 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 12 ++
.../hadoop/hdfs/qjournal/server/Journal.java | 8 +-
.../hadoop/hdfs/server/balancer/Dispatcher.java | 2 +
.../hdfs/server/namenode/FSNamesystem.java | 2 +-
.../hadoop/hdfs/util/BestEffortLongFile.java | 1 +
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 20 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 187 +++++++++++++++++++
.../hadoop/hdfs/TestEncryptionZonesWithHA.java | 121 ++++++++++++
.../java/org/apache/hadoop/hdfs/TestRead.java | 22 +++
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../src/main/resources/mapred-default.xml | 7 +-
hadoop-yarn-project/CHANGES.txt | 3 +
.../fair/policies/ComputeFairShares.java | 7 +-
.../scheduler/fair/TestFairScheduler.java | 48 +++++
16 files changed, 447 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8de20a19/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8de20a19/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
index f2a1299,14d5989..e4b7d4e
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
@@@ -725,7 -654,15 +725,9 @@@ public class Dispatcher
&& (!srcBlocks.isEmpty() || blocksToReceive > 0)) {
final PendingMove p = chooseNextMove();
if (p != null) {
+ // Reset no pending move counter
+ noPendingMoveIteration=0;
- // move the block
- moveExecutor.execute(new Runnable() {
- @Override
- public void run() {
- p.dispatch();
- }
- });
+ executePendingMove(p);
continue;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8de20a19/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8de20a19/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
[07/11] git commit: YARN-2534. FairScheduler: Potential integer
overflow calculating totalMaxShare. (Zhihai Xu via kasha)
Posted by ji...@apache.org.
YARN-2534. FairScheduler: Potential integer overflow calculating totalMaxShare. (Zhihai Xu via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c11ada5e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c11ada5e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c11ada5e
Branch: refs/heads/HDFS-6584
Commit: c11ada5ea6d17321626e5a9a4152ff857d03aee2
Parents: ac35e34
Author: Karthik Kambatla <ka...@apache.org>
Authored: Thu Sep 11 12:06:06 2014 -0700
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Thu Sep 11 12:06:06 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../fair/policies/ComputeFairShares.java | 7 ++-
.../scheduler/fair/TestFairScheduler.java | 48 ++++++++++++++++++++
3 files changed, 54 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c11ada5e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6a871a5..eac9f3c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -323,6 +323,9 @@ Release 2.6.0 - UNRELEASED
YARN-2158. Fixed TestRMWebServicesAppsModification#testSingleAppKill test
failure. (Varun Vasudev via jianhe)
+ YARN-2534. FairScheduler: Potential integer overflow calculating totalMaxShare.
+ (Zhihai Xu via kasha)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c11ada5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
index 12ddde2..3bea985 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
@@ -124,11 +124,10 @@ public class ComputeFairShares {
int totalMaxShare = 0;
for (Schedulable sched : schedulables) {
int maxShare = getResourceValue(sched.getMaxShare(), type);
- if (maxShare == Integer.MAX_VALUE) {
- totalMaxShare = Integer.MAX_VALUE;
+ totalMaxShare = (int) Math.min((long)maxShare + (long)totalMaxShare,
+ Integer.MAX_VALUE);
+ if (totalMaxShare == Integer.MAX_VALUE) {
break;
- } else {
- totalMaxShare += maxShare;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c11ada5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 3a0dac3..1c9b289 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -309,6 +309,54 @@ public class TestFairScheduler extends FairSchedulerTestBase {
}
@Test
+ public void testFairShareWithMaxResources() throws IOException {
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ // set queueA and queueB maxResources,
+ // the sum of queueA and queueB maxResources is more than
+ // Integer.MAX_VALUE.
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"queueA\">");
+ out.println("<maxResources>1073741824 mb 1000 vcores</maxResources>");
+ out.println("<weight>.25</weight>");
+ out.println("</queue>");
+ out.println("<queue name=\"queueB\">");
+ out.println("<maxResources>1073741824 mb 1000 vcores</maxResources>");
+ out.println("<weight>.75</weight>");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add one big node (only care about aggregate capacity)
+ RMNode node1 =
+ MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 1,
+ "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ // Queue A wants 1 * 1024.
+ createSchedulingRequest(1 * 1024, "queueA", "user1");
+ // Queue B wants 6 * 1024
+ createSchedulingRequest(6 * 1024, "queueB", "user1");
+
+ scheduler.update();
+
+ FSLeafQueue queue = scheduler.getQueueManager().getLeafQueue(
+ "queueA", false);
+ // queueA's weight is 0.25, so its fair share should be 2 * 1024.
+ assertEquals(2 * 1024, queue.getFairShare().getMemory());
+ // queueB's weight is 0.75, so its fair share should be 6 * 1024.
+ queue = scheduler.getQueueManager().getLeafQueue(
+ "queueB", false);
+ assertEquals(6 * 1024, queue.getFairShare().getMemory());
+ }
+
+ @Test
public void testFairShareWithZeroWeight() throws IOException {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
// set queueA and queueB weight zero.
[04/11] git commit: HDFS-7045. Fix NameNode deadlock when opening
file under /.reserved path. Contributed by Yi Liu.
Posted by ji...@apache.org.
HDFS-7045. Fix NameNode deadlock when opening file under /.reserved path. Contributed by Yi Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1e684995
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1e684995
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1e684995
Branch: refs/heads/HDFS-6584
Commit: 1e684995d7ac20d1c27ca1c1bef37dd2ee00e630
Parents: d1dab6c
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Sep 11 10:59:09 2014 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu Sep 11 10:59:09 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hdfs/server/namenode/FSNamesystem.java | 2 +-
.../java/org/apache/hadoop/hdfs/TestRead.java | 22 ++++++++++++++++++++
3 files changed, 26 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e684995/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index cfa9344..4c9dbaf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -641,6 +641,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6621. Hadoop Balancer prematurely exits iterations.
(Rafal Wojdyla and Benjamin Bowman via wang)
+ HDFS-7045. Fix NameNode deadlock when opening file under /.reserved path.
+ (Yi Liu via wang)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating & deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e684995/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index c1744f6..0ebbc1d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -1821,8 +1821,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
checkOperation(OperationCategory.WRITE);
writeLock(); // writelock is needed to set accesstime
}
- src = resolvePath(src, pathComponents);
try {
+ src = resolvePath(src, pathComponents);
if (isReadOp) {
checkOperation(OperationCategory.READ);
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e684995/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java
index c8d5029..378d8bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -80,4 +81,25 @@ public class TestRead {
testEOF(cluster, 10000);
cluster.shutdown();
}
+
+ /**
+ * Regression test for HDFS-7045.
+ * If deadlock happen, the test will time out.
+ * @throws Exception
+ */
+ @Test(timeout=60000)
+ public void testReadReservedPath() throws Exception {
+ Configuration conf = new Configuration();
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).
+ numDataNodes(1).format(true).build();
+ try {
+ FileSystem fs = cluster.getFileSystem();
+ fs.open(new Path("/.reserved/.inodes/file"));
+ Assert.fail("Open a non existing file should fail.");
+ } catch (FileNotFoundException e) {
+ // Expected
+ } finally {
+ cluster.shutdown();
+ }
+ }
}