You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by cm...@apache.org on 2014/08/20 01:50:25 UTC
svn commit: r1619012 [34/35] - in
/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project: hadoop-hdfs-httpfs/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/ hadoop...
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java Tue Aug 19 23:49:39 2014
@@ -143,10 +143,10 @@ public class TestSnapshotFileLength {
// Make sure we can read the entire file via its non-snapshot path.
fileStatus = hdfs.getFileStatus(file1);
- assertEquals(fileStatus.getLen(), BLOCKSIZE * 2);
+ assertEquals("Unexpected file length", BLOCKSIZE * 2, fileStatus.getLen());
fis = hdfs.open(file1);
bytesRead = fis.read(buffer, 0, buffer.length);
- assertEquals(bytesRead, BLOCKSIZE * 2);
+ assertEquals("Unexpected # bytes read", BLOCKSIZE * 2, bytesRead);
fis.close();
Path file1snap1 =
@@ -156,21 +156,23 @@ public class TestSnapshotFileLength {
assertEquals(fileStatus.getLen(), BLOCKSIZE);
// Make sure we can only read up to the snapshot length.
bytesRead = fis.read(buffer, 0, buffer.length);
- assertEquals(bytesRead, BLOCKSIZE);
+ assertEquals("Unexpected # bytes read", BLOCKSIZE, bytesRead);
fis.close();
- PrintStream psBackup = System.out;
+ PrintStream outBackup = System.out;
+ PrintStream errBackup = System.err;
ByteArrayOutputStream bao = new ByteArrayOutputStream();
System.setOut(new PrintStream(bao));
System.setErr(new PrintStream(bao));
// Make sure we can cat the file upto to snapshot length
FsShell shell = new FsShell();
- try{
+ try {
ToolRunner.run(conf, shell, new String[] { "-cat",
"/TestSnapshotFileLength/sub1/.snapshot/snapshot1/file1" });
- assertEquals(bao.size(), BLOCKSIZE);
- }finally{
- System.setOut(psBackup);
+ assertEquals("Unexpected # bytes from -cat", BLOCKSIZE, bao.size());
+ } finally {
+ System.setOut(outBackup);
+ System.setErr(errBackup);
}
}
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotManager.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotManager.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotManager.java Tue Aug 19 23:49:39 2014
@@ -18,13 +18,19 @@
package org.apache.hadoop.hdfs.server.namenode.snapshot;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+
import java.util.ArrayList;
import org.apache.hadoop.hdfs.protocol.SnapshotException;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.INode;
-import org.junit.*;
-import static org.mockito.Mockito.*;
+import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
+import org.junit.Assert;
+import org.junit.Test;
/**
@@ -40,7 +46,7 @@ public class TestSnapshotManager {
public void testSnapshotLimits() throws Exception {
// Setup mock objects for SnapshotManager.createSnapshot.
//
- INodeDirectorySnapshottable ids = mock(INodeDirectorySnapshottable.class);
+ INodeDirectory ids = mock(INodeDirectory.class);
FSDirectory fsdir = mock(FSDirectory.class);
SnapshotManager sm = spy(new SnapshotManager(fsdir));
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotRename.java Tue Aug 19 23:49:39 2014
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.protocol.H
import org.apache.hadoop.hdfs.protocol.SnapshotException;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff;
import org.apache.hadoop.hdfs.util.ReadOnlyList;
import org.apache.hadoop.ipc.RemoteException;
@@ -88,12 +89,13 @@ public class TestSnapshotRename {
public ExpectedException exception = ExpectedException.none();
/**
- * Check the correctness of snapshot list within
- * {@link INodeDirectorySnapshottable}
+ * Check the correctness of snapshot list within snapshottable dir
*/
- private void checkSnapshotList(INodeDirectorySnapshottable srcRoot,
+ private void checkSnapshotList(INodeDirectory srcRoot,
String[] sortedNames, String[] names) {
- ReadOnlyList<Snapshot> listByName = srcRoot.getSnapshotsByNames();
+ assertTrue(srcRoot.isSnapshottable());
+ ReadOnlyList<Snapshot> listByName = srcRoot
+ .getDirectorySnapshottableFeature().getSnapshotList();
assertEquals(sortedNames.length, listByName.size());
for (int i = 0; i < listByName.size(); i++) {
assertEquals(sortedNames[i], listByName.get(i).getRoot().getLocalName());
@@ -101,7 +103,8 @@ public class TestSnapshotRename {
List<DirectoryDiff> listByTime = srcRoot.getDiffs().asList();
assertEquals(names.length, listByTime.size());
for (int i = 0; i < listByTime.size(); i++) {
- Snapshot s = srcRoot.getSnapshotById(listByTime.get(i).getSnapshotId());
+ Snapshot s = srcRoot.getDirectorySnapshottableFeature().getSnapshotById(
+ listByTime.get(i).getSnapshotId());
assertEquals(names[i], s.getRoot().getLocalName());
}
}
@@ -121,8 +124,7 @@ public class TestSnapshotRename {
// Rename s3 to s22
hdfs.renameSnapshot(sub1, "s3", "s22");
// Check the snapshots list
- INodeDirectorySnapshottable srcRoot = INodeDirectorySnapshottable.valueOf(
- fsdir.getINode(sub1.toString()), sub1.toString());
+ INodeDirectory srcRoot = fsdir.getINode(sub1.toString()).asDirectory();
checkSnapshotList(srcRoot, new String[] { "s1", "s2", "s22" },
new String[] { "s1", "s2", "s22" });
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/web/resources/TestWebHdfsDataLocality.java Tue Aug 19 23:49:39 2014
@@ -92,7 +92,7 @@ public class TestWebHdfsDataLocality {
//The chosen datanode must be the same as the client address
final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode(
- namenode, f, PutOpParam.Op.CREATE, -1L, blocksize);
+ namenode, f, PutOpParam.Op.CREATE, -1L, blocksize, null);
Assert.assertEquals(ipAddr, chosen.getIpAddr());
}
}
@@ -117,23 +117,104 @@ public class TestWebHdfsDataLocality {
{ //test GETFILECHECKSUM
final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode(
- namenode, f, GetOpParam.Op.GETFILECHECKSUM, -1L, blocksize);
+ namenode, f, GetOpParam.Op.GETFILECHECKSUM, -1L, blocksize, null);
Assert.assertEquals(expected, chosen);
}
{ //test OPEN
final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode(
- namenode, f, GetOpParam.Op.OPEN, 0, blocksize);
+ namenode, f, GetOpParam.Op.OPEN, 0, blocksize, null);
Assert.assertEquals(expected, chosen);
}
{ //test APPEND
final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode(
- namenode, f, PostOpParam.Op.APPEND, -1L, blocksize);
+ namenode, f, PostOpParam.Op.APPEND, -1L, blocksize, null);
Assert.assertEquals(expected, chosen);
}
} finally {
cluster.shutdown();
}
}
+
+ @Test
+ public void testExcludeDataNodes() throws Exception {
+ final Configuration conf = WebHdfsTestUtil.createConf();
+ final String[] racks = {RACK0, RACK0, RACK1, RACK1, RACK2, RACK2};
+ final String[] hosts = {"DataNode1", "DataNode2", "DataNode3","DataNode4","DataNode5","DataNode6"};
+ final int nDataNodes = hosts.length;
+ LOG.info("nDataNodes=" + nDataNodes + ", racks=" + Arrays.asList(racks)
+ + ", hosts=" + Arrays.asList(hosts));
+
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .hosts(hosts).numDataNodes(nDataNodes).racks(racks).build();
+
+ try {
+ cluster.waitActive();
+
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ final NameNode namenode = cluster.getNameNode();
+ final DatanodeManager dm = namenode.getNamesystem().getBlockManager(
+ ).getDatanodeManager();
+ LOG.info("dm=" + dm);
+
+ final long blocksize = DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT;
+ final String f = "/foo";
+
+ //create a file with three replica.
+ final Path p = new Path(f);
+ final FSDataOutputStream out = dfs.create(p, (short)3);
+ out.write(1);
+ out.close();
+
+ //get replica location.
+ final LocatedBlocks locatedblocks = NameNodeAdapter.getBlockLocations(
+ namenode, f, 0, 1);
+ final List<LocatedBlock> lb = locatedblocks.getLocatedBlocks();
+ Assert.assertEquals(1, lb.size());
+ final DatanodeInfo[] locations = lb.get(0).getLocations();
+ Assert.assertEquals(3, locations.length);
+
+
+ //For GETFILECHECKSUM, OPEN and APPEND,
+ //the chosen datanode must be different with exclude nodes.
+
+ StringBuffer sb = new StringBuffer();
+ for (int i = 0; i < 2; i++) {
+ sb.append(locations[i].getXferAddr());
+ { // test GETFILECHECKSUM
+ final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode(
+ namenode, f, GetOpParam.Op.GETFILECHECKSUM, -1L, blocksize,
+ sb.toString());
+ for (int j = 0; j <= i; j++) {
+ Assert.assertNotEquals(locations[j].getHostName(),
+ chosen.getHostName());
+ }
+ }
+
+ { // test OPEN
+ final DatanodeInfo chosen = NamenodeWebHdfsMethods.chooseDatanode(
+ namenode, f, GetOpParam.Op.OPEN, 0, blocksize, sb.toString());
+ for (int j = 0; j <= i; j++) {
+ Assert.assertNotEquals(locations[j].getHostName(),
+ chosen.getHostName());
+ }
+ }
+
+ { // test APPEND
+ final DatanodeInfo chosen = NamenodeWebHdfsMethods
+ .chooseDatanode(namenode, f, PostOpParam.Op.APPEND, -1L,
+ blocksize, sb.toString());
+ for (int j = 0; j <= i; j++) {
+ Assert.assertNotEquals(locations[j].getHostName(),
+ chosen.getHostName());
+ }
+ }
+
+ sb.append(",");
+ }
+ } finally {
+ cluster.shutdown();
+ }
+ }
}
\ No newline at end of file
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java Tue Aug 19 23:49:39 2014
@@ -23,6 +23,7 @@ import static org.apache.hadoop.hdfs.DFS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_EXPIRY_MS_KEY;
import static org.hamcrest.CoreMatchers.equalTo;
import java.io.DataOutputStream;
@@ -30,7 +31,9 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.util.Arrays;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.Map;
import org.apache.commons.lang.mutable.MutableBoolean;
@@ -197,11 +200,12 @@ public class TestShortCircuitCache {
@Test(timeout=60000)
public void testExpiry() throws Exception {
final ShortCircuitCache cache =
- new ShortCircuitCache(2, 1, 1, 10000000, 1, 10000, 0);
+ new ShortCircuitCache(2, 1, 1, 10000000, 1, 10000000, 0);
final TestFileDescriptorPair pair = new TestFileDescriptorPair();
ShortCircuitReplicaInfo replicaInfo1 =
cache.fetchOrCreate(
- new ExtendedBlockId(123, "test_bp1"), new SimpleReplicaCreator(123, cache, pair));
+ new ExtendedBlockId(123, "test_bp1"),
+ new SimpleReplicaCreator(123, cache, pair));
Preconditions.checkNotNull(replicaInfo1.getReplica());
Preconditions.checkState(replicaInfo1.getInvalidTokenException() == null);
pair.compareWith(replicaInfo1.getReplica().getDataStream(),
@@ -461,6 +465,7 @@ public class TestShortCircuitCache {
}
}, 10, 60000);
cluster.shutdown();
+ sockDir.close();
}
@Test(timeout=60000)
@@ -515,4 +520,98 @@ public class TestShortCircuitCache {
});
cluster.shutdown();
}
+
+ /**
+ * Test unlinking a file whose blocks we are caching in the DFSClient.
+ * The DataNode will notify the DFSClient that the replica is stale via the
+ * ShortCircuitShm.
+ */
+ @Test(timeout=60000)
+ public void testUnlinkingReplicasInFileDescriptorCache() throws Exception {
+ BlockReaderTestUtil.enableShortCircuitShmTracing();
+ TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
+ Configuration conf = createShortCircuitConf(
+ "testUnlinkingReplicasInFileDescriptorCache", sockDir);
+ // We don't want the CacheCleaner to time out short-circuit shared memory
+ // segments during the test, so set the timeout really high.
+ conf.setLong(DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_EXPIRY_MS_KEY,
+ 1000000000L);
+ MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ final ShortCircuitCache cache =
+ fs.getClient().getClientContext().getShortCircuitCache();
+ cache.getDfsClientShmManager().visit(new Visitor() {
+ @Override
+ public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info)
+ throws IOException {
+ // The ClientShmManager starts off empty.
+ Assert.assertEquals(0, info.size());
+ }
+ });
+ final Path TEST_PATH = new Path("/test_file");
+ final int TEST_FILE_LEN = 8193;
+ final int SEED = 0xFADE0;
+ DFSTestUtil.createFile(fs, TEST_PATH, TEST_FILE_LEN,
+ (short)1, SEED);
+ byte contents[] = DFSTestUtil.readFileBuffer(fs, TEST_PATH);
+ byte expected[] = DFSTestUtil.
+ calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
+ Assert.assertTrue(Arrays.equals(contents, expected));
+ // Loading this file brought the ShortCircuitReplica into our local
+ // replica cache.
+ final DatanodeInfo datanode =
+ new DatanodeInfo(cluster.getDataNodes().get(0).getDatanodeId());
+ cache.getDfsClientShmManager().visit(new Visitor() {
+ @Override
+ public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info)
+ throws IOException {
+ Assert.assertTrue(info.get(datanode).full.isEmpty());
+ Assert.assertFalse(info.get(datanode).disabled);
+ Assert.assertEquals(1, info.get(datanode).notFull.values().size());
+ DfsClientShm shm =
+ info.get(datanode).notFull.values().iterator().next();
+ Assert.assertFalse(shm.isDisconnected());
+ }
+ });
+ // Remove the file whose blocks we just read.
+ fs.delete(TEST_PATH, false);
+
+ // Wait for the replica to be purged from the DFSClient's cache.
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ MutableBoolean done = new MutableBoolean(true);
+ @Override
+ public Boolean get() {
+ try {
+ done.setValue(true);
+ cache.getDfsClientShmManager().visit(new Visitor() {
+ @Override
+ public void visit(HashMap<DatanodeInfo,
+ PerDatanodeVisitorInfo> info) throws IOException {
+ Assert.assertTrue(info.get(datanode).full.isEmpty());
+ Assert.assertFalse(info.get(datanode).disabled);
+ Assert.assertEquals(1,
+ info.get(datanode).notFull.values().size());
+ DfsClientShm shm = info.get(datanode).notFull.values().
+ iterator().next();
+ // Check that all slots have been invalidated.
+ for (Iterator<Slot> iter = shm.slotIterator();
+ iter.hasNext(); ) {
+ Slot slot = iter.next();
+ if (slot.isValid()) {
+ done.setValue(false);
+ }
+ }
+ }
+ });
+ } catch (IOException e) {
+ LOG.error("error running visitor", e);
+ }
+ return done.booleanValue();
+ }
+ }, 10, 60000);
+ cluster.shutdown();
+ sockDir.close();
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitLocalRead.java Tue Aug 19 23:49:39 2014
@@ -291,17 +291,17 @@ public class TestShortCircuitLocalRead {
}
}
- @Test(timeout=10000)
+ @Test(timeout=60000)
public void testFileLocalReadNoChecksum() throws Exception {
doTestShortCircuitRead(true, 3*blockSize+100, 0);
}
- @Test(timeout=10000)
+ @Test(timeout=60000)
public void testFileLocalReadChecksum() throws Exception {
doTestShortCircuitRead(false, 3*blockSize+100, 0);
}
- @Test(timeout=10000)
+ @Test(timeout=60000)
public void testSmallFileLocalRead() throws Exception {
doTestShortCircuitRead(false, 13, 0);
doTestShortCircuitRead(false, 13, 5);
@@ -309,7 +309,7 @@ public class TestShortCircuitLocalRead {
doTestShortCircuitRead(true, 13, 5);
}
- @Test(timeout=10000)
+ @Test(timeout=60000)
public void testLocalReadLegacy() throws Exception {
doTestShortCircuitReadLegacy(true, 13, 0, getCurrentUser(),
getCurrentUser(), false);
@@ -320,18 +320,18 @@ public class TestShortCircuitLocalRead {
* to use short circuit. The test ensures reader falls back to non
* shortcircuit reads when shortcircuit is disallowed.
*/
- @Test(timeout=10000)
+ @Test(timeout=60000)
public void testLocalReadFallback() throws Exception {
doTestShortCircuitReadLegacy(true, 13, 0, getCurrentUser(), "notallowed", true);
}
- @Test(timeout=10000)
+ @Test(timeout=60000)
public void testReadFromAnOffset() throws Exception {
doTestShortCircuitRead(false, 3*blockSize+100, 777);
doTestShortCircuitRead(true, 3*blockSize+100, 777);
}
- @Test(timeout=10000)
+ @Test(timeout=60000)
public void testLongFile() throws Exception {
doTestShortCircuitRead(false, 10*blockSize+100, 777);
doTestShortCircuitRead(true, 10*blockSize+100, 777);
@@ -578,6 +578,7 @@ public class TestShortCircuitLocalRead {
fs.delete(file1, false);
}
+ @Test(timeout=60000)
public void testReadWithRemoteBlockReader() throws IOException, InterruptedException {
doTestShortCircuitReadWithRemoteBlockReader(true, 3*blockSize+100, getCurrentUser(), 0, false);
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdmin.java Tue Aug 19 23:49:39 2014
@@ -176,6 +176,7 @@ public class TestDFSHAAdmin {
@Test
public void testTransitionToActive() throws Exception {
+ Mockito.doReturn(STANDBY_READY_RESULT).when(mockProtocol).getServiceStatus();
assertEquals(0, runTool("-transitionToActive", "nn1"));
Mockito.verify(mockProtocol).transitionToActive(
reqInfoCaptor.capture());
@@ -414,6 +415,6 @@ public class TestDFSHAAdmin {
}
private StateChangeRequestInfo anyReqInfo() {
- return Mockito.<StateChangeRequestInfo>any();
+ return Mockito.any();
}
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSHAAdminMiniCluster.java Tue Aug 19 23:49:39 2014
@@ -31,6 +31,7 @@ import org.apache.commons.logging.LogFac
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAAdmin;
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
@@ -204,6 +205,70 @@ public class TestDFSHAAdminMiniCluster {
assertEquals(0, runTool("-checkHealth", "nn2"));
}
+ /**
+ * Test case to check whether both the name node is active or not
+ * @throws Exception
+ */
+ @Test
+ public void testTransitionToActiveWhenOtherNamenodeisActive()
+ throws Exception {
+ NameNode nn1 = cluster.getNameNode(0);
+ NameNode nn2 = cluster.getNameNode(1);
+ if(nn1.getState() != null && !nn1.getState().
+ equals(HAServiceState.STANDBY.name()) ) {
+ cluster.transitionToStandby(0);
+ }
+ if(nn2.getState() != null && !nn2.getState().
+ equals(HAServiceState.STANDBY.name()) ) {
+ cluster.transitionToStandby(1);
+ }
+ //Making sure both the namenode are in standby state
+ assertTrue(nn1.isStandbyState());
+ assertTrue(nn2.isStandbyState());
+ // Triggering the transition for both namenode to Active
+ runTool("-transitionToActive", "nn1");
+ runTool("-transitionToActive", "nn2");
+
+ assertFalse("Both namenodes cannot be active", nn1.isActiveState()
+ && nn2.isActiveState());
+
+ /* This test case doesn't allow nn2 to transition to Active even with
+ forceActive switch since nn1 is already active */
+ if(nn1.getState() != null && !nn1.getState().
+ equals(HAServiceState.STANDBY.name()) ) {
+ cluster.transitionToStandby(0);
+ }
+ if(nn2.getState() != null && !nn2.getState().
+ equals(HAServiceState.STANDBY.name()) ) {
+ cluster.transitionToStandby(1);
+ }
+ //Making sure both the namenode are in standby state
+ assertTrue(nn1.isStandbyState());
+ assertTrue(nn2.isStandbyState());
+
+ runTool("-transitionToActive", "nn1");
+ runTool("-transitionToActive", "nn2","--forceactive");
+
+ assertFalse("Both namenodes cannot be active even though with forceActive",
+ nn1.isActiveState() && nn2.isActiveState());
+
+ /* In this test case, we have deliberately shut down nn1 and this will
+ cause HAAAdmin#isOtherTargetNodeActive to throw an Exception
+ and transitionToActive for nn2 with forceActive switch will succeed
+ even with Exception */
+ cluster.shutdownNameNode(0);
+ if(nn2.getState() != null && !nn2.getState().
+ equals(HAServiceState.STANDBY.name()) ) {
+ cluster.transitionToStandby(1);
+ }
+ //Making sure both the namenode (nn2) is in standby state
+ assertTrue(nn2.isStandbyState());
+ assertFalse(cluster.isNameNodeUp(0));
+
+ runTool("-transitionToActive", "nn2", "--forceactive");
+ assertTrue("Namenode nn2 should be active", nn2.isActiveState());
+ }
+
private int runTool(String ... args) throws Exception {
errOutBytes.reset();
LOG.info("Running: DFSHAAdmin " + Joiner.on(" ").join(args));
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java Tue Aug 19 23:49:39 2014
@@ -56,8 +56,7 @@ public class TestOfflineEditsViewer {
@SuppressWarnings("deprecation")
private static ImmutableSet<FSEditLogOpCodes> skippedOps() {
- ImmutableSet.Builder<FSEditLogOpCodes> b = ImmutableSet
- .<FSEditLogOpCodes> builder();
+ ImmutableSet.Builder<FSEditLogOpCodes> b = ImmutableSet.builder();
// Deprecated opcodes
b.add(FSEditLogOpCodes.OP_DATANODE_ADD)
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java Tue Aug 19 23:49:39 2014
@@ -120,6 +120,11 @@ public class TestOfflineImageViewer {
}
}
+ // Create an empty directory
+ Path emptydir = new Path("/emptydir");
+ hdfs.mkdirs(emptydir);
+ writtenFiles.put(emptydir.toString(), hdfs.getFileStatus(emptydir));
+
// Get delegation tokens so we log the delegation token op
Token<?>[] delegationTokens = hdfs
.addDelegationTokens(TEST_RENEWER, null);
@@ -133,6 +138,15 @@ public class TestOfflineImageViewer {
hdfs.mkdirs(new Path("/snapshot/1"));
hdfs.delete(snapshot, true);
+ // Set XAttrs so the fsimage contains XAttr ops
+ final Path xattr = new Path("/xattr");
+ hdfs.mkdirs(xattr);
+ hdfs.setXAttr(xattr, "user.a1", new byte[]{ 0x31, 0x32, 0x33 });
+ hdfs.setXAttr(xattr, "user.a2", new byte[]{ 0x37, 0x38, 0x39 });
+ // OIV should be able to handle empty value XAttrs
+ hdfs.setXAttr(xattr, "user.a3", null);
+ writtenFiles.put(xattr.toString(), hdfs.getFileStatus(xattr));
+
// Write results to the fsimage file
hdfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER, false);
hdfs.saveNamespace();
@@ -205,8 +219,8 @@ public class TestOfflineImageViewer {
matcher = p.matcher(output.getBuffer());
assertTrue(matcher.find() && matcher.groupCount() == 1);
int totalDirs = Integer.parseInt(matcher.group(1));
- // totalDirs includes root directory
- assertEquals(NUM_DIRS + 1, totalDirs);
+ // totalDirs includes root directory, empty directory, and xattr directory
+ assertEquals(NUM_DIRS + 3, totalDirs);
FileStatus maxFile = Collections.max(writtenFiles.values(),
new Comparator<FileStatus>() {
@@ -259,7 +273,7 @@ public class TestOfflineImageViewer {
// verify the number of directories
FileStatus[] statuses = webhdfs.listStatus(new Path("/"));
- assertEquals(NUM_DIRS, statuses.length);
+ assertEquals(NUM_DIRS + 2, statuses.length); // contains empty and xattr directory
// verify the number of files in the directory
statuses = webhdfs.listStatus(new Path("/dir0"));
@@ -270,6 +284,10 @@ public class TestOfflineImageViewer {
FileStatus expected = writtenFiles.get("/dir0/file0");
compareFile(expected, status);
+ // LISTSTATUS operation to an empty directory
+ statuses = webhdfs.listStatus(new Path("/emptydir"));
+ assertEquals(0, statuses.length);
+
// LISTSTATUS operation to a invalid path
URL url = new URL("http://localhost:" + port +
"/webhdfs/v1/invalid/?op=LISTSTATUS");
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java Tue Aug 19 23:49:39 2014
@@ -22,16 +22,22 @@ import static org.apache.hadoop.fs.permi
import static org.apache.hadoop.fs.permission.FsAction.*;
import static org.apache.hadoop.hdfs.server.namenode.AclTestHelpers.*;
+import java.io.IOException;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.fs.XAttrCodec;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.server.namenode.INodeId;
@@ -186,6 +192,60 @@ public class TestJsonUtil {
JsonUtil.toJsonString(aclStatusBuilder.build()));
}
+
+ @Test
+ public void testToJsonFromXAttrs() throws IOException {
+ String jsonString =
+ "{\"XAttrs\":[{\"name\":\"user.a1\",\"value\":\"0x313233\"}," +
+ "{\"name\":\"user.a2\",\"value\":\"0x313131\"}]}";
+ XAttr xAttr1 = (new XAttr.Builder()).setNameSpace(XAttr.NameSpace.USER).
+ setName("a1").setValue(XAttrCodec.decodeValue("0x313233")).build();
+ XAttr xAttr2 = (new XAttr.Builder()).setNameSpace(XAttr.NameSpace.USER).
+ setName("a2").setValue(XAttrCodec.decodeValue("0x313131")).build();
+ List<XAttr> xAttrs = Lists.newArrayList();
+ xAttrs.add(xAttr1);
+ xAttrs.add(xAttr2);
+
+ Assert.assertEquals(jsonString, JsonUtil.toJsonString(xAttrs,
+ XAttrCodec.HEX));
+ }
+
+ @Test
+ public void testToXAttrMap() throws IOException {
+ String jsonString =
+ "{\"XAttrs\":[{\"name\":\"user.a1\",\"value\":\"0x313233\"}," +
+ "{\"name\":\"user.a2\",\"value\":\"0x313131\"}]}";
+ Map<?, ?> json = (Map<?, ?>)JSON.parse(jsonString);
+ XAttr xAttr1 = (new XAttr.Builder()).setNameSpace(XAttr.NameSpace.USER).
+ setName("a1").setValue(XAttrCodec.decodeValue("0x313233")).build();
+ XAttr xAttr2 = (new XAttr.Builder()).setNameSpace(XAttr.NameSpace.USER).
+ setName("a2").setValue(XAttrCodec.decodeValue("0x313131")).build();
+ List<XAttr> xAttrs = Lists.newArrayList();
+ xAttrs.add(xAttr1);
+ xAttrs.add(xAttr2);
+ Map<String, byte[]> xAttrMap = XAttrHelper.buildXAttrMap(xAttrs);
+ Map<String, byte[]> parsedXAttrMap = JsonUtil.toXAttrs(json);
+
+ Assert.assertEquals(xAttrMap.size(), parsedXAttrMap.size());
+ Iterator<Entry<String, byte[]>> iter = xAttrMap.entrySet().iterator();
+ while(iter.hasNext()) {
+ Entry<String, byte[]> entry = iter.next();
+ Assert.assertArrayEquals(entry.getValue(),
+ parsedXAttrMap.get(entry.getKey()));
+ }
+ }
+
+ @Test
+ public void testGetXAttrFromJson() throws IOException {
+ String jsonString =
+ "{\"XAttrs\":[{\"name\":\"user.a1\",\"value\":\"0x313233\"}," +
+ "{\"name\":\"user.a2\",\"value\":\"0x313131\"}]}";
+ Map<?, ?> json = (Map<?, ?>) JSON.parse(jsonString);
+
+ // Get xattr: user.a2
+ byte[] value = JsonUtil.getXAttr(json, "user.a2");
+ Assert.assertArrayEquals(XAttrCodec.decodeValue("0x313131"), value);
+ }
private void checkDecodeFailure(Map<String, Object> map) {
try {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java Tue Aug 19 23:49:39 2014
@@ -18,6 +18,8 @@
package org.apache.hadoop.hdfs.web;
+import static org.junit.Assert.fail;
+
import java.io.IOException;
import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
@@ -34,15 +36,21 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
import org.apache.hadoop.hdfs.TestDFSClientRetries;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper;
+import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
+import org.apache.hadoop.ipc.RetriableException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level;
import org.junit.Assert;
import org.junit.Test;
+import org.mockito.internal.util.reflection.Whitebox;
/** Test WebHDFS */
public class TestWebHDFS {
@@ -326,4 +334,152 @@ public class TestWebHDFS {
Assert.assertTrue(conf.getBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY,
false));
}
+
+ /**
+ * Test snapshot creation through WebHdfs
+ */
+ @Test
+ public void testWebHdfsCreateSnapshot() throws Exception {
+ MiniDFSCluster cluster = null;
+ final Configuration conf = WebHdfsTestUtil.createConf();
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+ cluster.waitActive();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ final FileSystem webHdfs = WebHdfsTestUtil.getWebHdfsFileSystem(conf,
+ WebHdfsFileSystem.SCHEME);
+
+ final Path foo = new Path("/foo");
+ dfs.mkdirs(foo);
+
+ try {
+ webHdfs.createSnapshot(foo);
+ fail("Cannot create snapshot on a non-snapshottable directory");
+ } catch (Exception e) {
+ GenericTestUtils.assertExceptionContains(
+ "Directory is not a snapshottable directory", e);
+ }
+
+ // allow snapshots on /foo
+ dfs.allowSnapshot(foo);
+ // create snapshots on foo using WebHdfs
+ webHdfs.createSnapshot(foo, "s1");
+ // create snapshot without specifying name
+ final Path spath = webHdfs.createSnapshot(foo, null);
+
+ Assert.assertTrue(webHdfs.exists(spath));
+ final Path s1path = SnapshotTestHelper.getSnapshotRoot(foo, "s1");
+ Assert.assertTrue(webHdfs.exists(s1path));
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ /**
+ * Test snapshot deletion through WebHdfs
+ */
+ @Test
+ public void testWebHdfsDeleteSnapshot() throws Exception {
+ MiniDFSCluster cluster = null;
+ final Configuration conf = WebHdfsTestUtil.createConf();
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+ cluster.waitActive();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ final FileSystem webHdfs = WebHdfsTestUtil.getWebHdfsFileSystem(conf,
+ WebHdfsFileSystem.SCHEME);
+
+ final Path foo = new Path("/foo");
+ dfs.mkdirs(foo);
+ dfs.allowSnapshot(foo);
+
+ webHdfs.createSnapshot(foo, "s1");
+ final Path spath = webHdfs.createSnapshot(foo, null);
+ Assert.assertTrue(webHdfs.exists(spath));
+ final Path s1path = SnapshotTestHelper.getSnapshotRoot(foo, "s1");
+ Assert.assertTrue(webHdfs.exists(s1path));
+
+ // delete the two snapshots
+ webHdfs.deleteSnapshot(foo, "s1");
+ Assert.assertFalse(webHdfs.exists(s1path));
+ webHdfs.deleteSnapshot(foo, spath.getName());
+ Assert.assertFalse(webHdfs.exists(spath));
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ /**
+ * Test snapshot rename through WebHdfs
+ */
+ @Test
+ public void testWebHdfsRenameSnapshot() throws Exception {
+ MiniDFSCluster cluster = null;
+ final Configuration conf = WebHdfsTestUtil.createConf();
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+ cluster.waitActive();
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ final FileSystem webHdfs = WebHdfsTestUtil.getWebHdfsFileSystem(conf,
+ WebHdfsFileSystem.SCHEME);
+
+ final Path foo = new Path("/foo");
+ dfs.mkdirs(foo);
+ dfs.allowSnapshot(foo);
+
+ webHdfs.createSnapshot(foo, "s1");
+ final Path s1path = SnapshotTestHelper.getSnapshotRoot(foo, "s1");
+ Assert.assertTrue(webHdfs.exists(s1path));
+
+ // rename s1 to s2
+ webHdfs.renameSnapshot(foo, "s1", "s2");
+ Assert.assertFalse(webHdfs.exists(s1path));
+ final Path s2path = SnapshotTestHelper.getSnapshotRoot(foo, "s2");
+ Assert.assertTrue(webHdfs.exists(s2path));
+
+ webHdfs.deleteSnapshot(foo, "s2");
+ Assert.assertFalse(webHdfs.exists(s2path));
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ /**
+ * Make sure a RetriableException is thrown when rpcServer is null in
+ * NamenodeWebHdfsMethods.
+ */
+ @Test
+ public void testRaceWhileNNStartup() throws Exception {
+ MiniDFSCluster cluster = null;
+ final Configuration conf = WebHdfsTestUtil.createConf();
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+ cluster.waitActive();
+ final NameNode namenode = cluster.getNameNode();
+ final NamenodeProtocols rpcServer = namenode.getRpcServer();
+ Whitebox.setInternalState(namenode, "rpcServer", null);
+
+ final Path foo = new Path("/foo");
+ final FileSystem webHdfs = WebHdfsTestUtil.getWebHdfsFileSystem(conf,
+ WebHdfsFileSystem.SCHEME);
+ try {
+ webHdfs.mkdirs(foo);
+ fail("Expected RetriableException");
+ } catch (RetriableException e) {
+ GenericTestUtils.assertExceptionContains("Namenode is in startup mode",
+ e);
+ }
+ Whitebox.setInternalState(namenode, "rpcServer", rpcServer);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java Tue Aug 19 23:49:39 2014
@@ -18,6 +18,15 @@
package org.apache.hadoop.hdfs.web;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -28,17 +37,15 @@ import org.apache.hadoop.hdfs.DFSConfigK
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
+import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.token.Token;
import org.junit.Assert;
import org.junit.Test;
-
-import java.io.IOException;
-import java.net.URI;
-
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.verify;
+import org.mockito.internal.util.reflection.Whitebox;
public class TestWebHDFSForHA {
private static final String LOGICAL_NAME = "minidfs";
@@ -119,6 +126,8 @@ public class TestWebHDFSForHA {
@Test
public void testFailoverAfterOpen() throws IOException {
Configuration conf = DFSTestUtil.newHAConfiguration(LOGICAL_NAME);
+ conf.set(FS_DEFAULT_NAME_KEY, HdfsConstants.HDFS_URI_SCHEME +
+ "://" + LOGICAL_NAME);
MiniDFSCluster cluster = null;
FileSystem fs = null;
final Path p = new Path("/test");
@@ -152,4 +161,87 @@ public class TestWebHDFSForHA {
}
}
}
-}
\ No newline at end of file
+
+ @Test
+ public void testMultipleNamespacesConfigured() throws Exception {
+ Configuration conf = DFSTestUtil.newHAConfiguration(LOGICAL_NAME);
+ MiniDFSCluster cluster = null;
+ WebHdfsFileSystem fs = null;
+
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).nnTopology(topo)
+ .numDataNodes(1).build();
+
+ HATestUtil.setFailoverConfigurations(cluster, conf, LOGICAL_NAME);
+
+ cluster.waitActive();
+ DFSTestUtil.addHAConfiguration(conf, LOGICAL_NAME + "remote");
+ DFSTestUtil.setFakeHttpAddresses(conf, LOGICAL_NAME + "remote");
+
+ fs = (WebHdfsFileSystem)FileSystem.get(WEBHDFS_URI, conf);
+ Assert.assertEquals(2, fs.getResolvedNNAddr().length);
+ } finally {
+ IOUtils.cleanup(null, fs);
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ /**
+ * Make sure the WebHdfsFileSystem will retry based on RetriableException when
+ * rpcServer is null in NamenodeWebHdfsMethods while NameNode starts up.
+ */
+ @Test (timeout=120000)
+ public void testRetryWhileNNStartup() throws Exception {
+ final Configuration conf = DFSTestUtil.newHAConfiguration(LOGICAL_NAME);
+ MiniDFSCluster cluster = null;
+ final Map<String, Boolean> resultMap = new HashMap<String, Boolean>();
+
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).nnTopology(topo)
+ .numDataNodes(0).build();
+ HATestUtil.setFailoverConfigurations(cluster, conf, LOGICAL_NAME);
+ cluster.waitActive();
+ cluster.transitionToActive(0);
+
+ final NameNode namenode = cluster.getNameNode(0);
+ final NamenodeProtocols rpcServer = namenode.getRpcServer();
+ Whitebox.setInternalState(namenode, "rpcServer", null);
+
+ new Thread() {
+ @Override
+ public void run() {
+ boolean result = false;
+ FileSystem fs = null;
+ try {
+ fs = FileSystem.get(WEBHDFS_URI, conf);
+ final Path dir = new Path("/test");
+ result = fs.mkdirs(dir);
+ } catch (IOException e) {
+ result = false;
+ } finally {
+ IOUtils.cleanup(null, fs);
+ }
+ synchronized (TestWebHDFSForHA.this) {
+ resultMap.put("mkdirs", result);
+ TestWebHDFSForHA.this.notifyAll();
+ }
+ }
+ }.start();
+
+ Thread.sleep(1000);
+ Whitebox.setInternalState(namenode, "rpcServer", rpcServer);
+ synchronized (this) {
+ while (!resultMap.containsKey("mkdirs")) {
+ this.wait();
+ }
+ Assert.assertTrue(resultMap.get("mkdirs"));
+ }
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java Tue Aug 19 23:49:39 2014
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FSDataOutput
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystemContractBaseTest;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.AppendTestUtil;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -49,6 +50,7 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.Assert;
+import org.junit.Test;
public class TestWebHdfsFileSystemContract extends FileSystemContractBaseTest {
private static final Configuration conf = new Configuration();
@@ -530,4 +532,35 @@ public class TestWebHdfsFileSystemContra
}
}
}
+
+ @Test
+ public void testAccess() throws IOException, InterruptedException {
+ Path p1 = new Path("/pathX");
+ try {
+ UserGroupInformation ugi = UserGroupInformation.createUserForTesting("alpha",
+ new String[]{"beta"});
+ WebHdfsFileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystemAs(ugi, conf,
+ WebHdfsFileSystem.SCHEME);
+
+ fs.mkdirs(p1);
+ fs.setPermission(p1, new FsPermission((short) 0444));
+ fs.access(p1, FsAction.READ);
+ try {
+ fs.access(p1, FsAction.WRITE);
+ fail("The access call should have failed.");
+ } catch (AccessControlException e) {
+ // expected
+ }
+
+ Path badPath = new Path("/bad");
+ try {
+ fs.access(badPath, FsAction.READ);
+ fail("The access call should have failed");
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+ } finally {
+ fs.delete(p1, true);
+ }
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java Tue Aug 19 23:49:39 2014
@@ -19,47 +19,63 @@
package org.apache.hadoop.hdfs.web;
import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
+import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.SIMPLE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.reset;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.verify;
+import static org.mockito.Matchers.*;
+import static org.mockito.Mockito.*;
+import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetSocketAddress;
import java.net.URI;
+import java.security.PrivilegedExceptionAction;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.web.resources.DeleteOpParam;
import org.apache.hadoop.hdfs.web.resources.GetOpParam;
import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
import org.apache.hadoop.hdfs.web.resources.PostOpParam;
import org.apache.hadoop.hdfs.web.resources.PutOpParam;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.security.token.Token;
+import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.mockito.internal.util.reflection.Whitebox;
public class TestWebHdfsTokens {
private static Configuration conf;
+ URI uri = null;
@BeforeClass
public static void setUp() {
conf = new Configuration();
SecurityUtil.setAuthenticationMethod(KERBEROS, conf);
UserGroupInformation.setConfiguration(conf);
+ UserGroupInformation.setLoginUser(
+ UserGroupInformation.createUserForTesting(
+ "LoginUser", new String[]{"supergroup"}));
}
private WebHdfsFileSystem spyWebhdfsInSecureSetup() throws IOException {
WebHdfsFileSystem fsOrig = new WebHdfsFileSystem();
fsOrig.initialize(URI.create("webhdfs://127.0.0.1:0"), conf);
WebHdfsFileSystem fs = spy(fsOrig);
- Whitebox.setInternalState(fsOrig.tokenAspect, "fs", fs);
return fs;
}
@@ -89,7 +105,7 @@ public class TestWebHdfsTokens {
}
@Test(timeout = 5000)
- public void testNoTokenForCanclToken() throws IOException {
+ public void testNoTokenForRenewToken() throws IOException {
checkNoTokenForOperation(PutOpParam.Op.RENEWDELEGATIONTOKEN);
}
@@ -139,4 +155,277 @@ public class TestWebHdfsTokens {
assertFalse(op.getRequireAuth());
}
}
+
+ @SuppressWarnings("unchecked") // for any(Token.class)
+ @Test
+ public void testLazyTokenFetchForWebhdfs() throws Exception {
+ MiniDFSCluster cluster = null;
+ WebHdfsFileSystem fs = null;
+ try {
+ final Configuration clusterConf = new HdfsConfiguration(conf);
+ SecurityUtil.setAuthenticationMethod(SIMPLE, clusterConf);
+ clusterConf.setBoolean(DFSConfigKeys
+ .DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
+
+ // trick the NN into thinking security is enabled w/o it trying
+ // to login from a keytab
+ UserGroupInformation.setConfiguration(clusterConf);
+ cluster = new MiniDFSCluster.Builder(clusterConf).numDataNodes(1).build();
+ cluster.waitActive();
+ SecurityUtil.setAuthenticationMethod(KERBEROS, clusterConf);
+ UserGroupInformation.setConfiguration(clusterConf);
+
+ uri = DFSUtil.createUri(
+ "webhdfs", cluster.getNameNode().getHttpAddress());
+ validateLazyTokenFetch(clusterConf);
+ } finally {
+ IOUtils.cleanup(null, fs);
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked") // for any(Token.class)
+ @Test
+ public void testLazyTokenFetchForSWebhdfs() throws Exception {
+ MiniDFSCluster cluster = null;
+ SWebHdfsFileSystem fs = null;
+ try {
+ final Configuration clusterConf = new HdfsConfiguration(conf);
+ SecurityUtil.setAuthenticationMethod(SIMPLE, clusterConf);
+ clusterConf.setBoolean(DFSConfigKeys
+ .DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
+ String BASEDIR = System.getProperty("test.build.dir",
+ "target/test-dir") + "/" + TestWebHdfsTokens.class.getSimpleName();
+ String keystoresDir;
+ String sslConfDir;
+
+ clusterConf.setBoolean(DFSConfigKeys.DFS_WEBHDFS_ENABLED_KEY, true);
+ clusterConf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name());
+ clusterConf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0");
+ clusterConf.set(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0");
+
+ File base = new File(BASEDIR);
+ FileUtil.fullyDelete(base);
+ base.mkdirs();
+ keystoresDir = new File(BASEDIR).getAbsolutePath();
+ sslConfDir = KeyStoreTestUtil.getClasspathDir(TestWebHdfsTokens.class);
+ KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, clusterConf, false);
+
+ // trick the NN into thinking security is enabled w/o it trying
+ // to login from a keytab
+ UserGroupInformation.setConfiguration(clusterConf);
+ cluster = new MiniDFSCluster.Builder(clusterConf).numDataNodes(1).build();
+ cluster.waitActive();
+ InetSocketAddress addr = cluster.getNameNode().getHttpsAddress();
+ String nnAddr = NetUtils.getHostPortString(addr);
+ clusterConf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, nnAddr);
+ SecurityUtil.setAuthenticationMethod(KERBEROS, clusterConf);
+ UserGroupInformation.setConfiguration(clusterConf);
+
+ uri = DFSUtil.createUri(
+ "swebhdfs", cluster.getNameNode().getHttpsAddress());
+ validateLazyTokenFetch(clusterConf);
+ } finally {
+ IOUtils.cleanup(null, fs);
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private void validateLazyTokenFetch(final Configuration clusterConf) throws Exception{
+ final String testUser = "DummyUser";
+ UserGroupInformation ugi = UserGroupInformation.createUserForTesting(
+ testUser, new String[]{"supergroup"});
+ WebHdfsFileSystem fs = ugi.doAs(new PrivilegedExceptionAction<WebHdfsFileSystem>() {
+ @Override
+ public WebHdfsFileSystem run() throws IOException {
+ return spy((WebHdfsFileSystem) FileSystem.newInstance(uri, clusterConf));
+ }
+ });
+ // verify token ops don't get a token
+ Assert.assertNull(fs.getRenewToken());
+ Token<?> token = fs.getDelegationToken(null);
+ fs.renewDelegationToken(token);
+ fs.cancelDelegationToken(token);
+ verify(fs, never()).getDelegationToken();
+ verify(fs, never()).replaceExpiredDelegationToken();
+ verify(fs, never()).setDelegationToken(any(Token.class));
+ Assert.assertNull(fs.getRenewToken());
+ reset(fs);
+
+ // verify first non-token op gets a token
+ final Path p = new Path("/f");
+ fs.create(p, (short)1).close();
+ verify(fs, times(1)).getDelegationToken();
+ verify(fs, never()).replaceExpiredDelegationToken();
+ verify(fs, times(1)).getDelegationToken(anyString());
+ verify(fs, times(1)).setDelegationToken(any(Token.class));
+ token = fs.getRenewToken();
+ Assert.assertNotNull(token);
+ Assert.assertEquals(testUser, getTokenOwner(token));
+ Assert.assertEquals(fs.getTokenKind(), token.getKind());
+ reset(fs);
+
+ // verify prior token is reused
+ fs.getFileStatus(p);
+ verify(fs, times(1)).getDelegationToken();
+ verify(fs, never()).replaceExpiredDelegationToken();
+ verify(fs, never()).getDelegationToken(anyString());
+ verify(fs, never()).setDelegationToken(any(Token.class));
+ Token<?> token2 = fs.getRenewToken();
+ Assert.assertNotNull(token2);
+ Assert.assertEquals(fs.getTokenKind(), token.getKind());
+ Assert.assertSame(token, token2);
+ reset(fs);
+
+ // verify renew of expired token fails w/o getting a new token
+ token = fs.getRenewToken();
+ fs.cancelDelegationToken(token);
+ try {
+ fs.renewDelegationToken(token);
+ Assert.fail("should have failed");
+ } catch (InvalidToken it) {
+ } catch (Exception ex) {
+ Assert.fail("wrong exception:"+ex);
+ }
+ verify(fs, never()).getDelegationToken();
+ verify(fs, never()).replaceExpiredDelegationToken();
+ verify(fs, never()).getDelegationToken(anyString());
+ verify(fs, never()).setDelegationToken(any(Token.class));
+ token2 = fs.getRenewToken();
+ Assert.assertNotNull(token2);
+ Assert.assertEquals(fs.getTokenKind(), token.getKind());
+ Assert.assertSame(token, token2);
+ reset(fs);
+
+ // verify cancel of expired token fails w/o getting a new token
+ try {
+ fs.cancelDelegationToken(token);
+ Assert.fail("should have failed");
+ } catch (InvalidToken it) {
+ } catch (Exception ex) {
+ Assert.fail("wrong exception:"+ex);
+ }
+ verify(fs, never()).getDelegationToken();
+ verify(fs, never()).replaceExpiredDelegationToken();
+ verify(fs, never()).getDelegationToken(anyString());
+ verify(fs, never()).setDelegationToken(any(Token.class));
+ token2 = fs.getRenewToken();
+ Assert.assertNotNull(token2);
+ Assert.assertEquals(fs.getTokenKind(), token.getKind());
+ Assert.assertSame(token, token2);
+ reset(fs);
+
+ // verify an expired token is replaced with a new token
+ fs.open(p).close();
+ verify(fs, times(2)).getDelegationToken(); // first bad, then good
+ verify(fs, times(1)).replaceExpiredDelegationToken();
+ verify(fs, times(1)).getDelegationToken(null);
+ verify(fs, times(1)).setDelegationToken(any(Token.class));
+ token2 = fs.getRenewToken();
+ Assert.assertNotNull(token2);
+ Assert.assertNotSame(token, token2);
+ Assert.assertEquals(fs.getTokenKind(), token.getKind());
+ Assert.assertEquals(testUser, getTokenOwner(token2));
+ reset(fs);
+
+ // verify with open because it's a little different in how it
+ // opens connections
+ fs.cancelDelegationToken(fs.getRenewToken());
+ InputStream is = fs.open(p);
+ is.read();
+ is.close();
+ verify(fs, times(2)).getDelegationToken(); // first bad, then good
+ verify(fs, times(1)).replaceExpiredDelegationToken();
+ verify(fs, times(1)).getDelegationToken(null);
+ verify(fs, times(1)).setDelegationToken(any(Token.class));
+ token2 = fs.getRenewToken();
+ Assert.assertNotNull(token2);
+ Assert.assertNotSame(token, token2);
+ Assert.assertEquals(fs.getTokenKind(), token.getKind());
+ Assert.assertEquals(testUser, getTokenOwner(token2));
+ reset(fs);
+
+ // verify fs close cancels the token
+ fs.close();
+ verify(fs, never()).getDelegationToken();
+ verify(fs, never()).replaceExpiredDelegationToken();
+ verify(fs, never()).getDelegationToken(anyString());
+ verify(fs, never()).setDelegationToken(any(Token.class));
+ verify(fs, times(1)).cancelDelegationToken(eq(token2));
+
+ // add a token to ugi for a new fs, verify it uses that token
+ token = fs.getDelegationToken(null);
+ ugi.addToken(token);
+ fs = ugi.doAs(new PrivilegedExceptionAction<WebHdfsFileSystem>() {
+ @Override
+ public WebHdfsFileSystem run() throws IOException {
+ return spy((WebHdfsFileSystem) FileSystem.newInstance(uri, clusterConf));
+ }
+ });
+ Assert.assertNull(fs.getRenewToken());
+ fs.getFileStatus(new Path("/"));
+ verify(fs, times(1)).getDelegationToken();
+ verify(fs, never()).replaceExpiredDelegationToken();
+ verify(fs, never()).getDelegationToken(anyString());
+ verify(fs, times(1)).setDelegationToken(eq(token));
+ token2 = fs.getRenewToken();
+ Assert.assertNotNull(token2);
+ Assert.assertEquals(fs.getTokenKind(), token.getKind());
+ Assert.assertSame(token, token2);
+ reset(fs);
+
+ // verify it reuses the prior ugi token
+ fs.getFileStatus(new Path("/"));
+ verify(fs, times(1)).getDelegationToken();
+ verify(fs, never()).replaceExpiredDelegationToken();
+ verify(fs, never()).getDelegationToken(anyString());
+ verify(fs, never()).setDelegationToken(any(Token.class));
+ token2 = fs.getRenewToken();
+ Assert.assertNotNull(token2);
+ Assert.assertEquals(fs.getTokenKind(), token.getKind());
+ Assert.assertSame(token, token2);
+ reset(fs);
+
+ // verify an expired ugi token is NOT replaced with a new token
+ fs.cancelDelegationToken(token);
+ for (int i=0; i<2; i++) {
+ try {
+ fs.getFileStatus(new Path("/"));
+ Assert.fail("didn't fail");
+ } catch (InvalidToken it) {
+ } catch (Exception ex) {
+ Assert.fail("wrong exception:"+ex);
+ }
+ verify(fs, times(1)).getDelegationToken();
+ verify(fs, times(1)).replaceExpiredDelegationToken();
+ verify(fs, never()).getDelegationToken(anyString());
+ verify(fs, never()).setDelegationToken(any(Token.class));
+ token2 = fs.getRenewToken();
+ Assert.assertNotNull(token2);
+ Assert.assertEquals(fs.getTokenKind(), token.getKind());
+ Assert.assertSame(token, token2);
+ reset(fs);
+ }
+
+ // verify fs close does NOT cancel the ugi token
+ fs.close();
+ verify(fs, never()).getDelegationToken();
+ verify(fs, never()).replaceExpiredDelegationToken();
+ verify(fs, never()).getDelegationToken(anyString());
+ verify(fs, never()).setDelegationToken(any(Token.class));
+ verify(fs, never()).cancelDelegationToken(any(Token.class));
+ }
+
+ private String getTokenOwner(Token<?> token) throws IOException {
+ // webhdfs doesn't register properly with the class loader
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ Token<?> clone = new Token(token);
+ clone.setKind(DelegationTokenIdentifier.HDFS_DELEGATION_KIND);
+ return clone.decodeIdentifier().getUser().getUserName();
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java Tue Aug 19 23:49:39 2014
@@ -31,6 +31,7 @@ import java.util.Arrays;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@@ -40,6 +41,7 @@ import org.apache.hadoop.hdfs.web.resour
import org.apache.hadoop.hdfs.web.resources.PutOpParam;
import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam;
import org.apache.hadoop.hdfs.web.resources.UserParam;
+import org.apache.hadoop.hdfs.web.resources.FsActionParam;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
@@ -283,6 +285,28 @@ public class TestWebHdfsUrl {
},
fileStatusUrl);
}
+
+ @Test(timeout=60000)
+ public void testCheckAccessUrl() throws IOException {
+ Configuration conf = new Configuration();
+
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser("test-user");
+ UserGroupInformation.setLoginUser(ugi);
+
+ WebHdfsFileSystem webhdfs = getWebHdfsFileSystem(ugi, conf);
+ Path fsPath = new Path("/p1");
+
+ URL checkAccessUrl = webhdfs.toUrl(GetOpParam.Op.CHECKACCESS,
+ fsPath, new FsActionParam(FsAction.READ_WRITE));
+ checkQueryParams(
+ new String[]{
+ GetOpParam.Op.CHECKACCESS.toQueryString(),
+ new UserParam(ugi.getShortUserName()).toString(),
+ FsActionParam.NAME + "=" + FsAction.READ_WRITE.SYMBOL
+ },
+ checkAccessUrl);
+ }
private void checkQueryParams(String[] expected, URL url) {
Arrays.sort(expected);
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/WebHdfsTestUtil.java Tue Aug 19 23:49:39 2014
@@ -94,10 +94,4 @@ public class WebHdfsTestUtil {
Assert.assertEquals(expectedResponseCode, conn.getResponseCode());
return WebHdfsFileSystem.jsonParse(conn, false);
}
-
- public static FSDataOutputStream write(final WebHdfsFileSystem webhdfs,
- final HttpOpParam.Op op, final HttpURLConnection conn,
- final int bufferSize) throws IOException {
- return webhdfs.write(op, conn, bufferSize);
- }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java Tue Aug 19 23:49:39 2014
@@ -20,14 +20,19 @@ package org.apache.hadoop.hdfs.web.resou
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
+import java.io.IOException;
import java.util.Arrays;
+import java.util.EnumSet;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.XAttrCodec;
+import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -346,4 +351,52 @@ public class TestParam {
}
}
+ @Test
+ public void testXAttrNameParam() {
+ final XAttrNameParam p = new XAttrNameParam("user.a1");
+ Assert.assertEquals(p.getXAttrName(), "user.a1");
+ }
+
+ @Test
+ public void testXAttrValueParam() throws IOException {
+ final XAttrValueParam p = new XAttrValueParam("0x313233");
+ Assert.assertArrayEquals(p.getXAttrValue(),
+ XAttrCodec.decodeValue("0x313233"));
+ }
+
+ @Test
+ public void testXAttrEncodingParam() {
+ final XAttrEncodingParam p = new XAttrEncodingParam(XAttrCodec.BASE64);
+ Assert.assertEquals(p.getEncoding(), XAttrCodec.BASE64);
+ final XAttrEncodingParam p1 = new XAttrEncodingParam(p.getValueString());
+ Assert.assertEquals(p1.getEncoding(), XAttrCodec.BASE64);
+ }
+
+ @Test
+ public void testXAttrSetFlagParam() {
+ EnumSet<XAttrSetFlag> flag = EnumSet.of(
+ XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE);
+ final XAttrSetFlagParam p = new XAttrSetFlagParam(flag);
+ Assert.assertEquals(p.getFlag(), flag);
+ final XAttrSetFlagParam p1 = new XAttrSetFlagParam(p.getValueString());
+ Assert.assertEquals(p1.getFlag(), flag);
+ }
+
+ @Test
+ public void testRenameOptionSetParam() {
+ final RenameOptionSetParam p = new RenameOptionSetParam(
+ Options.Rename.OVERWRITE, Options.Rename.NONE);
+ final RenameOptionSetParam p1 = new RenameOptionSetParam(
+ p.getValueString());
+ Assert.assertEquals(p1.getValue(), EnumSet.of(
+ Options.Rename.OVERWRITE, Options.Rename.NONE));
+ }
+
+ @Test
+ public void testSnapshotNameParam() {
+ final OldSnapshotNameParam s1 = new OldSnapshotNameParam("s1");
+ final SnapshotNameParam s2 = new SnapshotNameParam("s2");
+ Assert.assertEquals("s1", s1.getValue());
+ Assert.assertEquals("s2", s2.getValue());
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java Tue Aug 19 23:49:39 2014
@@ -54,11 +54,26 @@ public class TestNetworkTopology {
DFSTestUtil.getDatanodeDescriptor("4.4.4.4", "/d1/r2"),
DFSTestUtil.getDatanodeDescriptor("5.5.5.5", "/d1/r2"),
DFSTestUtil.getDatanodeDescriptor("6.6.6.6", "/d2/r3"),
- DFSTestUtil.getDatanodeDescriptor("7.7.7.7", "/d2/r3")
+ DFSTestUtil.getDatanodeDescriptor("7.7.7.7", "/d2/r3"),
+ DFSTestUtil.getDatanodeDescriptor("8.8.8.8", "/d2/r3"),
+ DFSTestUtil.getDatanodeDescriptor("9.9.9.9", "/d3/r1"),
+ DFSTestUtil.getDatanodeDescriptor("10.10.10.10", "/d3/r1"),
+ DFSTestUtil.getDatanodeDescriptor("11.11.11.11", "/d3/r1"),
+ DFSTestUtil.getDatanodeDescriptor("12.12.12.12", "/d3/r2"),
+ DFSTestUtil.getDatanodeDescriptor("13.13.13.13", "/d3/r2"),
+ DFSTestUtil.getDatanodeDescriptor("14.14.14.14", "/d4/r1"),
+ DFSTestUtil.getDatanodeDescriptor("15.15.15.15", "/d4/r1"),
+ DFSTestUtil.getDatanodeDescriptor("16.16.16.16", "/d4/r1"),
+ DFSTestUtil.getDatanodeDescriptor("17.17.17.17", "/d4/r1"),
+ DFSTestUtil.getDatanodeDescriptor("18.18.18.18", "/d4/r1"),
+ DFSTestUtil.getDatanodeDescriptor("19.19.19.19", "/d4/r1"),
+ DFSTestUtil.getDatanodeDescriptor("20.20.20.20", "/d4/r1"),
};
for (int i = 0; i < dataNodes.length; i++) {
cluster.add(dataNodes[i]);
}
+ dataNodes[9].setDecommissioned();
+ dataNodes[10].setDecommissioned();
}
@Test
@@ -99,7 +114,7 @@ public class TestNetworkTopology {
@Test
public void testRacks() throws Exception {
- assertEquals(cluster.getNumOfRacks(), 3);
+ assertEquals(cluster.getNumOfRacks(), 6);
assertTrue(cluster.isOnSameRack(dataNodes[0], dataNodes[1]));
assertFalse(cluster.isOnSameRack(dataNodes[1], dataNodes[2]));
assertTrue(cluster.isOnSameRack(dataNodes[2], dataNodes[3]));
@@ -117,23 +132,40 @@ public class TestNetworkTopology {
}
@Test
- public void testPseudoSortByDistance() throws Exception {
+ public void testSortByDistance() throws Exception {
DatanodeDescriptor[] testNodes = new DatanodeDescriptor[3];
// array contains both local node & local rack node
testNodes[0] = dataNodes[1];
testNodes[1] = dataNodes[2];
testNodes[2] = dataNodes[0];
- cluster.pseudoSortByDistance(dataNodes[0], testNodes );
+ cluster.sortByDistance(dataNodes[0], testNodes,
+ testNodes.length, 0xDEADBEEF, false);
assertTrue(testNodes[0] == dataNodes[0]);
assertTrue(testNodes[1] == dataNodes[1]);
assertTrue(testNodes[2] == dataNodes[2]);
+ // array contains both local node & local rack node & decommissioned node
+ DatanodeDescriptor[] dtestNodes = new DatanodeDescriptor[5];
+ dtestNodes[0] = dataNodes[8];
+ dtestNodes[1] = dataNodes[12];
+ dtestNodes[2] = dataNodes[11];
+ dtestNodes[3] = dataNodes[9];
+ dtestNodes[4] = dataNodes[10];
+ cluster.sortByDistance(dataNodes[8], dtestNodes,
+ dtestNodes.length - 2, 0xDEADBEEF, false);
+ assertTrue(dtestNodes[0] == dataNodes[8]);
+ assertTrue(dtestNodes[1] == dataNodes[11]);
+ assertTrue(dtestNodes[2] == dataNodes[12]);
+ assertTrue(dtestNodes[3] == dataNodes[9]);
+ assertTrue(dtestNodes[4] == dataNodes[10]);
+
// array contains local node
testNodes[0] = dataNodes[1];
testNodes[1] = dataNodes[3];
testNodes[2] = dataNodes[0];
- cluster.pseudoSortByDistance(dataNodes[0], testNodes );
+ cluster.sortByDistance(dataNodes[0], testNodes,
+ testNodes.length, 0xDEADBEEF, false);
assertTrue(testNodes[0] == dataNodes[0]);
assertTrue(testNodes[1] == dataNodes[1]);
assertTrue(testNodes[2] == dataNodes[3]);
@@ -142,21 +174,74 @@ public class TestNetworkTopology {
testNodes[0] = dataNodes[5];
testNodes[1] = dataNodes[3];
testNodes[2] = dataNodes[1];
- cluster.pseudoSortByDistance(dataNodes[0], testNodes );
+ cluster.sortByDistance(dataNodes[0], testNodes,
+ testNodes.length, 0xDEADBEEF, false);
assertTrue(testNodes[0] == dataNodes[1]);
assertTrue(testNodes[1] == dataNodes[3]);
assertTrue(testNodes[2] == dataNodes[5]);
-
+
// array contains local rack node which happens to be in position 0
testNodes[0] = dataNodes[1];
testNodes[1] = dataNodes[5];
testNodes[2] = dataNodes[3];
- cluster.pseudoSortByDistance(dataNodes[0], testNodes );
- // peudoSortByDistance does not take the "data center" layer into consideration
+ cluster.sortByDistance(dataNodes[0], testNodes,
+ testNodes.length, 0xDEADBEEF, false);
+ assertTrue(testNodes[0] == dataNodes[1]);
+ assertTrue(testNodes[1] == dataNodes[3]);
+ assertTrue(testNodes[2] == dataNodes[5]);
+
+ // Same as previous, but with a different random seed to test randomization
+ testNodes[0] = dataNodes[1];
+ testNodes[1] = dataNodes[5];
+ testNodes[2] = dataNodes[3];
+ cluster.sortByDistance(dataNodes[0], testNodes,
+ testNodes.length, 0xDEAD, false);
+ // sortByDistance does not take the "data center" layer into consideration
// and it doesn't sort by getDistance, so 1, 5, 3 is also valid here
assertTrue(testNodes[0] == dataNodes[1]);
assertTrue(testNodes[1] == dataNodes[5]);
assertTrue(testNodes[2] == dataNodes[3]);
+
+ // Array is just local rack nodes
+ // Expect a random first node depending on the seed (normally the block ID).
+ DatanodeDescriptor first = null;
+ boolean foundRandom = false;
+ for (int i=5; i<=7; i++) {
+ testNodes[0] = dataNodes[5];
+ testNodes[1] = dataNodes[6];
+ testNodes[2] = dataNodes[7];
+ cluster.sortByDistance(dataNodes[i], testNodes,
+ testNodes.length, 0xBEADED+i, false);
+ if (first == null) {
+ first = testNodes[0];
+ } else {
+ if (first != testNodes[0]) {
+ foundRandom = true;
+ break;
+ }
+ }
+ }
+ assertTrue("Expected to find a different first location", foundRandom);
+ // Array of rack local nodes with randomizeBlockLocationsPerBlock set to
+ // true
+ // Expect random order of block locations for same block
+ first = null;
+ for (int i = 1; i <= 4; i++) {
+ testNodes[0] = dataNodes[13];
+ testNodes[1] = dataNodes[14];
+ testNodes[2] = dataNodes[15];
+ cluster.sortByDistance(dataNodes[15 + i], testNodes, testNodes.length,
+ 0xBEADED, true);
+ if (first == null) {
+ first = testNodes[0];
+ } else {
+ if (first != testNodes[0]) {
+ foundRandom = true;
+ break;
+ }
+ }
+ }
+ assertTrue("Expected to find a different first location", foundRandom);
}
@Test
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermissionSymlinks.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermissionSymlinks.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermissionSymlinks.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermissionSymlinks.java Tue Aug 19 23:49:39 2014
@@ -27,6 +27,7 @@ import static org.junit.Assert.assertTru
import static org.junit.Assert.fail;
import java.io.IOException;
+import java.io.FileNotFoundException;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
@@ -39,6 +40,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestWrapper;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
@@ -393,4 +395,37 @@ public class TestPermissionSymlinks {
GenericTestUtils.assertExceptionContains("Permission denied", e);
}
}
+
+ @Test
+ public void testAccess() throws Exception {
+ fs.setPermission(target, new FsPermission((short) 0002));
+ fs.setAcl(target, Arrays.asList(
+ aclEntry(ACCESS, USER, ALL),
+ aclEntry(ACCESS, GROUP, NONE),
+ aclEntry(ACCESS, USER, user.getShortUserName(), WRITE),
+ aclEntry(ACCESS, OTHER, WRITE)));
+ FileContext myfc = user.doAs(new PrivilegedExceptionAction<FileContext>() {
+ @Override
+ public FileContext run() throws IOException {
+ return FileContext.getFileContext(conf);
+ }
+ });
+
+ // Path to targetChild via symlink
+ myfc.access(link, FsAction.WRITE);
+ try {
+ myfc.access(link, FsAction.ALL);
+ fail("The access call should have failed.");
+ } catch (AccessControlException e) {
+ // expected
+ }
+
+ Path badPath = new Path(link, "bad");
+ try {
+ myfc.access(badPath, FsAction.READ);
+ fail("The access call should have failed");
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestRefreshUserMappings.java Tue Aug 19 23:49:39 2014
@@ -41,6 +41,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.tools.DFSAdmin;
import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.security.authorize.DefaultImpersonationProvider;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.junit.After;
import org.junit.Before;
@@ -150,8 +151,10 @@ public class TestRefreshUserMappings {
final String [] GROUP_NAMES2 = new String [] {"gr3" , "gr4"};
//keys in conf
- String userKeyGroups = ProxyUsers.getProxySuperuserGroupConfKey(SUPER_USER);
- String userKeyHosts = ProxyUsers.getProxySuperuserIpConfKey (SUPER_USER);
+ String userKeyGroups = DefaultImpersonationProvider.getTestProvider().
+ getProxySuperuserGroupConfKey(SUPER_USER);
+ String userKeyHosts = DefaultImpersonationProvider.getTestProvider().
+ getProxySuperuserIpConfKey (SUPER_USER);
config.set(userKeyGroups, "gr3,gr4,gr5"); // superuser can proxy for this group
config.set(userKeyHosts,"127.0.0.1");
Modified: hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
Binary files - no diff available.