You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by wa...@apache.org on 2014/07/30 07:53:21 UTC
svn commit: r1614545 - in
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt
src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithXAttrs.java
src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsWithXAttrs.java
Author: wang
Date: Wed Jul 30 05:53:20 2014
New Revision: 1614545
URL: http://svn.apache.org/r1614545
Log:
HDFS-6665. Add tests for XAttrs in combination with viewfs. Contributed by Stephen Chu.
Added:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithXAttrs.java (with props)
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsWithXAttrs.java (with props)
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1614545&r1=1614544&r2=1614545&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Jul 30 05:53:20 2014
@@ -326,6 +326,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6739. Add getDatanodeStorageReport to ClientProtocol. (szetszwo)
+ HDFS-6665. Add tests for XAttrs in combination with viewfs.
+ (Stephen Chu via wang)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
Added: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithXAttrs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithXAttrs.java?rev=1614545&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithXAttrs.java (added)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithXAttrs.java Wed Jul 30 05:53:20 2014
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.viewfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileSystemTestHelper;
+import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Verify XAttrs through ViewFileSystem functionality.
+ */
+public class TestViewFileSystemWithXAttrs {
+
+ private static MiniDFSCluster cluster;
+ private static Configuration clusterConf = new Configuration();
+ private static FileSystem fHdfs;
+ private static FileSystem fHdfs2;
+ private FileSystem fsView;
+ private Configuration fsViewConf;
+ private FileSystem fsTarget, fsTarget2;
+ private Path targetTestRoot, targetTestRoot2, mountOnNn1, mountOnNn2;
+ private FileSystemTestHelper fileSystemTestHelper =
+ new FileSystemTestHelper("/tmp/TestViewFileSystemWithXAttrs");
+
+ // XAttrs
+ protected static final String name1 = "user.a1";
+ protected static final byte[] value1 = {0x31, 0x32, 0x33};
+ protected static final String name2 = "user.a2";
+ protected static final byte[] value2 = {0x37, 0x38, 0x39};
+
+ @BeforeClass
+ public static void clusterSetupAtBeginning() throws IOException {
+ cluster = new MiniDFSCluster.Builder(clusterConf)
+ .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2))
+ .numDataNodes(2)
+ .build();
+ cluster.waitClusterUp();
+
+ fHdfs = cluster.getFileSystem(0);
+ fHdfs2 = cluster.getFileSystem(1);
+ }
+
+ @AfterClass
+ public static void ClusterShutdownAtEnd() throws Exception {
+ cluster.shutdown();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ fsTarget = fHdfs;
+ fsTarget2 = fHdfs2;
+ targetTestRoot = fileSystemTestHelper.getAbsoluteTestRootPath(fsTarget);
+ targetTestRoot2 = fileSystemTestHelper.getAbsoluteTestRootPath(fsTarget2);
+
+ fsTarget.delete(targetTestRoot, true);
+ fsTarget2.delete(targetTestRoot2, true);
+ fsTarget.mkdirs(targetTestRoot);
+ fsTarget2.mkdirs(targetTestRoot2);
+
+ fsViewConf = ViewFileSystemTestSetup.createConfig();
+ setupMountPoints();
+ fsView = FileSystem.get(FsConstants.VIEWFS_URI, fsViewConf);
+ }
+
+ private void setupMountPoints() {
+ mountOnNn1 = new Path("/mountOnNn1");
+ mountOnNn2 = new Path("/mountOnNn2");
+ ConfigUtil.addLink(fsViewConf, mountOnNn1.toString(),
+ targetTestRoot.toUri());
+ ConfigUtil.addLink(fsViewConf, mountOnNn2.toString(),
+ targetTestRoot2.toUri());
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ fsTarget.delete(fileSystemTestHelper.getTestRootPath(fsTarget), true);
+ fsTarget2.delete(fileSystemTestHelper.getTestRootPath(fsTarget2), true);
+ }
+
+ /**
+ * Verify a ViewFileSystem wrapped over multiple federated NameNodes will
+ * dispatch the XAttr operations to the correct NameNode.
+ */
+ @Test
+ public void testXAttrOnMountEntry() throws Exception {
+ // Set XAttrs on the first namespace and verify they are correct
+ fsView.setXAttr(mountOnNn1, name1, value1);
+ fsView.setXAttr(mountOnNn1, name2, value2);
+ assertEquals(2, fsView.getXAttrs(mountOnNn1).size());
+ assertArrayEquals(value1, fsView.getXAttr(mountOnNn1, name1));
+ assertArrayEquals(value2, fsView.getXAttr(mountOnNn1, name2));
+ // Double-check by getting the XAttrs using FileSystem
+ // instead of ViewFileSystem
+ assertArrayEquals(value1, fHdfs.getXAttr(targetTestRoot, name1));
+ assertArrayEquals(value2, fHdfs.getXAttr(targetTestRoot, name2));
+
+ // Paranoid check: verify the other namespace does not
+ // have XAttrs set on the same path.
+ assertEquals(0, fsView.getXAttrs(mountOnNn2).size());
+ assertEquals(0, fHdfs2.getXAttrs(targetTestRoot2).size());
+
+ // Remove the XAttr entries on the first namespace
+ fsView.removeXAttr(mountOnNn1, name1);
+ fsView.removeXAttr(mountOnNn1, name2);
+ assertEquals(0, fsView.getXAttrs(mountOnNn1).size());
+ assertEquals(0, fHdfs.getXAttrs(targetTestRoot).size());
+
+ // Now set XAttrs on the second namespace
+ fsView.setXAttr(mountOnNn2, name1, value1);
+ fsView.setXAttr(mountOnNn2, name2, value2);
+ assertEquals(2, fsView.getXAttrs(mountOnNn2).size());
+ assertArrayEquals(value1, fsView.getXAttr(mountOnNn2, name1));
+ assertArrayEquals(value2, fsView.getXAttr(mountOnNn2, name2));
+ assertArrayEquals(value1, fHdfs2.getXAttr(targetTestRoot2, name1));
+ assertArrayEquals(value2, fHdfs2.getXAttr(targetTestRoot2, name2));
+
+ fsView.removeXAttr(mountOnNn2, name1);
+ fsView.removeXAttr(mountOnNn2, name2);
+ assertEquals(0, fsView.getXAttrs(mountOnNn2).size());
+ assertEquals(0, fHdfs2.getXAttrs(targetTestRoot2).size());
+ }
+}
Propchange: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemWithXAttrs.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsWithXAttrs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsWithXAttrs.java?rev=1614545&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsWithXAttrs.java (added)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsWithXAttrs.java Wed Jul 30 05:53:20 2014
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.viewfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileContextTestHelper;
+import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Verify XAttrs through ViewFs functionality.
+ */
+public class TestViewFsWithXAttrs {
+
+ private static MiniDFSCluster cluster;
+ private static Configuration clusterConf = new Configuration();
+ private static FileContext fc, fc2;
+ private FileContext fcView, fcTarget, fcTarget2;
+ private Configuration fsViewConf;
+ private Path targetTestRoot, targetTestRoot2, mountOnNn1, mountOnNn2;
+ private FileContextTestHelper fileContextTestHelper =
+ new FileContextTestHelper("/tmp/TestViewFsWithXAttrs");
+
+ // XAttrs
+ protected static final String name1 = "user.a1";
+ protected static final byte[] value1 = {0x31, 0x32, 0x33};
+ protected static final String name2 = "user.a2";
+ protected static final byte[] value2 = {0x37, 0x38, 0x39};
+
+ @BeforeClass
+ public static void clusterSetupAtBeginning() throws IOException {
+ cluster = new MiniDFSCluster.Builder(clusterConf)
+ .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2))
+ .numDataNodes(2)
+ .build();
+ cluster.waitClusterUp();
+
+ fc = FileContext.getFileContext(cluster.getURI(0), clusterConf);
+ fc2 = FileContext.getFileContext(cluster.getURI(1), clusterConf);
+ }
+
+ @AfterClass
+ public static void ClusterShutdownAtEnd() throws Exception {
+ cluster.shutdown();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ fcTarget = fc;
+ fcTarget2 = fc2;
+ targetTestRoot = fileContextTestHelper.getAbsoluteTestRootPath(fc);
+ targetTestRoot2 = fileContextTestHelper.getAbsoluteTestRootPath(fc2);
+
+ fcTarget.delete(targetTestRoot, true);
+ fcTarget2.delete(targetTestRoot2, true);
+ fcTarget.mkdir(targetTestRoot, new FsPermission((short) 0750), true);
+ fcTarget2.mkdir(targetTestRoot2, new FsPermission((short) 0750), true);
+
+ fsViewConf = ViewFileSystemTestSetup.createConfig();
+ setupMountPoints();
+ fcView = FileContext.getFileContext(FsConstants.VIEWFS_URI, fsViewConf);
+ }
+
+ private void setupMountPoints() {
+ mountOnNn1 = new Path("/mountOnNn1");
+ mountOnNn2 = new Path("/mountOnNn2");
+ ConfigUtil.addLink(fsViewConf, mountOnNn1.toString(), targetTestRoot.toUri());
+ ConfigUtil.addLink(fsViewConf, mountOnNn2.toString(), targetTestRoot2.toUri());
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ fcTarget.delete(fileContextTestHelper.getTestRootPath(fcTarget), true);
+ fcTarget2.delete(fileContextTestHelper.getTestRootPath(fcTarget2), true);
+ }
+
+ /**
+ * Verify a ViewFs wrapped over multiple federated NameNodes will
+ * dispatch the XAttr operations to the correct NameNode.
+ */
+ @Test
+ public void testXAttrOnMountEntry() throws Exception {
+ // Set XAttrs on the first namespace and verify they are correct
+ fcView.setXAttr(mountOnNn1, name1, value1);
+ fcView.setXAttr(mountOnNn1, name2, value2);
+ assertEquals(2, fcView.getXAttrs(mountOnNn1).size());
+ assertArrayEquals(value1, fcView.getXAttr(mountOnNn1, name1));
+ assertArrayEquals(value2, fcView.getXAttr(mountOnNn1, name2));
+ // Double-check by getting the XAttrs using FileSystem
+ // instead of ViewFs
+ assertArrayEquals(value1, fc.getXAttr(targetTestRoot, name1));
+ assertArrayEquals(value2, fc.getXAttr(targetTestRoot, name2));
+
+ // Paranoid check: verify the other namespace does not
+ // have XAttrs set on the same path.
+ assertEquals(0, fcView.getXAttrs(mountOnNn2).size());
+ assertEquals(0, fc2.getXAttrs(targetTestRoot2).size());
+
+ // Remove the XAttr entries on the first namespace
+ fcView.removeXAttr(mountOnNn1, name1);
+ fcView.removeXAttr(mountOnNn1, name2);
+ assertEquals(0, fcView.getXAttrs(mountOnNn1).size());
+ assertEquals(0, fc.getXAttrs(targetTestRoot).size());
+
+ // Now set XAttrs on the second namespace
+ fcView.setXAttr(mountOnNn2, name1, value1);
+ fcView.setXAttr(mountOnNn2, name2, value2);
+ assertEquals(2, fcView.getXAttrs(mountOnNn2).size());
+ assertArrayEquals(value1, fcView.getXAttr(mountOnNn2, name1));
+ assertArrayEquals(value2, fcView.getXAttr(mountOnNn2, name2));
+ assertArrayEquals(value1, fc2.getXAttr(targetTestRoot2, name1));
+ assertArrayEquals(value2, fc2.getXAttr(targetTestRoot2, name2));
+
+ fcView.removeXAttr(mountOnNn2, name1);
+ fcView.removeXAttr(mountOnNn2, name2);
+ assertEquals(0, fcView.getXAttrs(mountOnNn2).size());
+ assertEquals(0, fc2.getXAttrs(targetTestRoot2).size());
+ }
+}
Propchange: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFsWithXAttrs.java
------------------------------------------------------------------------------
svn:eol-style = native