You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xi...@apache.org on 2016/12/08 20:43:14 UTC
hadoop git commit: HDFS-11197. Listing encryption zones fails when
deleting a EZ that is on a snapshotted directory. Contributed by Wellington
Chevreuil.
Repository: hadoop
Updated Branches:
refs/heads/trunk a5a55a54a -> 401c73187
HDFS-11197. Listing encryption zones fails when deleting a EZ that is on a snapshotted directory. Contributed by Wellington Chevreuil.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/401c7318
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/401c7318
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/401c7318
Branch: refs/heads/trunk
Commit: 401c7318723d8d62c7fc29728f7f4e8d336b4d2f
Parents: a5a55a5
Author: Xiao Chen <xi...@apache.org>
Authored: Thu Dec 8 12:40:20 2016 -0800
Committer: Xiao Chen <xi...@apache.org>
Committed: Thu Dec 8 12:40:20 2016 -0800
----------------------------------------------------------------------
.../server/namenode/EncryptionZoneManager.java | 8 +-
.../apache/hadoop/cli/TestCryptoAdminCLI.java | 3 +-
.../namenode/TestEncryptionZoneManager.java | 138 +++++++++++++++++++
.../src/test/resources/testCryptoConf.xml | 98 ++++++++++++-
4 files changed, 242 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/401c7318/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index d23963d..6dff62b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -371,8 +371,12 @@ public class EncryptionZoneManager {
contain a reference INode.
*/
final String pathName = getFullPathName(ezi);
- INodesInPath iip = dir.getINodesInPath(pathName, DirOp.READ_LINK);
- INode lastINode = iip.getLastINode();
+ INode inode = dir.getInode(ezi.getINodeId());
+ INode lastINode = null;
+ if (inode.getParent() != null || inode.isRoot()) {
+ INodesInPath iip = dir.getINodesInPath(pathName, DirOp.READ_LINK);
+ lastINode = iip.getLastINode();
+ }
if (lastINode == null || lastINode.getId() != ezi.getINodeId()) {
continue;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/401c7318/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
index 99a7c2a..afc668c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
@@ -63,6 +63,7 @@ public class TestCryptoAdminCLI extends CLITestHelperDFS {
conf.setClass(PolicyProvider.POLICY_PROVIDER_CONFIG,
HDFSPolicyProvider.class, PolicyProvider.class);
conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
+ conf.setLong(CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY, 10);
tmpDir = GenericTestUtils.getTestDir(UUID.randomUUID().toString());
final Path jksPath = new Path(tmpDir.toString(), "test.jks");
@@ -127,7 +128,7 @@ public class TestCryptoAdminCLI extends CLITestHelperDFS {
}
private class TestConfigFileParserCryptoAdmin extends
- CLITestHelper.TestConfigFileParser {
+ CLITestHelperDFS.TestConfigFileParserDFS {
@Override
public void endElement(String uri, String localName, String qName)
throws SAXException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/401c7318/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEncryptionZoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEncryptionZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEncryptionZoneManager.java
new file mode 100644
index 0000000..728e15b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEncryptionZoneManager.java
@@ -0,0 +1,138 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.CipherSuite;
+import org.apache.hadoop.crypto.CryptoProtocolVersion;
+import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.hdfs.protocol.EncryptionZone;
+import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test class for EncryptionZoneManager methods. Added tests for
+ * listEncryptionZones method, for cases where inode can and cannot have a
+ * parent inode.
+ */
+public class TestEncryptionZoneManager {
+
+ private FSDirectory mockedDir;
+ private INodesInPath mockedINodesInPath;
+ private INodeDirectory firstINode;
+ private INodeDirectory secondINode;
+ private INodeDirectory rootINode;
+ private PermissionStatus defaultPermission;
+ private EncryptionZoneManager ezManager;
+
+ @Before
+ public void setup() {
+ this.mockedDir = mock(FSDirectory.class);
+ this.mockedINodesInPath = mock(INodesInPath.class);
+ this.defaultPermission = new PermissionStatus("test", "test",
+ new FsPermission((short) 755));
+ this.rootINode =
+ new INodeDirectory(0L, "".getBytes(), defaultPermission,
+ System.currentTimeMillis());
+ this.firstINode =
+ new INodeDirectory(1L, "first".getBytes(), defaultPermission,
+ System.currentTimeMillis());
+ this.secondINode =
+ new INodeDirectory(2L, "second".getBytes(), defaultPermission,
+ System.currentTimeMillis());
+ when(this.mockedDir.hasReadLock()).thenReturn(true);
+ when(this.mockedDir.hasWriteLock()).thenReturn(true);
+ when(this.mockedDir.getInode(0L)).thenReturn(rootINode);
+ when(this.mockedDir.getInode(1L)).thenReturn(firstINode);
+ when(this.mockedDir.getInode(2L)).thenReturn(secondINode);
+ }
+
+ @Test
+ public void testListEncryptionZonesOneValidOnly() throws Exception{
+ this.ezManager = new EncryptionZoneManager(mockedDir, new Configuration());
+ this.ezManager.addEncryptionZone(1L, CipherSuite.AES_CTR_NOPADDING,
+ CryptoProtocolVersion.ENCRYPTION_ZONES, "test_key");
+ this.ezManager.addEncryptionZone(2L, CipherSuite.AES_CTR_NOPADDING,
+ CryptoProtocolVersion.ENCRYPTION_ZONES, "test_key");
+ // sets root as proper parent for firstINode only
+ this.firstINode.setParent(rootINode);
+ when(mockedDir.getINodesInPath("/first", DirOp.READ_LINK)).
+ thenReturn(mockedINodesInPath);
+ when(mockedINodesInPath.getLastINode()).
+ thenReturn(firstINode);
+ BatchedListEntries<EncryptionZone> result = ezManager.
+ listEncryptionZones(0);
+ assertEquals(1, result.size());
+ assertEquals(1L, result.get(0).getId());
+ assertEquals("/first", result.get(0).getPath());
+ }
+
+ @Test
+ public void testListEncryptionZonesTwoValids() throws Exception {
+ this.ezManager = new EncryptionZoneManager(mockedDir, new Configuration());
+ this.ezManager.addEncryptionZone(1L, CipherSuite.AES_CTR_NOPADDING,
+ CryptoProtocolVersion.ENCRYPTION_ZONES, "test_key");
+ this.ezManager.addEncryptionZone(2L, CipherSuite.AES_CTR_NOPADDING,
+ CryptoProtocolVersion.ENCRYPTION_ZONES, "test_key");
+ // sets root as proper parent for both inodes
+ this.firstINode.setParent(rootINode);
+ this.secondINode.setParent(rootINode);
+ when(mockedDir.getINodesInPath("/first", DirOp.READ_LINK)).
+ thenReturn(mockedINodesInPath);
+ when(mockedINodesInPath.getLastINode()).
+ thenReturn(firstINode);
+ INodesInPath mockedINodesInPathForSecond =
+ mock(INodesInPath.class);
+ when(mockedDir.getINodesInPath("/second", DirOp.READ_LINK)).
+ thenReturn(mockedINodesInPathForSecond);
+ when(mockedINodesInPathForSecond.getLastINode()).
+ thenReturn(secondINode);
+ BatchedListEntries<EncryptionZone> result =
+ ezManager.listEncryptionZones(0);
+ assertEquals(2, result.size());
+ assertEquals(1L, result.get(0).getId());
+ assertEquals("/first", result.get(0).getPath());
+ assertEquals(2L, result.get(1).getId());
+ assertEquals("/second", result.get(1).getPath());
+ }
+
+ @Test
+ public void testListEncryptionZonesForRoot() throws Exception{
+ this.ezManager = new EncryptionZoneManager(mockedDir, new Configuration());
+ this.ezManager.addEncryptionZone(0L, CipherSuite.AES_CTR_NOPADDING,
+ CryptoProtocolVersion.ENCRYPTION_ZONES, "test_key");
+ // sets root as proper parent for firstINode only
+ when(mockedDir.getINodesInPath("/", DirOp.READ_LINK)).
+ thenReturn(mockedINodesInPath);
+ when(mockedINodesInPath.getLastINode()).
+ thenReturn(rootINode);
+ BatchedListEntries<EncryptionZone> result = ezManager.
+ listEncryptionZones(-1);
+ assertEquals(1, result.size());
+ assertEquals(0L, result.get(0).getId());
+ assertEquals("/", result.get(0).getPath());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/401c7318/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml
index 0294368..7e372be 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml
@@ -278,8 +278,8 @@
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rmdir /src/.Trash</command>
- <command>-fs NAMENODE -rmdir /src</command>
- <command>-fs NAMENODE -rmdir /dst</command>
+ <command>-fs NAMENODE -rm -r /src</command>
+ <command>-fs NAMENODE -rm -r /dst</command>
</cleanup-commands>
<comparators>
<comparator>
@@ -478,5 +478,99 @@
</comparator>
</comparators>
</test>
+
+ <test>
+ <description>Test list encryption zones when no zone has been deleted</description>
+ <test-commands>
+ <command>-fs NAMENODE -rm -r .Trash/Current/*</command>
+ <command>-fs NAMENODE -mkdir /test1</command>
+ <crypto-admin-command>-createZone -path /test1 -keyName myKey</crypto-admin-command>
+ <crypto-admin-command>-listZones</crypto-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /test1</command>
+ <command>-fs NAMENODE -rm -r .Trash/Current/*</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>(/test1)\s*(myKey)\s*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>Test adding two zones, then deleting one and listing zones. The deleted zone should still be listed, as it's under user's Trash folder'</description>
+ <test-commands>
+ <command>-fs NAMENODE -rm -r .Trash/Current/*</command>
+ <command>-fs NAMENODE -mkdir /test1</command>
+ <command>-fs NAMENODE -mkdir /test2</command>
+ <crypto-admin-command>-createZone -path /test1 -keyName myKey</crypto-admin-command>
+ <crypto-admin-command>-createZone -path /test2 -keyName myKey</crypto-admin-command>
+ <command>-fs NAMENODE -rm -r /test2</command>
+ <crypto-admin-command>-listZones</crypto-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /test1</command>
+ <command>-fs NAMENODE -rm -r .Trash/Current/*</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>(/test1)\s*(myKey)\s*(/user/).*(/.Trash/Current/test2)\s*(myKey)\s*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>Test adding two zones, then permanently deleting one and listing zones.</description>
+ <test-commands>
+ <command>-fs NAMENODE -rm -r .Trash/Current/*</command>
+ <command>-fs NAMENODE -mkdir /test1</command>
+ <command>-fs NAMENODE -mkdir /test2</command>
+ <crypto-admin-command>-createZone -path /test1 -keyName myKey</crypto-admin-command>
+ <crypto-admin-command>-createZone -path /test2 -keyName myKey</crypto-admin-command>
+ <command>-fs NAMENODE -rm -r /test2</command>
+ <command>-fs NAMENODE -rm -r .Trash/Current/*</command>
+ <crypto-admin-command>-listZones</crypto-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /test1</command>
+ <command>-fs NAMENODE -rm -r .Trash/Current/*</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>(/test1)\s*(myKey)\s*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>Test adding two zones to a snapshotable directory, take snapshot, permanently delete one of the EZs, then list zones</description>
+ <test-commands>
+ <command>-fs NAMENODE -rm -r .Trash/Current/*</command>
+ <command>-fs NAMENODE -mkdir /snapshotable</command>
+ <command>-fs NAMENODE -mkdir /snapshotable/test1</command>
+ <command>-fs NAMENODE -mkdir /snapshotable/test2</command>
+ <dfs-admin-command>-fs NAMENODE -allowSnapshot /snapshotable</dfs-admin-command>
+ <command>-fs NAMENODE -createSnapshot /snapshotable snapshot1</command>
+ <crypto-admin-command>-createZone -path /snapshotable/test1 -keyName myKey</crypto-admin-command>
+ <crypto-admin-command>-createZone -path /snapshotable/test2 -keyName myKey</crypto-admin-command>
+ <command>-fs NAMENODE -rm -r /snapshotable/test2</command>
+ <command>-fs NAMENODE -rm -r .Trash/Current/*</command>
+ <crypto-admin-command>-listZones</crypto-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /snapshotable</command>
+ <command>-fs NAMENODE -rm -r .Trash/Current/*</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>(/test1)\s*(myKey)\s*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
</tests>
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org