You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2014/09/17 02:59:17 UTC
[22/22] git commit: Merge changes from trunk
Merge changes from trunk
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/432b8637
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/432b8637
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/432b8637
Branch: refs/heads/HDFS-6584
Commit: 432b863719e1a36459812cdcbe4785e6a19f3f3a
Parents: 2e4033e b6d3230
Author: Jing Zhao <ji...@hortonworks.com>
Authored: Tue Sep 16 17:58:39 2014 -0700
Committer: Jing Zhao <ji...@hortonworks.com>
Committed: Tue Sep 16 17:58:39 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-auth/pom.xml | 13 +
.../server/AuthenticationFilter.java | 152 ++++--
.../util/RandomSignerSecretProvider.java | 4 +-
.../util/RolloverSignerSecretProvider.java | 7 +-
.../util/SignerSecretProvider.java | 9 +-
.../util/StringSignerSecretProvider.java | 15 +-
.../util/ZKSignerSecretProvider.java | 506 +++++++++++++++++++
.../src/site/apt/Configuration.apt.vm | 148 +++++-
.../hadoop-auth/src/site/apt/index.apt.vm | 5 +
.../server/TestAuthenticationFilter.java | 117 ++++-
.../util/TestJaasConfiguration.java | 55 ++
.../util/TestRandomSignerSecretProvider.java | 2 +-
.../util/TestRolloverSignerSecretProvider.java | 2 +-
.../authentication/util/TestSigner.java | 23 +-
.../util/TestStringSignerSecretProvider.java | 9 +-
.../util/TestZKSignerSecretProvider.java | 270 ++++++++++
hadoop-common-project/hadoop-common/CHANGES.txt | 16 +
.../hadoop-common/src/main/bin/hadoop-config.sh | 1 +
.../src/main/bin/hadoop-functions.sh | 127 +++--
.../hadoop-common/src/main/conf/hadoop-env.sh | 81 ---
.../src/main/conf/hadoop-policy.xml | 9 +
.../main/conf/hadoop-user-functions.sh.example | 94 ++++
.../main/java/org/apache/hadoop/ha/HAAdmin.java | 6 +-
.../io/nativeio/SharedFileDescriptorFactory.c | 32 +-
.../src/site/apt/CommandsManual.apt.vm | 53 ++
hadoop-common-project/hadoop-kms/pom.xml | 25 +-
.../hadoop/crypto/key/kms/server/MiniKMS.java | 47 +-
.../test/resources/mini-kms-acls-default.xml | 135 +++++
.../hadoop/fs/http/server/TestHttpFSServer.java | 8 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 20 +
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 13 +
.../hadoop-hdfs/src/main/bin/hdfs-config.sh | 7 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 5 +-
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 40 +-
.../hadoop/hdfs/protocol/ClientProtocol.java | 4 +-
.../hadoop/hdfs/protocol/EncryptionZone.java | 18 +-
.../hdfs/protocol/EncryptionZoneIterator.java | 20 +-
.../hdfs/protocol/EncryptionZoneWithId.java | 81 ---
.../protocol/EncryptionZoneWithIdIterator.java | 53 --
.../datatransfer/DataTransferProtoUtil.java | 44 +-
.../hdfs/protocol/datatransfer/Receiver.java | 123 +++--
.../hdfs/protocol/datatransfer/Sender.java | 26 +-
...tNamenodeProtocolServerSideTranslatorPB.java | 6 +-
.../ClientNamenodeProtocolTranslatorPB.java | 14 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 12 +-
.../web/resources/DatanodeWebHdfsMethods.java | 13 +-
.../server/namenode/EncryptionZoneManager.java | 24 +-
.../hdfs/server/namenode/FSDirectory.java | 6 +-
.../hdfs/server/namenode/FSNamesystem.java | 10 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 6 +-
.../src/main/proto/datatransfer.proto | 8 +
.../hadoop-hdfs/src/main/proto/encryption.proto | 6 +-
.../hadoop/hdfs/TestDFSClientFailover.java | 5 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 62 ++-
.../hadoop/hdfs/TestEncryptionZonesWithKMS.java | 56 ++
.../blockmanagement/TestBlockManager.java | 56 +-
.../server/datanode/SimulatedFSDataset.java | 7 +-
.../hdfs/tools/TestDFSHAAdminMiniCluster.java | 20 -
.../org/apache/hadoop/tracing/TestTracing.java | 52 +-
.../TestTracingShortCircuitLocalRead.java | 97 ++++
hadoop-mapreduce-project/bin/mapred-config.sh | 7 +-
hadoop-project/pom.xml | 25 +
.../org/apache/hadoop/fs/s3a/Constants.java | 36 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 46 +-
.../apache/hadoop/fs/s3a/S3AOutputStream.java | 6 +-
hadoop-yarn-project/CHANGES.txt | 15 +
.../hadoop-yarn/bin/yarn-config.sh | 11 +-
.../hadoop-yarn/conf/yarn-env.sh | 52 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 14 +
.../applications/distributedshell/Client.java | 18 +-
.../TestDSSleepingAppMaster.java | 58 +++
.../distributedshell/TestDistributedShell.java | 76 +++
.../src/main/resources/yarn-default.xml | 10 +
.../ApplicationHistoryClientService.java | 15 +-
.../authorize/TimelinePolicyProvider.java | 44 ++
.../hadoop-yarn-server-nodemanager/pom.xml | 14 +-
.../util/CgroupsLCEResourcesHandler.java | 29 +-
.../util/TestCgroupsLCEResourcesHandler.java | 165 +++++-
78 files changed, 2892 insertions(+), 634 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/432b8637/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 0e13e83,1a8262f..9b980dc
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@@ -38,16 -40,19 +40,21 @@@ import org.apache.hadoop.fs.Path
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.StorageType;
+ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
+ import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+ import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+ import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.NetworkTopology;
+import org.junit.Assert;
+ import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
@@@ -601,19 -606,53 +608,66 @@@ public class TestBlockManager
new BlockListAsLongs(null, null));
assertEquals(1, ds.getBlockReportCount());
}
+
-
+ /**
+ * Tests that a namenode doesn't choose a datanode with full disks to
+ * store blocks.
+ * @throws Exception
+ */
+ @Test
+ public void testStorageWithRemainingCapacity() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+ FileSystem fs = FileSystem.get(conf);
+ Path file1 = null;
+ try {
+ cluster.waitActive();
+ final FSNamesystem namesystem = cluster.getNamesystem();
+ final String poolId = namesystem.getBlockPoolId();
+ final DatanodeRegistration nodeReg =
+ DataNodeTestUtils.getDNRegistrationForBP(cluster.getDataNodes().
+ get(0), poolId);
+ final DatanodeDescriptor dd = NameNodeAdapter.getDatanode(namesystem,
+ nodeReg);
+ // By default, MiniDFSCluster will create 1 datanode with 2 storages.
+ // Assigning 64k for remaining storage capacity and will
+ //create a file with 100k.
+ for(DatanodeStorageInfo storage: dd.getStorageInfos()) {
+ storage.setUtilizationForTesting(65536, 0, 65536, 0);
+ }
+ //sum of the remaining capacity of both the storages
+ dd.setRemaining(131072);
+ file1 = new Path("testRemainingStorage.dat");
+ try {
+ DFSTestUtil.createFile(fs, file1, 102400, 102400, 102400, (short)1,
+ 0x1BAD5EED);
+ }
+ catch (RemoteException re) {
+ GenericTestUtils.assertExceptionContains("nodes instead of "
+ + "minReplication", re);
+ }
+ }
+ finally {
+ // Clean up
+ assertTrue(fs.exists(file1));
+ fs.delete(file1, true);
+ assertTrue(!fs.exists(file1));
+ cluster.shutdown();
+ }
+ }
-}
+ @Test
+ public void testUseDelHint() {
+ DatanodeStorageInfo delHint = new DatanodeStorageInfo(
+ DFSTestUtil.getLocalDatanodeDescriptor(), new DatanodeStorage("id"));
+ List<DatanodeStorageInfo> moreThan1Racks = Arrays.asList(delHint);
+ List<StorageType> excessTypes = new ArrayList<StorageType>();
+
+ excessTypes.add(StorageType.DEFAULT);
+ Assert.assertTrue(BlockManager.useDelHint(true, delHint, null,
+ moreThan1Racks, excessTypes));
+ excessTypes.add(StorageType.SSD);
+ Assert.assertFalse(BlockManager.useDelHint(true, delHint, null,
+ moreThan1Racks, excessTypes));
+ }
- }
++}