You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2015/11/21 08:56:54 UTC
[50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into HDFS-7240
Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e7a7fd5e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e7a7fd5e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e7a7fd5e
Branch: refs/heads/HDFS-7240
Commit: e7a7fd5ee1b363945673987783239be9f918c974
Parents: 3100002 a30eccb
Author: Anu Engineer <ae...@apache.org>
Authored: Fri Nov 20 23:53:35 2015 -0800
Committer: Anu Engineer <ae...@apache.org>
Committed: Fri Nov 20 23:53:35 2015 -0800
----------------------------------------------------------------------
BUILDING.txt | 35 +-
dev-support/docker/Dockerfile | 89 ++--
hadoop-common-project/hadoop-common/CHANGES.txt | 71 ++-
hadoop-common-project/hadoop-common/pom.xml | 21 +-
.../hadoop-common/src/CMakeLists.txt | 25 +
.../hadoop-common/src/config.h.cmake | 1 +
.../hadoop-common/src/main/bin/hadoop | 11 +
.../org/apache/hadoop/http/HttpServer2.java | 3 +
.../org/apache/hadoop/io/BytesWritable.java | 4 +-
.../io/erasurecode/ErasureCodeNative.java | 86 ++++
.../metrics2/impl/MetricsSourceAdapter.java | 11 +-
.../apache/hadoop/security/SaslRpcServer.java | 5 +-
.../hadoop/security/authorize/package-info.java | 2 +-
.../org/apache/hadoop/tools/package-info.java | 3 -
.../apache/hadoop/util/NativeCodeLoader.java | 11 +-
.../hadoop/util/NativeLibraryChecker.java | 20 +-
.../org/apache/hadoop/util/StringUtils.java | 35 ++
.../src/main/native/native.vcxproj | 22 +-
.../io/erasurecode/coder/erasure_code_native.c | 49 ++
...he_hadoop_io_erasurecode_ErasureCodeNative.h | 29 ++
.../apache/hadoop/io/erasurecode/erasure_code.c | 271 +++++++++++
.../io/erasurecode/include/erasure_code.h | 125 +++++
.../hadoop/io/erasurecode/include/gf_util.h | 111 +++++
.../org/apache/hadoop/net/unix/DomainSocket.c | 18 +-
.../org/apache/hadoop/util/NativeCodeLoader.c | 10 +
.../hadoop/io/erasurecode/erasure_code_test.c | 310 ++++++++++++
.../src/main/resources/core-default.xml | 19 +-
.../test/aop/org/apache/hadoop/fi/FiConfig.java | 50 --
.../org/apache/hadoop/fi/ProbabilityModel.java | 108 -----
.../java/org/apache/hadoop/io/AvroTestUtil.java | 2 +-
.../org/apache/hadoop/io/TestArrayFile.java | 17 +-
.../hadoop/io/TestArrayPrimitiveWritable.java | 10 +-
.../org/apache/hadoop/io/TestArrayWritable.java | 20 +-
.../org/apache/hadoop/io/TestBloomMapFile.java | 31 +-
.../io/TestBoundedByteArrayOutputStream.java | 12 +-
.../hadoop/io/TestDefaultStringifier.java | 10 +-
.../apache/hadoop/io/TestEnumSetWritable.java | 15 +-
.../apache/hadoop/io/TestGenericWritable.java | 20 +-
.../java/org/apache/hadoop/io/TestMD5Hash.java | 14 +-
.../org/apache/hadoop/io/TestMapWritable.java | 15 +-
.../org/apache/hadoop/io/TestSequenceFile.java | 22 +-
.../io/TestSequenceFileSerialization.java | 21 +-
.../java/org/apache/hadoop/io/TestSetFile.java | 17 +-
.../java/org/apache/hadoop/io/TestText.java | 85 ++--
.../org/apache/hadoop/io/TestTextNonUTF8.java | 12 +-
.../java/org/apache/hadoop/io/TestUTF8.java | 16 +-
.../apache/hadoop/io/TestVersionedWritable.java | 11 +-
.../java/org/apache/hadoop/io/TestWritable.java | 20 +-
.../org/apache/hadoop/io/TestWritableName.java | 13 +-
.../org/apache/hadoop/io/TestWritableUtils.java | 14 +-
.../hadoop/io/compress/TestCodecFactory.java | 14 +-
.../io/compress/TestCompressionStreamReuse.java | 10 +-
.../hadoop/io/file/tfile/KVGenerator.java | 13 +-
.../apache/hadoop/io/file/tfile/KeySampler.java | 4 +-
.../apache/hadoop/io/file/tfile/TestTFile.java | 16 +-
.../io/file/tfile/TestTFileComparators.java | 14 +-
.../hadoop/io/file/tfile/TestTFileSeek.java | 29 +-
.../file/tfile/TestTFileSeqFileComparison.java | 25 +-
.../hadoop/io/file/tfile/TestTFileSplit.java | 19 +-
.../hadoop/io/file/tfile/TestTFileStreams.java | 55 ++-
.../file/tfile/TestTFileUnsortedByteArrays.java | 14 +-
.../apache/hadoop/io/file/tfile/TestVLong.java | 24 +-
.../serializer/avro/TestAvroSerialization.java | 11 +-
.../metrics2/impl/TestMetricsSourceAdapter.java | 195 +++++++-
.../org/apache/hadoop/hdfs/DFSUtilClient.java | 22 +-
.../hadoop/hdfs/DistributedFileSystem.java | 2 +-
.../hadoop/hdfs/NameNodeProxiesClient.java | 2 +-
.../hadoop/hdfs/util/ByteArrayManager.java | 63 ++-
.../apache/hadoop/hdfs/web/JsonUtilClient.java | 26 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 12 +
.../hadoop/hdfs/web/resources/PutOpParam.java | 2 +
.../hadoop/hdfs/util/TestByteArrayManager.java | 2 +-
.../hdfs/web/TestWebHdfsContentLength.java | 23 +-
.../src/test/resources/log4j.properties | 49 ++
.../hadoop-hdfs-native-client/pom.xml | 46 +-
.../src/CMakeLists.txt | 8 +-
.../main/native/libhdfs-tests/CMakeLists.txt | 1 +
.../src/main/native/libhdfs/CMakeLists.txt | 17 +-
.../apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java | 22 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 114 ++++-
.../dev-support/findbugsExcludeFile.xml | 7 -
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 -
.../hadoop-hdfs/src/main/bin/hdfs | 10 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +
.../server/blockmanagement/BlockManager.java | 8 +-
.../BlockPlacementPolicyDefault.java | 16 +-
.../server/blockmanagement/DatanodeManager.java | 19 +-
.../server/datanode/BlockRecoveryWorker.java | 69 ++-
.../hdfs/server/datanode/BlockSender.java | 2 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 2 +-
.../hdfs/server/datanode/DataStorage.java | 12 -
.../datanode/fsdataset/impl/FsDatasetUtil.java | 5 +-
.../hadoop/hdfs/server/namenode/BackupNode.java | 4 +-
.../hdfs/server/namenode/FSDirSnapshotOp.java | 3 +-
.../hdfs/server/namenode/FSDirectory.java | 8 +-
.../hdfs/server/namenode/FSNamesystem.java | 38 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 4 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 13 +-
.../web/resources/NamenodeWebHdfsMethods.java | 8 +
.../server/protocol/ReplicaRecoveryInfo.java | 6 +
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 15 +
.../src/main/resources/hdfs-default.xml | 9 +
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 84 +++-
.../apache/hadoop/fi/DataTransferTestUtil.java | 485 -------------------
.../test/aop/org/apache/hadoop/fi/FiConfig.java | 55 ---
.../org/apache/hadoop/fi/FiHFlushTestUtil.java | 66 ---
.../aop/org/apache/hadoop/fi/FiTestUtil.java | 209 --------
.../test/aop/org/apache/hadoop/fi/Pipeline.java | 50 --
.../aop/org/apache/hadoop/fi/PipelineTest.java | 27 --
.../org/apache/hadoop/fi/ProbabilityModel.java | 107 ----
.../org/apache/hadoop/fs/TestFiListPath.java | 105 ----
.../aop/org/apache/hadoop/fs/TestFiRename.java | 272 -----------
.../org/apache/hadoop/hdfs/DFSClientAspects.aj | 103 ----
.../aop/org/apache/hadoop/hdfs/HFlushAspects.aj | 64 ---
.../apache/hadoop/hdfs/PipelinesTestUtil.java | 153 ------
.../org/apache/hadoop/hdfs/TestFiHFlush.java | 180 -------
.../org/apache/hadoop/hdfs/TestFiPipelines.java | 247 ----------
.../hdfs/protocol/ClientProtocolAspects.aj | 38 --
.../server/datanode/BlockReceiverAspects.aj | 232 ---------
.../datanode/DataTransferProtocolAspects.aj | 80 ---
.../hdfs/server/datanode/FSDatasetAspects.aj | 63 ---
.../datanode/TestFiDataTransferProtocol.java | 312 ------------
.../datanode/TestFiDataTransferProtocol2.java | 289 -----------
.../server/datanode/TestFiPipelineClose.java | 250 ----------
.../server/namenode/FileDataServletAspects.aj | 46 --
.../hdfs/server/namenode/ListPathAspects.aj | 48 --
.../hdfs/server/namenode/RenameAspects.aj | 66 ---
.../hdfs/ErasureCodeBenchmarkThroughput.java | 423 ++++++++++++++++
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 2 +
.../hadoop/hdfs/TestDFSClientFailover.java | 35 ++
.../org/apache/hadoop/hdfs/TestDFSShell.java | 69 +--
.../apache/hadoop/hdfs/TestDecommission.java | 35 ++
.../TestErasureCodeBenchmarkThroughput.java | 116 +++++
.../hadoop/hdfs/TestFsShellPermission.java | 12 +-
.../java/org/apache/hadoop/hdfs/TestQuota.java | 74 ++-
.../blockmanagement/TestHostFileManager.java | 2 +-
.../server/blockmanagement/TestNodeCount.java | 24 +-
.../TestReplicationPolicyConsiderLoad.java | 56 +++
.../datanode/FsDatasetImplTestUtilsFactory.java | 5 +
.../server/datanode/FsDatasetTestUtils.java | 29 ++
.../hdfs/server/datanode/TestBlockRecovery.java | 37 ++
.../TestDataNodeMultipleRegistrations.java | 8 +-
.../fsdataset/impl/FsDatasetImplTestUtils.java | 43 +-
.../server/namenode/NNThroughputBenchmark.java | 42 +-
.../hdfs/server/namenode/TestBackupNode.java | 71 +++
.../hdfs/server/namenode/TestClusterId.java | 36 +-
.../TestCommitBlockSynchronization.java | 4 +-
.../hdfs/server/namenode/TestFileTruncate.java | 35 +-
.../hdfs/server/namenode/TestHDFSConcat.java | 13 +
.../namenode/TestMetadataVersionOutput.java | 25 +-
.../namenode/TestNamenodeCapacityReport.java | 14 +-
.../hdfs/server/namenode/ha/TestHAMetrics.java | 56 +++
.../server/namenode/ha/TestQuotasWithHA.java | 15 +
.../namenode/snapshot/TestSnapshotDeletion.java | 45 +-
.../namenode/snapshot/TestSnapshotRename.java | 45 +-
.../hadoop/hdfs/tools/TestDFSAdminWithHA.java | 17 +-
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 80 +++
.../org/apache/hadoop/tools/TestJMXGet.java | 17 +-
.../java/org/apache/hadoop/tools/TestTools.java | 5 +
.../apache/hadoop/tracing/TestTraceAdmin.java | 9 +-
hadoop-mapreduce-project/CHANGES.txt | 29 +-
hadoop-mapreduce-project/bin/mapred | 10 +
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 63 ++-
.../v2/app/commit/CommitterEventHandler.java | 25 +-
.../hadoop/mapred/FileOutputCommitter.java | 7 +-
.../apache/hadoop/mapred/OutputCommitter.java | 34 +-
.../hadoop/mapreduce/OutputCommitter.java | 26 +
.../lib/output/FileOutputCommitter.java | 71 ++-
.../hadoop/mapred/TestFileOutputCommitter.java | 151 ++++++
.../TestClientDistributedCacheManager.java | 40 +-
.../lib/output/TestFileOutputCommitter.java | 197 +++++++-
.../mapreduce/v2/hs/webapp/HsJobsBlock.java | 48 +-
.../mapred/TestMRTimelineEventHandling.java | 10 +
.../apache/hadoop/mapred/ShuffleHandler.java | 2 +-
hadoop-project-dist/pom.xml | 13 +-
hadoop-project/pom.xml | 1 +
.../s3a/BlockingThreadPoolExecutorService.java | 272 +++++++++++
.../org/apache/hadoop/fs/s3a/Constants.java | 13 +-
.../hadoop/fs/s3a/S3AFastOutputStream.java | 4 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 82 +---
.../src/site/markdown/tools/hadoop-aws/index.md | 10 +-
.../TestBlockingThreadPoolExecutorService.java | 182 +++++++
.../fs/s3a/TestS3ABlockingThreadPool.java | 80 +++
.../org/apache/hadoop/tools/package-info.java | 4 +-
.../src/main/native/pipes/impl/HadoopPipes.cc | 4 +-
.../main/native/utils/api/hadoop/SerialUtils.hh | 1 +
.../src/main/native/utils/impl/SerialUtils.cc | 7 +
hadoop-yarn-project/CHANGES.txt | 47 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 10 +
.../yarn/api/records/ApplicationAttemptId.java | 2 +-
.../hadoop/yarn/api/records/ApplicationId.java | 2 +-
.../distributedshell/ApplicationMaster.java | 20 +-
.../hadoop/yarn/client/api/AMRMClient.java | 25 +-
.../apache/hadoop/yarn/client/api/NMClient.java | 16 +
.../yarn/client/api/async/AMRMClientAsync.java | 172 ++++++-
.../yarn/client/api/async/NMClientAsync.java | 154 +++++-
.../api/async/impl/AMRMClientAsyncImpl.java | 53 +-
.../api/async/impl/NMClientAsyncImpl.java | 127 ++++-
.../yarn/client/api/impl/AMRMClientImpl.java | 149 +++++-
.../yarn/client/api/impl/NMClientImpl.java | 35 +-
.../hadoop/yarn/client/cli/RMAdminCLI.java | 3 +-
.../api/async/impl/TestAMRMClientAsync.java | 74 ++-
.../api/async/impl/TestNMClientAsync.java | 93 +++-
.../yarn/client/api/impl/TestAMRMClient.java | 161 +++++-
.../api/impl/TestAMRMClientOnRMRestart.java | 80 ++-
.../yarn/client/api/impl/TestNMClient.java | 31 +-
.../hadoop/yarn/client/cli/TestRMAdminCLI.java | 32 ++
.../src/main/resources/yarn-default.xml | 8 +-
.../ApplicationHistoryServer.java | 10 +-
.../localizer/LocalResourcesTrackerImpl.java | 10 +-
.../TestLocalResourcesTrackerImpl.java | 56 ++-
.../TestResourceLocalizationService.java | 4 +-
.../metrics/SystemMetricsPublisher.java | 4 +-
.../ProportionalCapacityPreemptionPolicy.java | 3 +-
.../recovery/FileSystemRMStateStore.java | 12 -
.../recovery/LeveldbRMStateStore.java | 8 -
.../recovery/MemoryRMStateStore.java | 17 -
.../recovery/NullRMStateStore.java | 10 -
.../resourcemanager/recovery/RMStateStore.java | 51 --
.../recovery/RMStateStoreEventType.java | 1 -
.../recovery/ZKRMStateStore.java | 11 -
.../server/resourcemanager/rmapp/RMAppImpl.java | 2 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 26 +-
.../scheduler/AppSchedulingInfo.java | 4 +-
.../resourcemanager/scheduler/QueueMetrics.java | 37 ++
.../CapacitySchedulerConfiguration.java | 14 +-
.../scheduler/capacity/LeafQueue.java | 45 +-
.../allocator/AbstractContainerAllocator.java | 2 +-
.../allocator/RegularContainerAllocator.java | 17 +-
.../webapp/CapacitySchedulerPage.java | 3 +-
.../TestWorkPreservingRMRestart.java | 41 ++
.../metrics/TestSystemMetricsPublisher.java | 2 +-
...estProportionalCapacityPreemptionPolicy.java | 7 +-
...pacityPreemptionPolicyForNodePartitions.java | 6 +
.../recovery/RMStateStoreTestBase.java | 5 +-
.../scheduler/TestQueueMetrics.java | 55 +++
.../scheduler/capacity/TestLeafQueue.java | 426 +++++++++++++++-
237 files changed, 7002 insertions(+), 4766 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7a7fd5e/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7a7fd5e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7a7fd5e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7a7fd5e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7a7fd5e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
----------------------------------------------------------------------