You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by li...@apache.org on 2016/11/30 05:01:38 UTC
[48/48] hadoop git commit: Merge branch 'trunk' into HADOOP-13345
Merge branch 'trunk' into HADOOP-13345
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e93093e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e93093e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e93093e
Branch: refs/heads/HADOOP-13345
Commit: 5e93093e6fbd2493f020bba770385764ef593114
Parents: 86a67ff 8f6e143
Author: Mingliang Liu <li...@apache.org>
Authored: Tue Nov 29 16:45:38 2016 -0800
Committer: Mingliang Liu <li...@apache.org>
Committed: Tue Nov 29 16:45:38 2016 -0800
----------------------------------------------------------------------
hadoop-assemblies/pom.xml | 1 -
hadoop-build-tools/pom.xml | 2 +
hadoop-client/pom.xml | 1 -
.../hadoop-cloud-storage/pom.xml | 1 -
hadoop-cloud-storage-project/pom.xml | 1 -
.../hadoop-annotations/pom.xml | 1 -
.../hadoop-auth-examples/pom.xml | 1 -
hadoop-common-project/hadoop-auth/pom.xml | 1 -
hadoop-common-project/hadoop-common/pom.xml | 15 +-
.../org/apache/hadoop/conf/Configuration.java | 8 +-
.../crypto/key/kms/KMSClientProvider.java | 43 +-
.../java/org/apache/hadoop/fs/FileSystem.java | 1455 +++++++----
.../org/apache/hadoop/fs/shell/Display.java | 11 +-
.../org/apache/hadoop/fs/shell/FsUsage.java | 124 +-
.../hadoop/fs/viewfs/ChRootedFileSystem.java | 6 +
.../apache/hadoop/fs/viewfs/ViewFileSystem.java | 73 +-
.../hadoop/fs/viewfs/ViewFileSystemUtil.java | 164 ++
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 12 +-
.../java/org/apache/hadoop/io/SequenceFile.java | 74 +-
.../io/erasurecode/ErasureCodeConstants.java | 3 +
.../apache/hadoop/ipc/DecayRpcScheduler.java | 2 +-
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 75 +-
.../hadoop/ipc/ProtobufRpcEngineCallback.java | 29 +
.../main/java/org/apache/hadoop/ipc/RPC.java | 5 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 199 +-
.../apache/hadoop/ipc/WritableRpcEngine.java | 3 +-
.../hadoop/ipc/metrics/RpcDetailedMetrics.java | 6 +
.../apache/hadoop/ipc/metrics/RpcMetrics.java | 33 +
.../org/apache/hadoop/jmx/JMXJsonServlet.java | 6 +-
.../java/org/apache/hadoop/log/Log4Json.java | 14 +-
.../hadoop/metrics2/impl/MetricsSystemImpl.java | 4 +-
.../hadoop/security/AccessControlException.java | 2 +-
.../org/apache/hadoop/security/Credentials.java | 2 +-
.../java/org/apache/hadoop/security/KDiag.java | 67 +
.../apache/hadoop/security/SecurityUtil.java | 10 +-
.../hadoop/security/UserGroupInformation.java | 6 +-
.../authorize/AuthorizationException.java | 2 +-
.../org/apache/hadoop/security/token/Token.java | 2 +-
.../AbstractDelegationTokenIdentifier.java | 2 +-
.../AbstractDelegationTokenSecretManager.java | 2 +-
.../DelegationTokenAuthenticationHandler.java | 6 +-
.../web/DelegationTokenAuthenticator.java | 2 +-
.../apache/hadoop/util/HttpExceptionUtils.java | 2 +-
.../src/site/markdown/DeprecatedProperties.md | 7 +-
.../src/site/markdown/filesystem/filesystem.md | 2 +-
.../apache/hadoop/conf/TestConfiguration.java | 2 +-
.../org/apache/hadoop/fs/SymlinkBaseTest.java | 3 +-
.../org/apache/hadoop/fs/TestDefaultUri.java | 40 +-
.../apache/hadoop/fs/TestFileSystemCaching.java | 8 +-
.../hadoop/fs/viewfs/TestViewfsFileStatus.java | 9 +-
.../fs/viewfs/ViewFileSystemBaseTest.java | 94 +-
.../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 22 +
.../apache/hadoop/io/TestSequenceFileSync.java | 113 +-
.../ipc/TestProtoBufRpcServerHandoff.java | 167 ++
.../apache/hadoop/ipc/TestRpcServerHandoff.java | 218 ++
.../org/apache/hadoop/log/TestLog4Json.java | 13 +-
.../apache/hadoop/security/TestKDiagNoKDC.java | 8 +
...tionTokenAuthenticationHandlerWithMocks.java | 3 +-
.../delegation/web/TestWebDelegationToken.java | 4 +-
.../hadoop/util/TestHttpExceptionUtils.java | 2 +-
.../hadoop-common/src/test/proto/test.proto | 11 +-
.../src/test/proto/test_rpc_service.proto | 6 +-
hadoop-common-project/hadoop-kms/pom.xml | 5 +-
.../crypto/key/kms/server/KMSJSONReader.java | 3 +-
.../crypto/key/kms/server/KMSJSONWriter.java | 3 +-
.../hadoop-kms/src/site/markdown/index.md.vm | 2 +
.../hadoop/crypto/key/kms/server/TestKMS.java | 97 +-
hadoop-common-project/hadoop-minikdc/pom.xml | 1 -
hadoop-common-project/hadoop-nfs/pom.xml | 1 -
hadoop-common-project/pom.xml | 1 -
hadoop-dist/pom.xml | 1 -
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 1 -
.../apache/hadoop/hdfs/HdfsConfiguration.java | 10 +-
.../hdfs/client/HdfsClientConfigKeys.java | 10 +-
.../hadoop/hdfs/protocol/HdfsConstants.java | 1 +
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 1 -
.../src/site/markdown/ServerSetup.md.vm | 10 +-
.../hadoop-hdfs-native-client/pom.xml | 1 -
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 1 -
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 -
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 19 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 4 +-
.../server/blockmanagement/BlockManager.java | 62 +-
.../BlockPlacementPolicyDefault.java | 8 +-
.../apache/hadoop/hdfs/server/mover/Mover.java | 4 +-
.../hadoop/hdfs/server/namenode/BackupNode.java | 4 +-
.../namenode/ErasureCodingPolicyManager.java | 23 +-
.../hdfs/server/namenode/FSNamesystem.java | 7 +
.../hadoop/hdfs/server/namenode/INodeFile.java | 8 +-
.../src/main/resources/hdfs-default.xml | 8 +-
.../src/site/markdown/TransparentEncryption.md | 2 +-
.../fs/viewfs/TestViewFileSystemHdfs.java | 53 +
.../apache/hadoop/hdfs/AdminStatesBaseTest.java | 4 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 28 +-
.../hadoop/hdfs/TestBlockStoragePolicy.java | 2 +-
.../hadoop/hdfs/TestDFSStripedInputStream.java | 50 +-
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 29 +-
.../TestDFSStripedOutputStreamWithFailure.java | 39 +-
.../hdfs/TestDFSXORStripedInputStream.java | 33 +
.../hdfs/TestDFSXORStripedOutputStream.java | 35 +
...estDFSXORStripedOutputStreamWithFailure.java | 36 +
.../hdfs/TestDecommissionWithStriped.java | 10 +-
.../apache/hadoop/hdfs/TestDeprecatedKeys.java | 22 +-
.../hadoop/hdfs/TestEncryptedTransfer.java | 2 +-
.../org/apache/hadoop/hdfs/TestFileAppend4.java | 2 +-
.../apache/hadoop/hdfs/TestFileChecksum.java | 2 +-
.../hadoop/hdfs/TestLeaseRecoveryStriped.java | 2 +-
.../hadoop/hdfs/TestMissingBlocksAlert.java | 3 +-
.../hdfs/TestReadStripedFileWithDecoding.java | 3 +-
.../hadoop/hdfs/TestReconstructStripedFile.java | 4 +-
.../hdfs/TestReplaceDatanodeOnFailure.java | 3 +-
.../org/apache/hadoop/hdfs/TestReplication.java | 3 +-
.../hadoop/hdfs/TestWriteReadStripedFile.java | 2 +-
.../hdfs/server/balancer/TestBalancer.java | 26 +-
.../blockmanagement/BlockManagerTestUtil.java | 21 +-
.../TestBlocksWithNotEnoughRacks.java | 4 +-
.../server/blockmanagement/TestNodeCount.java | 2 +-
.../TestPendingInvalidateBlock.java | 2 +-
.../TestPendingReconstruction.java | 2 +-
...constructStripedBlocksWithRackAwareness.java | 8 +-
.../blockmanagement/TestReplicationPolicy.java | 2 +-
.../TestReplicationPolicyConsiderLoad.java | 2 +-
.../TestDataNodeErasureCodingMetrics.java | 2 +-
.../fsdataset/impl/TestLazyPersistFiles.java | 12 +-
.../hadoop/hdfs/server/mover/TestMover.java | 9 +-
.../hdfs/server/mover/TestStorageMover.java | 4 +-
.../server/namenode/NNThroughputBenchmark.java | 10 +-
.../TestAddOverReplicatedStripedBlocks.java | 2 +-
.../namenode/TestDecommissioningStatus.java | 5 +-
.../server/namenode/TestFSEditLogLoader.java | 2 +-
.../hdfs/server/namenode/TestHostsFiles.java | 4 +-
.../hdfs/server/namenode/TestMetaSave.java | 5 +-
.../namenode/TestReconstructStripedBlocks.java | 9 +-
.../server/namenode/TestStripedINodeFile.java | 5 +-
.../hdfs/server/namenode/ha/TestDNFencing.java | 11 +-
.../ha/TestDNFencingWithReplication.java | 2 +-
.../namenode/ha/TestPipelinesFailover.java | 5 +-
.../TestNNMetricFilesInGetListingOps.java | 2 +-
.../namenode/metrics/TestNameNodeMetrics.java | 12 +-
hadoop-hdfs-project/pom.xml | 1 -
.../hadoop-mapreduce-client-app/pom.xml | 1 -
.../org/apache/hadoop/mapred/YarnChild.java | 2 +
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 3 +
.../hadoop-mapreduce-client-common/pom.xml | 1 -
.../hadoop-mapreduce-client-core/pom.xml | 1 -
.../hadoop-mapreduce-client-hs-plugins/pom.xml | 1 -
.../hadoop-mapreduce-client-hs/pom.xml | 1 -
.../hadoop-mapreduce-client-jobclient/pom.xml | 1 -
.../hadoop/mapred/ClientServiceDelegate.java | 4 +-
.../hadoop-mapreduce-client-nativetask/pom.xml | 11 +-
.../src/CMakeLists.txt | 2 +
.../src/main/native/gtest/gtest-all.cc | 1483 ++++++++---
.../src/main/native/gtest/include/gtest/gtest.h | 2341 +++++++++++++-----
.../hadoop-mapreduce-client-shuffle/pom.xml | 1 -
.../hadoop-mapreduce-client/pom.xml | 1 -
.../hadoop-mapreduce-examples/pom.xml | 1 -
hadoop-mapreduce-project/pom.xml | 1 -
hadoop-maven-plugins/pom.xml | 1 -
hadoop-minicluster/pom.xml | 1 -
hadoop-project-dist/pom.xml | 1 -
hadoop-project/pom.xml | 3 +-
hadoop-tools/hadoop-archive-logs/pom.xml | 1 -
hadoop-tools/hadoop-archives/pom.xml | 1 -
.../hadoop/fs/s3a/RenameFailedException.java | 70 +
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 89 +-
.../fs/s3a/ITestS3AFileSystemContract.java | 15 -
.../fs/s3a/yarn/ITestS3AMiniYarnCluster.java | 11 +-
.../src/test/resources/contract/s3a.xml | 2 +-
hadoop-tools/hadoop-azure-datalake/pom.xml | 1 -
hadoop-tools/hadoop-datajoin/pom.xml | 1 -
hadoop-tools/hadoop-distcp/pom.xml | 1 -
hadoop-tools/hadoop-extras/pom.xml | 1 -
hadoop-tools/hadoop-gridmix/pom.xml | 1 -
hadoop-tools/hadoop-pipes/pom.xml | 1 -
hadoop-tools/hadoop-rumen/pom.xml | 6 -
hadoop-tools/hadoop-sls/pom.xml | 1 -
hadoop-tools/hadoop-streaming/pom.xml | 1 -
hadoop-tools/hadoop-tools-dist/pom.xml | 1 -
hadoop-tools/pom.xml | 1 -
.../hadoop-yarn/hadoop-yarn-api/pom.xml | 1 -
.../api/records/AbstractResourceRequest.java | 55 +
.../yarn/api/records/ApplicationReport.java | 10 +
.../yarn/api/records/ApplicationTimeout.java | 99 +
.../yarn/api/records/ResourceRequest.java | 19 +-
.../api/records/UpdateContainerRequest.java | 18 +-
.../src/main/proto/yarn_protos.proto | 7 +
.../pom.xml | 1 -
.../pom.xml | 1 -
.../hadoop-yarn-applications/pom.xml | 1 -
.../hadoop-yarn/hadoop-yarn-client/pom.xml | 1 -
.../hadoop/yarn/client/api/YarnClient.java | 11 +
.../yarn/client/api/impl/YarnClientImpl.java | 9 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 45 +-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 40 +
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 1 -
.../impl/pb/ApplicationReportPBImpl.java | 83 +
.../impl/pb/ApplicationTimeoutPBImpl.java | 130 +
.../hadoop/yarn/event/AsyncDispatcher.java | 6 +-
.../yarn/util/ResourceCalculatorPlugin.java | 3 +
.../resource/DefaultResourceCalculator.java | 19 +-
.../resource/DominantResourceCalculator.java | 25 +-
.../yarn/util/resource/ResourceCalculator.java | 17 +-
.../hadoop/yarn/util/resource/Resources.java | 4 +
.../src/main/resources/yarn-default.xml | 6 +-
.../hadoop/yarn/api/TestPBImplRecords.java | 2 +
.../hadoop/yarn/event/DrainDispatcher.java | 9 +-
.../hadoop/yarn/event/TestAsyncDispatcher.java | 42 +
.../util/resource/TestResourceCalculator.java | 79 +
.../pom.xml | 1 -
.../hadoop-yarn-server-common/pom.xml | 1 -
.../hadoop-yarn-server-nodemanager/pom.xml | 1 -
.../nodemanager/NodeResourceMonitorImpl.java | 5 +
.../monitor/ContainersMonitorImpl.java | 39 +-
.../util/NodeManagerHardwareUtils.java | 160 +-
.../TestContainersMonitorResourceChange.java | 16 +-
.../hadoop-yarn-server-resourcemanager/pom.xml | 1 -
.../server/resourcemanager/ClientRMService.java | 2 +-
...pportunisticContainerAllocatorAMService.java | 13 +-
.../server/resourcemanager/RMAppManager.java | 6 +-
.../server/resourcemanager/RMAuditLogger.java | 1 +
.../server/resourcemanager/RMServerUtils.java | 11 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 17 +
.../scheduler/AbstractYarnScheduler.java | 20 +
.../scheduler/AppSchedulingInfo.java | 17 +
.../scheduler/SchedulerApplicationAttempt.java | 16 +
.../scheduler/SchedulerUtils.java | 59 +-
.../scheduler/YarnScheduler.java | 8 +
.../scheduler/capacity/CapacityScheduler.java | 4 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 16 +
.../distributed/NodeQueueLoadMonitor.java | 8 +-
.../scheduler/fair/FSAppAttempt.java | 242 +-
.../scheduler/fair/FSContext.java | 54 +
.../scheduler/fair/FSLeafQueue.java | 319 +--
.../scheduler/fair/FSParentQueue.java | 52 +-
.../scheduler/fair/FSPreemptionThread.java | 188 ++
.../resourcemanager/scheduler/fair/FSQueue.java | 37 +-
.../scheduler/fair/FSQueueMetrics.java | 60 +
.../scheduler/fair/FSSchedulerNode.java | 36 +-
.../scheduler/fair/FSStarvedApps.java | 85 +
.../scheduler/fair/FairScheduler.java | 309 +--
.../scheduler/fair/Schedulable.java | 29 +-
.../scheduler/fifo/FifoScheduler.java | 4 +-
.../webapp/JAXBContextResolver.java | 3 +-
.../resourcemanager/webapp/RMWebServices.java | 186 ++
.../resourcemanager/webapp/dao/AppInfo.java | 24 +
.../webapp/dao/AppTimeoutInfo.java | 71 +
.../webapp/dao/AppTimeoutsInfo.java | 47 +
...pportunisticContainerAllocatorAMService.java | 97 +-
.../yarn/server/resourcemanager/TestRMHA.java | 3 +-
...tyPreemptionPolicyForReservedContainers.java | 430 ----
.../server/resourcemanager/rmapp/MockRMApp.java | 3 +-
.../rmapp/TestApplicationLifetimeMonitor.java | 30 +-
.../scheduler/TestSchedulerUtils.java | 22 +-
.../scheduler/fair/FairSchedulerTestBase.java | 39 +-
.../fair/FairSchedulerWithMockPreemption.java | 58 +
.../scheduler/fair/FakeSchedulable.java | 5 -
.../scheduler/fair/TestFSAppStarvation.java | 256 ++
.../scheduler/fair/TestFSLeafQueue.java | 165 +-
.../scheduler/fair/TestFairScheduler.java | 137 +
.../fair/TestFairSchedulerPreemption.java | 1540 ++----------
.../fair/TestQueueManagerRealScheduler.java | 128 +
.../scheduler/fair/TestSchedulingPolicy.java | 5 -
.../webapp/TestRMWebServicesApps.java | 2 +-
.../TestRMWebServicesAppsModification.java | 137 +-
.../pom.xml | 1 -
.../hadoop-yarn-server-tests/pom.xml | 1 -
.../hadoop/yarn/server/MiniYARNCluster.java | 7 +-
.../server/TestContainerManagerSecurity.java | 2 +-
.../pom.xml | 1 -
.../pom.xml | 1 -
...stTimelineReaderWebServicesHBaseStorage.java | 2 +-
.../storage/DataGeneratorForTest.java | 4 +-
.../storage/TestHBaseTimelineStorageApps.java | 8 +-
.../TestHBaseTimelineStorageEntities.java | 6 +-
.../flow/TestHBaseStorageFlowActivity.java | 6 +-
.../storage/flow/TestHBaseStorageFlowRun.java | 14 +-
.../flow/TestHBaseStorageFlowRunCompaction.java | 2 +-
.../hadoop-yarn-server-timelineservice/pom.xml | 1 -
.../storage/HBaseTimelineWriterImpl.java | 5 -
.../hadoop-yarn-server-web-proxy/pom.xml | 1 -
.../hadoop-yarn/hadoop-yarn-server/pom.xml | 1 -
.../hadoop-yarn/hadoop-yarn-site/pom.xml | 1 -
.../hadoop-yarn/hadoop-yarn-ui/pom.xml | 1 -
hadoop-yarn-project/hadoop-yarn/pom.xml | 1 -
hadoop-yarn-project/pom.xml | 1 -
pom.xml | 2 +
286 files changed, 9225 insertions(+), 4927 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e93093e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --cc hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 9b8692a,f1cde4e..cb08e57
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@@ -660,17 -659,21 +668,24 @@@ public class S3AFileSystem extends File
/**
* The inner rename operation. See {@link #rename(Path, Path)} for
* the description of the operation.
+ * This operation throws an exception on any failure which needs to be
+ * reported and downgraded to a failure. That is: if a rename
- * @param src path to be renamed
- * @param dst new path after rename
+ * @param source path to be renamed
+ * @param dest new path after rename
- * @return true if rename is successful
+ * @throws RenameFailedException if some criteria for a state changing
+ * rename was not met. This means work didn't happen; it's not something
+ * which is reported upstream to the FileSystem APIs, for which the semantics
+ * of "false" are pretty vague.
+ * @throws FileNotFoundException there's no source file.
* @throws IOException on IO failure.
* @throws AmazonClientException on failures inside the AWS SDK
*/
- private boolean innerRename(Path source, Path dest) throws IOException,
- AmazonClientException {
- private boolean innerRename(Path src, Path dst)
++ private boolean innerRename(Path source, Path dest)
+ throws RenameFailedException, FileNotFoundException, IOException,
+ AmazonClientException {
+ Path src = qualify(source);
+ Path dst = qualify(dest);
+
LOG.debug("Rename path {} to {}", src, dst);
incrementStatistic(INVOCATION_RENAME);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org