You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by li...@apache.org on 2017/02/22 23:34:36 UTC
[50/50] [abbrv] hadoop git commit: Merge branch 'trunk' into
HADOOP-13345
Merge branch 'trunk' into HADOOP-13345
After HADOOP-14040, we use shaded aws sdk uber-JAR so we don't have to
bring DynamoDB dependency explicitly. However, for tests we do need the
DynamoDBLocal dependency from its Maven repository.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4f7f2baf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4f7f2baf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4f7f2baf
Branch: refs/heads/HADOOP-13345
Commit: 4f7f2baf68dd5cd9639edc87b5034a7618e26099
Parents: 2ff5812 0013090
Author: Mingliang Liu <li...@apache.org>
Authored: Wed Feb 22 15:12:24 2017 -0800
Committer: Mingliang Liu <li...@apache.org>
Committed: Wed Feb 22 15:12:24 2017 -0800
----------------------------------------------------------------------
.gitignore | 12 +-
.../resources/assemblies/hadoop-httpfs-dist.xml | 25 +-
.../main/resources/checkstyle/checkstyle.xml | 4 +-
hadoop-client-modules/hadoop-client-api/pom.xml | 12 +-
.../hadoop-client-minicluster/pom.xml | 22 +-
.../hadoop-client-runtime/pom.xml | 16 +-
.../org/apache/hadoop/conf/Configuration.java | 12 +
.../hadoop/crypto/key/CachingKeyProvider.java | 14 +-
.../apache/hadoop/crypto/key/KeyProvider.java | 12 +
.../hadoop/crypto/key/KeyProviderExtension.java | 5 +
.../org/apache/hadoop/crypto/key/KeyShell.java | 67 +-
.../crypto/key/kms/KMSClientProvider.java | 13 +-
.../hadoop/crypto/key/kms/KMSRESTConstants.java | 1 +
.../key/kms/LoadBalancingKMSClientProvider.java | 24 +-
.../hadoop/crypto/key/kms/ValueQueue.java | 117 ++-
.../org/apache/hadoop/fs/BlockLocation.java | 7 +-
.../hadoop/fs/CommonConfigurationKeys.java | 11 +
.../apache/hadoop/fs/FSExceptionMessages.java | 3 +
.../java/org/apache/hadoop/fs/FileStatus.java | 21 +-
.../hadoop/fs/permission/FsCreateModes.java | 1 +
.../hadoop/fs/permission/FsPermission.java | 22 +-
.../java/org/apache/hadoop/fs/shell/Stat.java | 4 +-
.../apache/hadoop/ha/ActiveStandbyElector.java | 37 +-
.../org/apache/hadoop/http/HttpServer2.java | 119 ++-
.../hadoop/io/retry/RetryInvocationHandler.java | 8 +-
.../main/java/org/apache/hadoop/ipc/Client.java | 6 +-
.../org/apache/hadoop/ipc/FairCallQueue.java | 165 +---
.../main/java/org/apache/hadoop/ipc/Server.java | 206 ++--
.../java/org/apache/hadoop/net/InnerNode.java | 67 ++
.../org/apache/hadoop/net/InnerNodeImpl.java | 304 ++++++
.../org/apache/hadoop/net/NetworkTopology.java | 326 +-----
.../net/NetworkTopologyWithNodeGroup.java | 43 +-
.../AuthenticationWithProxyUserFilter.java | 43 +-
.../hadoop/security/UserGroupInformation.java | 129 ++-
.../ZKDelegationTokenSecretManager.java | 33 +-
.../src/site/markdown/CommandsManual.md | 3 +
.../src/site/markdown/CredentialProviderAPI.md | 1 +
.../src/site/markdown/FileSystemShell.md | 2 +-
.../hadoop-common/src/site/markdown/Metrics.md | 6 +
.../src/site/markdown/SecureMode.md | 2 +-
.../hadoop-common/src/site/markdown/Tracing.md | 2 +-
.../apache/hadoop/crypto/key/TestKeyShell.java | 9 +
.../org/apache/hadoop/fs/TestFileStatus.java | 19 +
.../hadoop/ha/TestActiveStandbyElector.java | 31 +
.../org/apache/hadoop/http/TestHttpServer.java | 38 +
.../hadoop/http/TestHttpServerWithSpengo.java | 15 +-
.../apache/hadoop/ipc/TestFairCallQueue.java | 57 +-
.../java/org/apache/hadoop/ipc/TestRPC.java | 119 +++
.../hadoop/security/TestUGIWithMiniKdc.java | 1 +
.../security/TestUserGroupInformation.java | 16 +-
.../src/test/resources/testConf.xml | 2 +-
...rKeyGeneratorKeyProviderCryptoExtension.java | 6 +
.../hadoop/crypto/key/kms/server/KMS.java | 33 +-
.../crypto/key/kms/server/KMSConfiguration.java | 2 +
.../hadoop/crypto/key/kms/server/KMSWebApp.java | 10 -
.../crypto/key/kms/server/KMSWebServer.java | 3 +
.../kms/server/KeyAuthorizationKeyProvider.java | 11 +
.../src/main/resources/kms-default.xml | 14 +
.../hadoop-kms/src/site/markdown/index.md.vm | 52 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 89 +-
.../crypto/key/kms/server/TestKMSAudit.java | 2 +
.../org/apache/hadoop/fs/HdfsBlockLocation.java | 17 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 2 +-
.../org/apache/hadoop/hdfs/DFSInputStream.java | 48 +-
.../org/apache/hadoop/hdfs/DataStreamer.java | 105 +-
.../hadoop/hdfs/DistributedFileSystem.java | 2 +-
.../apache/hadoop/hdfs/StripedDataStreamer.java | 8 +-
.../hadoop/hdfs/protocol/ClientProtocol.java | 2 +-
.../hadoop/hdfs/protocol/DatanodeInfo.java | 6 +-
.../hdfs/protocol/FsPermissionExtension.java | 2 +
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 51 +
.../token/block/BlockTokenIdentifier.java | 89 +-
.../src/main/proto/hdfs.proto | 33 +
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 103 +-
.../src/main/conf/httpfs-env.sh | 47 +-
.../http/server/HttpFSAuthenticationFilter.java | 2 +-
.../fs/http/server/HttpFSServerWebServer.java | 170 ++++
.../apache/hadoop/lib/servlet/MDCFilter.java | 4 +-
.../src/main/libexec/httpfs-config.sh | 76 --
.../libexec/shellprofile.d/hadoop-httpfs.sh | 67 ++
.../src/main/resources/httpfs-default.xml | 72 ++
.../main/resources/webapps/static/index.html | 36 +
.../resources/webapps/webhdfs/WEB-INF/web.xml | 98 ++
.../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 126 +--
.../src/main/tomcat/ROOT/WEB-INF/web.xml | 16 -
.../src/main/tomcat/ROOT/index.html | 21 -
.../src/main/tomcat/logging.properties | 67 --
.../src/main/tomcat/server.xml | 151 ---
.../src/main/tomcat/ssl-server.xml.conf | 136 ---
.../src/site/markdown/ServerSetup.md.vm | 136 ++-
.../src/site/markdown/index.md | 2 +-
.../http/server/TestHttpFSServerWebServer.java | 106 ++
.../nfs/nfs3/PrivilegedNfsGatewayStarter.java | 6 +-
.../hadoop-hdfs/src/main/bin/hdfs | 4 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 20 +-
.../hadoop/hdfs/qjournal/server/JNStorage.java | 7 +
.../token/block/BlockTokenSecretManager.java | 18 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 2 +-
.../hadoop/hdfs/server/balancer/Dispatcher.java | 3 +-
.../hadoop/hdfs/server/balancer/KeyManager.java | 6 +-
.../server/blockmanagement/BlockManager.java | 9 +-
.../BlockUnderConstructionFeature.java | 25 +
.../blockmanagement/DatanodeDescriptor.java | 2 +-
.../server/blockmanagement/DatanodeManager.java | 2 +-
.../PendingReconstructionBlocks.java | 4 +-
.../hadoop/hdfs/server/common/Storage.java | 2 +-
.../hdfs/server/datanode/BlockReceiver.java | 3 +-
.../hadoop/hdfs/server/datanode/DNConf.java | 12 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 15 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 3 +
.../datanode/web/webhdfs/WebHdfsHandler.java | 6 +
.../hadoop/hdfs/server/namenode/FSEditLog.java | 6 +-
.../hdfs/server/namenode/FSEditLogOp.java | 29 +-
.../server/namenode/FSPermissionChecker.java | 87 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 112 ++-
.../hdfs/server/namenode/NamenodeFsck.java | 28 +-
.../protocol/BalancerBandwidthCommand.java | 2 +-
.../apache/hadoop/hdfs/tools/AdminHelper.java | 12 +
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 2 +-
.../hadoop/hdfs/tools/StoragePolicyAdmin.java | 23 +-
.../hadoop/hdfs/tools/erasurecode/ECCli.java | 2 +-
.../src/main/resources/hdfs-default.xml | 11 +-
.../src/main/webapps/datanode/datanode.html | 2 +
.../src/main/webapps/static/hadoop.css | 9 +-
.../src/site/markdown/HDFSCommands.md | 12 +-
.../src/site/markdown/HDFSErasureCoding.md | 4 +-
.../src/site/markdown/HdfsImageViewer.md | 63 ++
.../hadoop/fs/permission/TestStickyBit.java | 63 ++
.../apache/hadoop/hdfs/TestDFSOutputStream.java | 3 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 24 +-
.../hadoop/hdfs/TestMaintenanceState.java | 6 +
.../java/org/apache/hadoop/hdfs/TestPread.java | 51 +
.../hdfs/qjournal/server/TestJournal.java | 17 +
.../security/token/block/TestBlockToken.java | 297 +++++-
.../hdfs/server/datanode/DataNodeTestUtils.java | 26 +
.../datanode/TestDataNodeHotSwapVolumes.java | 9 +-
.../datanode/TestDataNodeVolumeFailure.java | 65 +-
.../TestDataNodeVolumeFailureReporting.java | 12 +-
.../TestDataNodeVolumeFailureToleration.java | 6 +
.../shortcircuit/TestShortCircuitLocalRead.java | 2 +-
.../hdfs/tools/TestStoragePolicyCommands.java | 14 +-
.../TestOfflineImageViewer.java | 11 +
.../apache/hadoop/net/TestNetworkTopology.java | 2 +-
.../test/resources/testErasureCodingConf.xml | 2 +-
.../hadoop/mapred/TaskAttemptListenerImpl.java | 17 +-
.../jobhistory/JobHistoryEventHandler.java | 57 +-
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 14 +-
.../v2/app/client/MRClientService.java | 4 +-
.../v2/app/rm/RMContainerAllocator.java | 4 +-
.../mapreduce/v2/app/webapp/AppController.java | 7 +-
.../jobhistory/TestJobHistoryEventHandler.java | 8 +-
.../apache/hadoop/mapreduce/MRJobConfig.java | 8 +-
.../src/main/resources/mapred-default.xml | 8 +
.../site/markdown/DistributedCacheDeploy.md.vm | 2 +
.../src/site/markdown/EncryptedShuffle.md | 2 +
.../src/site/markdown/MapReduceTutorial.md | 45 +-
.../MapReduce_Compatibility_Hadoop1_Hadoop2.md | 2 +
.../src/site/markdown/MapredAppMasterRest.md | 16 +-
.../src/site/markdown/MapredCommands.md | 14 +-
.../PluggableShuffleAndPluggableSort.md | 2 +
.../src/site/markdown/HistoryServerRest.md | 22 +-
.../hadoop/mapred/ResourceMgrDelegate.java | 6 +-
.../org/apache/hadoop/examples/pi/package.html | 4 +-
hadoop-project/pom.xml | 27 +-
hadoop-project/src/site/site.xml | 4 +-
.../fs/aliyun/oss/AliyunOSSFileSystem.java | 5 +-
.../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 47 +-
.../fs/aliyun/oss/AliyunOSSInputStream.java | 4 +-
.../oss/TestAliyunOSSFileSystemContract.java | 7 +
.../fs/aliyun/oss/TestAliyunOSSInputStream.java | 9 +
.../oss/contract/TestAliyunOSSContractSeek.java | 26 +
.../src/test/resources/contract/aliyun-oss.xml | 5 +
hadoop-tools/hadoop-aws/pom.xml | 19 +-
.../s3a/BlockingThreadPoolExecutorService.java | 2 +-
.../org/apache/hadoop/fs/s3a/Constants.java | 21 +-
.../hadoop/fs/s3a/S3ABlockOutputStream.java | 1 +
.../org/apache/hadoop/fs/s3a/S3ADataBlocks.java | 26 +-
.../hadoop/fs/s3a/S3AEncryptionMethods.java | 61 ++
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 152 ++-
.../apache/hadoop/fs/s3a/S3AInputStream.java | 24 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 10 +
.../hadoop/fs/s3a/S3ObjectAttributes.java | 59 ++
.../src/site/markdown/tools/hadoop-aws/index.md | 579 +----------
.../site/markdown/tools/hadoop-aws/testing.md | 814 +++++++++++++++
.../fs/s3a/AbstractTestS3AEncryption.java | 132 +++
.../hadoop/fs/s3a/ITestS3AEncryption.java | 97 --
.../ITestS3AEncryptionAlgorithmPropagation.java | 76 --
.../ITestS3AEncryptionAlgorithmValidation.java | 152 +++
.../ITestS3AEncryptionBlockOutputStream.java | 36 -
.../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 90 ++
...ITestS3AEncryptionSSECBlockOutputStream.java | 46 +
.../s3a/ITestS3AEncryptionSSEKMSDefaultKey.java | 57 ++
.../ITestS3AEncryptionSSEKMSUserDefinedKey.java | 48 +
...onSSEKMSUserDefinedKeyBlockOutputStream.java | 52 +
.../hadoop/fs/s3a/ITestS3AEncryptionSSES3.java | 43 +
...TestS3AEncryptionSSES3BlockOutputStream.java | 45 +
.../hadoop/fs/s3a/TestS3AGetFileStatus.java | 57 +-
.../fs/s3a/scale/ITestS3AConcurrentOps.java | 167 ++++
.../NativeS3FileSystemContractBaseTest.java | 2 +-
.../org/apache/hadoop/fs/adl/AdlConfKeys.java | 4 +
.../org/apache/hadoop/fs/adl/AdlFileSystem.java | 31 +-
.../src/site/markdown/index.md | 30 +-
.../hadoop/fs/adl/TestADLResponseData.java | 21 +
.../apache/hadoop/fs/adl/TestGetFileStatus.java | 25 +
.../apache/hadoop/fs/adl/TestListStatus.java | 32 +
.../fs/adl/TestValidateConfiguration.java | 9 +
.../fs/adl/live/AdlStorageConfiguration.java | 42 +-
.../apache/hadoop/fs/adl/live/TestMetadata.java | 33 +
.../src/test/resources/adls.xml | 11 +
.../test/resources/contract-test-options.xml | 61 --
.../fs/swift/auth/PasswordCredentials.java | 3 +-
.../hadoop/yarn/api/ApplicationConstants.java | 7 -
.../GetLabelsToNodesResponse.java | 6 +-
.../GetNodesToLabelsResponse.java | 7 +-
.../yarn/api/records/UpdateContainerError.java | 19 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 10 +-
.../AddToClusterNodeLabelsRequest.java | 11 +-
..._server_resourcemanager_service_protos.proto | 5 +-
.../src/main/proto/yarn_protos.proto | 6 +-
.../src/main/proto/yarn_service_protos.proto | 6 +-
.../distributedshell/ApplicationMaster.java | 98 +-
.../hadoop/yarn/client/api/AMRMClient.java | 71 +-
.../hadoop/yarn/client/api/YarnClient.java | 6 +-
.../yarn/client/api/async/AMRMClientAsync.java | 52 +-
.../api/async/impl/AMRMClientAsyncImpl.java | 15 +-
.../yarn/client/api/impl/AMRMClientImpl.java | 111 ++-
.../yarn/client/api/impl/YarnClientImpl.java | 83 +-
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 17 -
.../hadoop/yarn/client/TestRMFailover.java | 100 +-
.../yarn/client/api/impl/TestAMRMClient.java | 60 +-
.../api/impl/TestAMRMClientOnRMRestart.java | 8 +-
.../TestOpportunisticContainerAllocation.java | 400 +++++++-
.../yarn/client/api/impl/TestYarnClient.java | 216 +++-
.../pb/GetClusterNodeLabelsResponsePBImpl.java | 4 +
.../impl/pb/GetLabelsToNodesResponsePBImpl.java | 20 +-
.../impl/pb/GetNodesToLabelsResponsePBImpl.java | 52 +-
.../impl/pb/UpdateContainerErrorPBImpl.java | 16 +
.../RequestHedgingRMFailoverProxyProvider.java | 6 +-
.../hadoop/yarn/client/api/TimelineClient.java | 94 +-
.../yarn/client/api/TimelineV2Client.java | 92 ++
.../client/api/impl/TimelineClientImpl.java | 825 ++--------------
.../yarn/client/api/impl/TimelineConnector.java | 440 +++++++++
.../client/api/impl/TimelineV2ClientImpl.java | 459 +++++++++
.../ContainerLogAggregationType.java | 31 +
.../yarn/logaggregation/ContainerLogType.java | 31 -
.../yarn/logaggregation/LogToolUtils.java | 159 +++
.../pb/AddToClusterNodeLabelsRequestPBImpl.java | 47 +-
.../pb/ReplaceLabelsOnNodeRequestPBImpl.java | 18 +-
.../hadoop/yarn/util/resource/Resources.java | 24 +
.../org/apache/hadoop/yarn/webapp/WebApps.java | 38 +-
.../hadoop/yarn/webapp/util/WebAppUtils.java | 31 +-
.../yarn/webapp/util/YarnWebServiceUtils.java | 62 ++
.../src/main/resources/yarn-default.xml | 23 +
.../client/api/impl/TestTimelineClient.java | 39 +-
.../api/impl/TestTimelineClientV2Impl.java | 4 +-
.../apache/hadoop/yarn/webapp/TestWebApp.java | 46 +
.../webapp/AHSWebServices.java | 354 +++----
.../webapp/TestAHSWebServices.java | 157 ++-
.../hadoop/yarn/server/webapp/AppBlock.java | 135 +--
.../server/webapp/YarnWebServiceParams.java | 37 +
.../server/webapp/dao/ContainerLogsInfo.java | 8 +-
.../timelineservice/NMTimelinePublisher.java | 22 +-
.../nodemanager/webapp/NMWebServices.java | 170 +++-
.../webapp/dao/NMContainerLogsInfo.java | 4 +-
.../TestNMTimelinePublisher.java | 10 +-
.../nodemanager/webapp/TestNMWebServices.java | 161 ++-
.../hadoop-yarn-server-resourcemanager/pom.xml | 16 +-
.../server/resourcemanager/AdminService.java | 7 +-
.../server/resourcemanager/ClientRMService.java | 6 +-
.../DecommissioningNodesWatcher.java | 4 +-
.../yarn/server/resourcemanager/RMContext.java | 2 +
.../server/resourcemanager/RMContextImpl.java | 10 +
...MCriticalThreadUncaughtExceptionHandler.java | 58 ++
.../resourcemanager/RMFatalEventType.java | 5 +-
.../server/resourcemanager/RMServerUtils.java | 36 +-
.../server/resourcemanager/ResourceManager.java | 65 +-
.../resourcemanager/amlauncher/AMLauncher.java | 13 -
.../FifoIntraQueuePreemptionPlugin.java | 6 +-
.../nodelabels/RMNodeLabelsManager.java | 16 +
.../resourcemanager/recovery/RMStateStore.java | 13 +-
.../recovery/ZKRMStateStore.java | 27 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 12 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 176 +++-
.../scheduler/AbstractUsersManager.java | 54 +
.../scheduler/AbstractYarnScheduler.java | 4 +
.../scheduler/ActiveUsersManager.java | 23 +-
.../scheduler/AppSchedulingInfo.java | 16 +-
.../server/resourcemanager/scheduler/Queue.java | 2 +-
.../scheduler/SchedulerApplicationAttempt.java | 4 +-
.../scheduler/capacity/CSQueue.java | 8 +-
.../capacity/CapacityHeadroomProvider.java | 4 +-
.../scheduler/capacity/CapacityScheduler.java | 3 +-
.../scheduler/capacity/LeafQueue.java | 589 +++--------
.../scheduler/capacity/ParentQueue.java | 2 +-
.../scheduler/capacity/UsersManager.java | 982 +++++++++++++++++++
.../scheduler/common/fica/FiCaSchedulerApp.java | 14 +-
.../scheduler/fair/AllocationConfiguration.java | 11 +-
.../scheduler/fair/FSAppAttempt.java | 185 +++-
.../scheduler/fair/FSLeafQueue.java | 122 ++-
.../scheduler/fair/FSOpDurations.java | 8 -
.../scheduler/fair/FSParentQueue.java | 15 +-
.../scheduler/fair/FSPreemptionThread.java | 132 +--
.../resourcemanager/scheduler/fair/FSQueue.java | 50 +-
.../scheduler/fair/FairScheduler.java | 30 +-
.../fair/FairSchedulerConfiguration.java | 23 +-
.../scheduler/fair/QueueManager.java | 28 +-
.../scheduler/fair/SchedulingPolicy.java | 36 +-
.../fair/VisitedResourceRequestTracker.java | 146 +++
.../DominantResourceFairnessPolicy.java | 5 -
.../fair/policies/FairSharePolicy.java | 15 +-
.../scheduler/fair/policies/FifoPolicy.java | 14 +-
.../scheduler/fifo/FifoScheduler.java | 2 +-
.../dao/CapacitySchedulerLeafQueueInfo.java | 2 +-
.../TestApplicationMasterLauncher.java | 5 -
.../resourcemanager/TestClientRMService.java | 42 +-
...pportunisticContainerAllocatorAMService.java | 5 +-
...alCapacityPreemptionPolicyMockFramework.java | 8 +-
.../rmapp/attempt/TestBoundedAppender.java | 116 +++
.../TestRMAppAttemptImplDiagnostics.java | 111 +++
.../TestSchedulerApplicationAttempt.java | 14 +-
.../capacity/TestApplicationLimits.java | 15 +-
.../TestApplicationLimitsByPartition.java | 23 +-
.../TestCapacitySchedulerNodeLabelUpdate.java | 77 +-
.../capacity/TestIncreaseAllocationExpirer.java | 4 +-
.../scheduler/capacity/TestLeafQueue.java | 98 +-
.../TestNodeLabelContainerAllocation.java | 7 +-
.../fair/FairSchedulerWithMockPreemption.java | 5 +-
.../fair/TestContinuousScheduling.java | 2 +-
.../scheduler/fair/TestFSAppStarvation.java | 30 +-
.../scheduler/fair/TestFairScheduler.java | 72 +-
.../fair/TestFairSchedulerPreemption.java | 73 +-
.../scheduler/fair/TestSchedulingPolicy.java | 302 ++++--
.../fair/TestVisitedResourceRequestTracker.java | 112 +++
.../TestTimelineServiceClientIntegration.java | 10 +-
.../reader/TimelineReaderServer.java | 13 +-
.../src/site/markdown/CapacityScheduler.md | 14 +-
.../src/site/markdown/DockerContainers.md | 293 ++++++
.../src/site/markdown/FairScheduler.md | 14 +-
.../src/site/markdown/NodeLabel.md | 14 +-
.../src/site/markdown/NodeManager.md | 8 +-
.../src/site/markdown/NodeManagerCgroups.md | 3 +-
.../src/site/markdown/NodeManagerRest.md | 116 ++-
.../site/markdown/OpportunisticContainers.md | 17 +-
.../src/site/markdown/ReservationSystem.md | 6 +-
.../src/site/markdown/ResourceManagerHA.md | 10 +-
.../src/site/markdown/ResourceManagerRest.md | 29 +-
.../src/site/markdown/ResourceManagerRestart.md | 13 +-
.../src/site/markdown/SecureContainer.md | 2 +-
.../src/site/markdown/TimelineServer.md | 16 +-
.../src/site/markdown/TimelineServiceV2.md | 154 ++-
.../src/site/markdown/WebApplicationProxy.md | 7 +-
.../src/site/markdown/WebServicesIntro.md | 13 +-
.../site/markdown/WritingYarnApplications.md | 14 +-
.../site/markdown/YarnApplicationSecurity.md | 2 +
.../src/site/markdown/YarnCommands.md | 26 +-
.../hadoop-yarn-ui/src/main/webapp/.jshintrc | 5 +-
.../src/main/webapp/app/adapters/abstract.js | 1 +
.../main/webapp/app/adapters/cluster-info.js | 4 +-
.../main/webapp/app/adapters/cluster-metric.js | 4 +-
.../webapp/app/adapters/yarn-app-attempt.js | 4 +-
.../src/main/webapp/app/adapters/yarn-app.js | 6 +-
.../webapp/app/adapters/yarn-container-log.js | 4 +-
.../main/webapp/app/adapters/yarn-container.js | 12 +-
.../src/main/webapp/app/adapters/yarn-node.js | 4 +-
.../src/main/webapp/app/adapters/yarn-queue.js | 4 +-
.../main/webapp/app/adapters/yarn-rm-node.js | 6 +-
.../webapp/app/components/app-attempt-table.js | 9 +-
.../app/components/app-usage-donut-chart.js | 6 +-
.../src/main/webapp/app/components/bar-chart.js | 36 +-
.../app/components/base-chart-component.js | 8 +-
.../app/components/base-usage-donut-chart.js | 11 +-
.../main/webapp/app/components/donut-chart.js | 10 +-
.../main/webapp/app/components/nodes-heatmap.js | 21 +-
...er-app-memusage-by-nodes-stacked-barchart.js | 4 +-
...app-ncontainers-by-nodes-stacked-barchart.js | 4 +-
.../app/components/queue-usage-donut-chart.js | 4 +-
.../main/webapp/app/components/queue-view.js | 11 +-
.../main/webapp/app/components/simple-table.js | 6 +-
.../webapp/app/components/stacked-barchart.js | 23 +-
.../main/webapp/app/components/timeline-view.js | 25 +-
.../main/webapp/app/components/tree-selector.js | 41 +-
.../main/webapp/app/controllers/application.js | 8 +-
.../webapp/app/controllers/yarn-app-attempt.js | 2 +-
.../webapp/app/controllers/yarn-app-attempts.js | 2 +-
.../src/main/webapp/app/controllers/yarn-app.js | 8 +-
.../main/webapp/app/controllers/yarn-apps.js | 2 +-
.../app/controllers/yarn-container-log.js | 7 +-
.../webapp/app/controllers/yarn-node-app.js | 7 +-
.../webapp/app/controllers/yarn-node-apps.js | 2 +-
.../app/controllers/yarn-node-container.js | 39 +
.../app/controllers/yarn-node-containers.js | 2 +-
.../main/webapp/app/controllers/yarn-node.js | 2 +-
.../webapp/app/controllers/yarn-queue-apps.js | 46 -
.../main/webapp/app/controllers/yarn-queue.js | 17 +-
.../main/webapp/app/controllers/yarn-queues.js | 1 +
.../webapp/app/controllers/yarn-services.js | 2 +-
.../src/main/webapp/app/helpers/divide.js | 2 +-
.../main/webapp/app/helpers/log-files-comma.js | 5 +-
.../src/main/webapp/app/helpers/node-link.js | 2 +-
.../src/main/webapp/app/helpers/node-menu.js | 10 +-
.../src/main/webapp/app/helpers/node-name.js | 4 +-
.../src/main/webapp/app/initializers/loader.js | 8 +-
.../main/webapp/app/models/yarn-app-attempt.js | 17 +-
.../src/main/webapp/app/models/yarn-app.js | 10 +-
.../src/main/webapp/app/models/yarn-node-app.js | 8 +-
.../webapp/app/models/yarn-node-container.js | 12 +-
.../src/main/webapp/app/models/yarn-queue.js | 14 +-
.../src/main/webapp/app/models/yarn-rm-node.js | 10 +-
.../src/main/webapp/app/models/yarn-user.js | 2 +-
.../src/main/webapp/app/router.js | 6 +-
.../src/main/webapp/app/routes/application.js | 3 +-
.../main/webapp/app/routes/cluster-overview.js | 4 +-
.../main/webapp/app/routes/yarn-app-attempt.js | 12 +-
.../main/webapp/app/routes/yarn-app-attempts.js | 2 -
.../src/main/webapp/app/routes/yarn-app.js | 2 +-
.../src/main/webapp/app/routes/yarn-apps.js | 4 +-
.../webapp/app/routes/yarn-container-log.js | 2 +-
.../src/main/webapp/app/routes/yarn-node-app.js | 2 +-
.../webapp/app/routes/yarn-node-container.js | 2 +-
.../src/main/webapp/app/routes/yarn-node.js | 4 +-
.../src/main/webapp/app/routes/yarn-nodes.js | 4 +-
.../main/webapp/app/routes/yarn-queue-apps.js | 42 -
.../src/main/webapp/app/routes/yarn-queue.js | 3 +-
.../main/webapp/app/routes/yarn-queue/apps.js | 22 +
.../main/webapp/app/routes/yarn-queue/info.js | 22 +
.../src/main/webapp/app/routes/yarn-queues.js | 35 +-
.../main/webapp/app/routes/yarn-queues/index.js | 2 +
.../webapp/app/serializers/yarn-app-attempt.js | 23 +-
.../src/main/webapp/app/serializers/yarn-app.js | 19 +-
.../app/serializers/yarn-container-log.js | 3 +-
.../webapp/app/serializers/yarn-container.js | 22 +-
.../webapp/app/serializers/yarn-node-app.js | 24 +-
.../app/serializers/yarn-node-container.js | 7 +-
.../main/webapp/app/serializers/yarn-node.js | 8 +-
.../main/webapp/app/serializers/yarn-queue.js | 10 +-
.../main/webapp/app/serializers/yarn-rm-node.js | 7 +-
.../src/main/webapp/app/styles/app.css | 28 +-
.../main/webapp/app/templates/application.hbs | 8 +-
.../webapp/app/templates/cluster-overview.hbs | 10 +-
.../templates/components/app-attempt-table.hbs | 6 +-
.../templates/components/container-table.hbs | 6 +-
.../src/main/webapp/app/templates/loading.hbs | 23 +
.../src/main/webapp/app/templates/yarn-app.hbs | 8 +-
.../webapp/app/templates/yarn-apps/loading.hbs | 23 +
.../webapp/app/templates/yarn-queue-apps.hbs | 64 --
.../main/webapp/app/templates/yarn-queue.hbs | 69 +-
.../webapp/app/templates/yarn-queue/apps.hbs | 28 +
.../webapp/app/templates/yarn-queue/info.hbs | 84 ++
.../main/webapp/app/templates/yarn-queues.hbs | 8 +-
.../src/main/webapp/app/utils/color-utils.js | 6 +-
.../src/main/webapp/app/utils/converter.js | 2 +-
.../main/webapp/app/utils/href-address-utils.js | 2 -
.../src/main/webapp/app/utils/mock.js | 4 +-
.../src/main/webapp/app/utils/sorter.js | 8 +-
.../src/main/webapp/ember-cli-build.js | 2 +-
.../webapp/public/assets/images/spinner.gif | Bin 0 -> 33076 bytes
.../controllers/yarn-node-container-test.js | 30 +
.../unit/controllers/yarn-queue-apps-test.js | 30 -
.../unit/models/yarn-container-log-test.js | 1 +
.../tests/unit/models/yarn-node-app-test.js | 1 +
.../unit/models/yarn-node-container-test.js | 1 +
.../webapp/tests/unit/models/yarn-node-test.js | 1 +
.../tests/unit/models/yarn-rm-node-test.js | 1 +
.../unit/routes/yarn-container-log-test.js | 24 +-
.../tests/unit/routes/yarn-node-app-test.js | 28 +-
.../tests/unit/routes/yarn-node-apps-test.js | 30 +-
.../unit/routes/yarn-node-container-test.js | 28 +-
.../unit/routes/yarn-node-containers-test.js | 30 +-
.../webapp/tests/unit/routes/yarn-node-test.js | 4 +-
.../webapp/tests/unit/routes/yarn-nodes-test.js | 2 +-
.../tests/unit/routes/yarn-queue-apps-test.js | 29 -
.../tests/unit/routes/yarn-queue/apps-test.js | 29 +
.../tests/unit/routes/yarn-queue/info-test.js | 29 +
pom.xml | 4 +-
474 files changed, 13689 insertions(+), 6637 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f7f2baf/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --cc hadoop-project/pom.xml
index eb47523,47e21d8..130460c
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@@ -1703,12 -1677,4 +1688,12 @@@
</build>
</profile>
</profiles>
+
+ <repositories>
+ <repository>
- <id>dynamodblocal</id>
- <name>AWS DynamoDB Local Release Repository</name>
- <url>http://dynamodb-local.s3-website-us-west-2.amazonaws.com/release</url>
++ <id>dynamodb-local-oregon</id>
++ <name>DynamoDB Local Release Repository</name>
++ <url>https://s3-us-west-2.amazonaws.com/dynamodb-local/release</url>
+ </repository>
+ </repositories>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f7f2baf/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --cc hadoop-tools/hadoop-aws/pom.xml
index 73e2a39,0fdbc5d..ea93e96
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@@ -449,35 -355,6 +451,22 @@@
<artifactId>joda-time</artifactId>
</dependency>
<dependency>
+ <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-dynamodb</artifactId>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
+ <artifactId>DynamoDBLocal</artifactId>
++ <version>${aws-java-sdk.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-http</artifactId>
+ </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-sts</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f7f2baf/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
----------------------------------------------------------------------
diff --cc hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 3636186,414f951..6991a13
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@@ -302,114 -308,12 +313,122 @@@ public final class Constants
@InterfaceAudience.Private
public static final int MAX_MULTIPART_COUNT = 10000;
+ /**
+ * Classname of the S3A-specific output committer factory. This
+ * is what must be declared when attempting to use
+ */
+ @InterfaceStability.Unstable
+ public static final String S3A_OUTPUT_COMMITTER_FACTORY =
+ "org.apache.hadoop.fs.s3a.commit.S3AOutputCommitterFactory";
+
+ /* Constants. */
+ public static final String S3_METADATA_STORE_IMPL =
+ "fs.s3a.metadatastore.impl";
+
+ /** Minimum period of time (in milliseconds) to keep metadata (may only be
+ * applied when a prune command is manually run).
+ */
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_CLI_PRUNE_AGE =
+ "fs.s3a.s3guard.cli.prune.age";
+
+ /**
+ * The endpoint of the DynamoDB service.
+ *
+ * This config has no default value. If the user does not set this, the AWS
+ * SDK will find the endpoint automatically by the Region.
+ */
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_DDB_ENDPOINT_KEY =
+ "fs.s3a.s3guard.ddb.endpoint";
+ /**
+ * The DynamoDB table name to use.
+ *
+ * This config has no default value. If the user does not set this, the
+ * S3Guard implementation will use the respective S3 bucket name.
+ */
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_DDB_TABLE_NAME_KEY =
+ "fs.s3a.s3guard.ddb.table";
+
+ /**
+ * Whether to create the DynamoDB table if the table does not exist.
+ */
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_DDB_TABLE_CREATE_KEY =
+ "fs.s3a.s3guard.ddb.table.create";
+
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_DDB_TABLE_CAPACITY_READ_KEY =
+ "fs.s3a.s3guard.ddb.table.capacity.read";
+ public static final long S3GUARD_DDB_TABLE_CAPACITY_READ_DEFAULT = 500;
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY =
+ "fs.s3a.s3guard.ddb.table.capacity.write";
+ public static final long S3GUARD_DDB_TABLE_CAPACITY_WRITE_DEFAULT = 100;
+
+ /**
+ * The maximum put or delete requests per BatchWriteItem request.
+ *
+ * Refer to Amazon API reference for this limit.
+ */
+ public static final int S3GUARD_DDB_BATCH_WRITE_REQUEST_LIMIT = 25;
+
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_DDB_MAX_RETRIES =
+ "fs.s3a.s3guard.ddb.max.retries";
+ /**
+ * Max retries on batched DynamoDB operations before giving up and
+ * throwing an IOException. Default is {@value}. See core-default.xml for
+ * more detail.
+ */
+ public static final int S3GUARD_DDB_MAX_RETRIES_DEFAULT = 9;
+
+ /**
+ * Period of time (in milliseconds) to sleep between batches of writes.
+ * Currently only applies to prune operations, as they are naturally a
+ * lower priority than other operations.
+ */
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_KEY =
+ "fs.s3a.s3guard.ddb.background.sleep";
+ public static final int S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_DEFAULT = 25;
+
+ /**
+ * V1 committer.
+ */
+ @InterfaceStability.Unstable
+ public static final String S3A_OUTPUT_COMMITTER_MRV1 =
+ "org.apache.hadoop.fs.s3a.commit.S3OutputCommitterMRv1";
+
+ /**
+ * The default "Null" metadata store: {@value}.
+ */
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_METASTORE_NULL
+ = "org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore";
+
+ /**
+ * Use Local memory for the metadata: {@value}.
+ * This is not coherent across processes and must be used for testing only.
+ */
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_METASTORE_LOCAL
+ = "org.apache.hadoop.fs.s3a.s3guard.LocalMetadataStore";
+
+ /**
+ * Use DynamoDB for the metadata: {@value}.
+ */
+ @InterfaceStability.Unstable
+ public static final String S3GUARD_METASTORE_DYNAMO
+ = "org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore";
+
+ @InterfaceAudience.Private
+ public static final String SSE_C_NO_KEY_ERROR = S3AEncryptionMethods.SSE_C
+ .getMethod() +" is enabled and no encryption key is provided.";
+
+
+ @InterfaceAudience.Private
+ public static final String SSE_S3_WITH_KEY_ERROR = S3AEncryptionMethods.SSE_S3
+ .getMethod() +" is configured and an " + "encryption key is provided";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f7f2baf/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f7f2baf/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --cc hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index b93e567,8b1a6d0..c3f7a87
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@@ -2073,9 -2070,8 +2187,10 @@@ public class S3AFileSystem extends File
if (blockFactory != null) {
sb.append(", blockFactory=").append(blockFactory);
}
+ sb.append(", metastore=").append(metadataStore);
+ sb.append(", authoritative=").append(allowAuthoritative);
- sb.append(", executor=").append(threadPoolExecutor);
+ sb.append(", boundedExecutor=").append(boundedThreadPool);
+ sb.append(", unboundedExecutor=").append(unboundedThreadPool);
sb.append(", statistics {")
.append(statistics)
.append("}");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f7f2baf/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org