You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2018/08/10 23:32:25 UTC
[25/25] hadoop git commit: Merge branch 'trunk' into HDFS-12943
Merge branch 'trunk' into HDFS-12943
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71a35813
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71a35813
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71a35813
Branch: refs/heads/HDFS-12943
Commit: 71a358131bd7430c95426a0d1d26771a6dfd1476
Parents: cc6f80f a2a8c48
Author: Chao Sun <su...@apache.org>
Authored: Fri Aug 10 16:31:17 2018 -0700
Committer: Chao Sun <su...@apache.org>
Committed: Fri Aug 10 16:31:17 2018 -0700
----------------------------------------------------------------------
LICENSE.txt | 2 +-
.../src/main/resources/core-default.xml | 18 +-
hadoop-hdds/common/pom.xml | 34 ++
.../java/org/apache/hadoop/ozone/OzoneAcl.java | 8 +-
.../apache/hadoop/utils/HddsVersionInfo.java | 182 +++++++
.../main/proto/DatanodeContainerProtocol.proto | 1 +
.../main/resources/hdds-version-info.properties | 26 +
.../container/common/impl/ContainerData.java | 1 -
.../container/keyvalue/KeyValueContainer.java | 54 +-
.../container/keyvalue/KeyValueHandler.java | 39 +-
.../container/keyvalue/helpers/KeyUtils.java | 50 +-
.../container/keyvalue/impl/KeyManagerImpl.java | 4 +-
.../keyvalue/interfaces/KeyManager.java | 3 +-
.../keyvalue/TestKeyValueContainer.java | 16 -
.../container/keyvalue/TestKeyValueHandler.java | 55 +-
.../hadoop/hdds/server/events/EventQueue.java | 7 +-
.../scm/container/ContainerReportHandler.java | 107 +++-
.../replication/ReplicationActivityStatus.java | 86 +++
.../ReplicationActivityStatusMXBean.java | 28 +
.../replication/ReplicationRequest.java | 28 +-
.../hadoop/hdds/scm/events/SCMEvents.java | 9 +
.../hdds/scm/node/states/Node2ContainerMap.java | 10 +-
.../hdds/scm/node/states/ReportResult.java | 18 +-
.../scm/server/StorageContainerManager.java | 27 +-
.../container/TestContainerReportHandler.java | 228 ++++++++
.../scm/node/states/Node2ContainerMapTest.java | 308 -----------
.../scm/node/states/TestNode2ContainerMap.java | 328 +++++++++++
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 +
...yAliasMapProtocolClientSideTranslatorPB.java | 6 +
.../qjournal/client/QuorumJournalManager.java | 11 +-
.../aliasmap/InMemoryLevelDBAliasMapServer.java | 8 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 1 -
.../src/main/resources/hdfs-default.xml | 18 +
.../impl/TestInMemoryLevelDBAliasMapClient.java | 39 ++
hadoop-ozone/common/pom.xml | 35 ++
hadoop-ozone/common/src/main/bin/ozone | 2 +-
.../hadoop/ozone/util/OzoneVersionInfo.java | 213 ++++++++
.../resources/ozone-version-info.properties | 27 +
.../common/impl/TestContainerPersistence.java | 8 -
.../ozone/scm/TestCommittedBlockLengthAPI.java | 216 --------
.../TestGetCommittedBlockLengthAndPutKey.java | 254 +++++++++
.../fs/s3a/AWSCredentialProviderList.java | 101 +++-
.../org/apache/hadoop/fs/s3a/Constants.java | 19 +-
.../hadoop/fs/s3a/DefaultS3ClientFactory.java | 190 ++-----
.../fs/s3a/InconsistentAmazonS3Client.java | 10 +
.../fs/s3a/InconsistentS3ClientFactory.java | 11 +
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 35 +-
.../apache/hadoop/fs/s3a/S3ARetryPolicy.java | 4 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 245 ++++++++-
.../apache/hadoop/fs/s3a/S3ClientFactory.java | 7 +-
.../s3a/auth/AssumedRoleCredentialProvider.java | 78 ++-
.../fs/s3a/auth/NoAuthWithAWSException.java | 37 ++
.../apache/hadoop/fs/s3a/auth/RoleModel.java | 8 +
.../apache/hadoop/fs/s3a/auth/RolePolicies.java | 143 ++++-
.../hadoop/fs/s3a/auth/STSClientFactory.java | 78 +++
.../fs/s3a/s3guard/DynamoDBClientFactory.java | 18 +-
.../fs/s3a/s3guard/DynamoDBMetadataStore.java | 62 ++-
.../markdown/tools/hadoop-aws/assumed_roles.md | 191 +++++--
.../src/site/markdown/tools/hadoop-aws/index.md | 6 +-
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 117 ++--
.../fs/s3a/ITestS3ATemporaryCredentials.java | 71 +--
.../fs/s3a/ITestS3GuardListConsistency.java | 68 ++-
.../hadoop/fs/s3a/ITestS3GuardWriteBack.java | 57 +-
.../hadoop/fs/s3a/MockS3ClientFactory.java | 6 +-
.../fs/s3a/TestS3AAWSCredentialsProvider.java | 76 ++-
.../hadoop/fs/s3a/auth/ITestAssumeRole.java | 151 +++++-
.../auth/ITestAssumedRoleCommitOperations.java | 5 +-
.../hadoop/fs/s3a/auth/RoleTestUtils.java | 24 +-
.../s3guard/AbstractS3GuardToolTestBase.java | 7 +-
.../s3a/s3guard/ITestS3GuardConcurrentOps.java | 147 ++---
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 8 +-
.../hadoop/yarn/webapp/view/JQueryUI.java | 4 +-
.../webapps/static/dt-1.10.7/css/demo_page.css | 110 ++++
.../webapps/static/dt-1.10.7/css/demo_table.css | 538 +++++++++++++++++++
.../webapps/static/dt-1.10.7/css/jui-dt.css | 322 +++++++++++
.../static/dt-1.10.7/images/Sorting icons.psd | Bin 0 -> 27490 bytes
.../static/dt-1.10.7/images/back_disabled.jpg | Bin 0 -> 612 bytes
.../static/dt-1.10.7/images/back_enabled.jpg | Bin 0 -> 807 bytes
.../webapps/static/dt-1.10.7/images/favicon.ico | Bin 0 -> 894 bytes
.../dt-1.10.7/images/forward_disabled.jpg | Bin 0 -> 635 bytes
.../static/dt-1.10.7/images/forward_enabled.jpg | Bin 0 -> 852 bytes
.../static/dt-1.10.7/images/sort_asc.png | Bin 0 -> 263 bytes
.../dt-1.10.7/images/sort_asc_disabled.png | Bin 0 -> 252 bytes
.../static/dt-1.10.7/images/sort_both.png | Bin 0 -> 282 bytes
.../static/dt-1.10.7/images/sort_desc.png | Bin 0 -> 260 bytes
.../dt-1.10.7/images/sort_desc_disabled.png | Bin 0 -> 251 bytes
.../dt-1.10.7/js/jquery.dataTables.min.js | 160 ++++++
.../webapps/static/dt-1.9.4/css/demo_page.css | 110 ----
.../webapps/static/dt-1.9.4/css/demo_table.css | 538 -------------------
.../webapps/static/dt-1.9.4/css/jui-dt.css | 322 -----------
.../static/dt-1.9.4/images/Sorting icons.psd | Bin 27490 -> 0 bytes
.../static/dt-1.9.4/images/back_disabled.jpg | Bin 612 -> 0 bytes
.../static/dt-1.9.4/images/back_enabled.jpg | Bin 807 -> 0 bytes
.../webapps/static/dt-1.9.4/images/favicon.ico | Bin 894 -> 0 bytes
.../static/dt-1.9.4/images/forward_disabled.jpg | Bin 635 -> 0 bytes
.../static/dt-1.9.4/images/forward_enabled.jpg | Bin 852 -> 0 bytes
.../webapps/static/dt-1.9.4/images/sort_asc.png | Bin 263 -> 0 bytes
.../dt-1.9.4/images/sort_asc_disabled.png | Bin 252 -> 0 bytes
.../static/dt-1.9.4/images/sort_both.png | Bin 282 -> 0 bytes
.../static/dt-1.9.4/images/sort_desc.png | Bin 260 -> 0 bytes
.../dt-1.9.4/images/sort_desc_disabled.png | Bin 251 -> 0 bytes
.../static/dt-1.9.4/js/jquery.dataTables.min.js | 157 ------
.../container/ContainerImpl.java | 13 +-
.../launcher/ContainerLaunch.java | 12 +-
.../launcher/ContainersLauncher.java | 14 +-
.../container/TestContainer.java | 46 +-
.../server/resourcemanager/RMAppManager.java | 81 ++-
.../server/resourcemanager/rmapp/RMApp.java | 6 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 8 +-
.../scheduler/MutableConfigurationProvider.java | 7 +
.../capacity/AutoCreatedLeafQueueConfig.java | 5 +
.../capacity/QueueManagementChange.java | 2 +-
.../QueueManagementDynamicEditPolicy.java | 36 +-
.../conf/MutableCSConfigurationProvider.java | 5 +
.../GuaranteedOrZeroCapacityOverTimePolicy.java | 50 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 12 +
.../constraint/AllocationTagsManager.java | 5 +
.../resourcemanager/webapp/RMWebServices.java | 34 ++
.../resourcemanager/webapp/dao/ConfInfo.java | 72 +++
.../yarn/server/resourcemanager/MockNodes.java | 6 +-
.../server/resourcemanager/TestAppManager.java | 241 +++++++--
.../resourcemanager/TestResourceManager.java | 16 +-
.../applicationsmanager/MockAsm.java | 11 +
.../server/resourcemanager/rmapp/MockRMApp.java | 20 +
.../TestCapacitySchedulerAsyncScheduling.java | 69 +++
.../scheduler/capacity/TestUtils.java | 2 +
.../constraint/TestAllocationTagsManager.java | 37 ++
.../TestPlacementConstraintsUtil.java | 51 +-
.../TestRMWebServicesConfigurationMutation.java | 40 ++
.../src/site/markdown/DockerContainers.md | 194 ++++++-
.../src/site/markdown/ResourceManagerHA.md | 2 +-
.../src/site/markdown/ResourceManagerRest.md | 40 ++
.../src/site/markdown/yarn-service/Examples.md | 5 +-
.../site/markdown/yarn-service/QuickStart.md | 1 +
134 files changed, 5438 insertions(+), 2466 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71a35813/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71a35813/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
index 928d901,4faaa98..7b749f4
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
@@@ -136,16 -124,7 +136,14 @@@ public class QuorumJournalManager imple
this.nsInfo = nsInfo;
this.nameServiceId = nameServiceId;
this.loggers = new AsyncLoggerSet(createLoggers(loggerFactory));
- this.connectionFactory = URLConnectionFactory
- .newDefaultURLConnectionFactory(conf);
+ this.maxTxnsPerRpc =
+ conf.getInt(QJM_RPC_MAX_TXNS_KEY, QJM_RPC_MAX_TXNS_DEFAULT);
+ Preconditions.checkArgument(maxTxnsPerRpc > 0,
+ "Must specify %s greater than 0!", QJM_RPC_MAX_TXNS_KEY);
+ this.inProgressTailingEnabled = conf.getBoolean(
+ DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY,
+ DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_DEFAULT);
// Configure timeouts.
this.startSegmentTimeoutMs = conf.getInt(
DFSConfigKeys.DFS_QJOURNAL_START_SEGMENT_TIMEOUT_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71a35813/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index e002323,fd856a3..bfdda90
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@@ -1765,24 -1755,8 +1765,23 @@@ public class NameNode extends Reconfigu
state.setState(haContext, STANDBY_STATE);
}
+ synchronized void transitionToObserver()
+ throws ServiceFailedException, AccessControlException {
+ namesystem.checkSuperuserPrivilege();
+ if (!haEnabled) {
+ throw new ServiceFailedException("HA for namenode is not enabled");
+ }
+ // Transition from ACTIVE to OBSERVER is forbidden.
+ if (state == ACTIVE_STATE) {
+ throw new ServiceFailedException(
+ "Cannot transition from '" + ACTIVE_STATE + "' to '" +
+ OBSERVER_STATE + "'");
+ }
+ state.setState(haContext, OBSERVER_STATE);
+ }
+
synchronized HAServiceStatus getServiceStatus()
throws ServiceFailedException, AccessControlException {
- namesystem.checkSuperuserPrivilege();
if (!haEnabled) {
throw new ServiceFailedException("HA for namenode is not enabled");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71a35813/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org