You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/08/26 00:45:00 UTC
[48/50] [abbrv] hadoop git commit: Merge commit
'96c4575d7373079becfa3e3db29ba98e6fb86388' into HDFS-12943
Merge commit '96c4575d7373079becfa3e3db29ba98e6fb86388' into HDFS-12943
# Conflicts:
# hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournal.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8aeb2d89
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8aeb2d89
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8aeb2d89
Branch: refs/heads/HDFS-12943
Commit: 8aeb2d8942116ec924f319569949516055f3f5c2
Parents: da18227 96c4575
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Sat Aug 25 17:28:33 2018 -0700
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Sat Aug 25 17:28:33 2018 -0700
----------------------------------------------------------------------
dev-support/byteman/README.md | 31 +
dev-support/byteman/hadooprpc.btm | 44 ++
.../fs/CommonConfigurationKeysPublic.java | 9 +
.../apache/hadoop/util/ShutdownHookManager.java | 169 ++++-
.../src/main/resources/core-default.xml | 16 +
.../site/markdown/InterfaceClassification.md | 7 +-
.../hadoop/util/TestShutdownHookManager.java | 328 ++++++--
.../src/main/compose/ozone/docker-config | 2 +
.../container/common/helpers/ContainerInfo.java | 1 +
.../org/apache/hadoop/utils/db/DBStore.java | 93 +++
.../org/apache/hadoop/utils/db/RDBStore.java | 252 +++++++
.../hadoop/utils/db/RDBStoreIterator.java | 88 +++
.../org/apache/hadoop/utils/db/RDBTable.java | 173 +++++
.../java/org/apache/hadoop/utils/db/Table.java | 150 ++++
.../apache/hadoop/utils/db/TableIterator.java | 50 ++
.../apache/hadoop/utils/db/package-info.java | 22 +
.../apache/hadoop/utils/db/TestRDBStore.java | 246 ++++++
.../hadoop/utils/db/TestRDBTableStore.java | 189 +++++
.../apache/hadoop/utils/db/package-info.java | 22 +
.../apache/hadoop/hdds/scm/HddsServerUtil.java | 6 +-
.../common/helpers/ContainerReport.java | 14 -
.../common/helpers/KeyValueContainerReport.java | 117 +++
.../container/common/impl/ContainerData.java | 53 --
.../common/impl/ContainerDataYaml.java | 94 ++-
.../container/common/impl/ContainerSet.java | 50 +-
.../RandomContainerDeletionChoosingPolicy.java | 3 +-
...NOrderedContainerDeletionChoosingPolicy.java | 24 +-
.../container/common/interfaces/Container.java | 39 +-
.../ContainerDeletionChoosingPolicy.java | 13 +
.../common/interfaces/ContainerPacker.java | 58 ++
.../report/CommandStatusReportPublisher.java | 2 +-
.../common/report/ContainerReportPublisher.java | 2 +-
.../common/report/NodeReportPublisher.java | 2 +-
.../statemachine/DatanodeStateMachine.java | 3 +-
.../statemachine/EndpointStateMachine.java | 9 +-
.../container/keyvalue/KeyValueContainer.java | 174 ++++-
.../keyvalue/KeyValueContainerData.java | 54 ++
.../container/keyvalue/KeyValueHandler.java | 19 +-
.../container/keyvalue/TarContainerPacker.java | 249 +++++++
.../keyvalue/helpers/KeyValueContainerUtil.java | 22 +-
.../common/TestKeyValueContainerData.java | 4 +-
.../keyvalue/TestKeyValueContainer.java | 95 ++-
.../keyvalue/TestTarContainerPacker.java | 231 ++++++
.../hdds/scm/container/ContainerMapping.java | 23 +-
.../scm/container/closer/ContainerCloser.java | 9 +-
.../scm/container/TestContainerMapping.java | 8 +
.../federation/resolver/MountTableResolver.java | 37 +-
.../server/federation/router/RBFConfigKeys.java | 4 +
.../federation/router/RouterRpcClient.java | 18 +-
.../federation/router/RouterRpcServer.java | 3 +-
.../src/main/resources/hdfs-rbf-default.xml | 9 +
.../hdfs/server/federation/MockResolver.java | 3 +
.../resolver/TestMountTableResolver.java | 31 +
.../server/federation/router/TestRouter.java | 18 +
.../hdfs/qjournal/client/AsyncLogger.java | 3 +-
.../hdfs/qjournal/client/AsyncLoggerSet.java | 4 +-
.../hdfs/qjournal/client/IPCLoggerChannel.java | 5 +-
.../qjournal/client/QuorumJournalManager.java | 4 +-
.../qjournal/protocol/QJournalProtocol.java | 2 +-
.../QJournalProtocolServerSideTranslatorPB.java | 2 +-
.../QJournalProtocolTranslatorPB.java | 6 +-
.../hadoop/hdfs/qjournal/server/JNStorage.java | 4 +-
.../hadoop/hdfs/qjournal/server/Journal.java | 6 +-
.../qjournal/server/JournalNodeRpcServer.java | 5 +-
.../server/namenode/BackupJournalManager.java | 2 +-
.../namenode/ErasureCodingPolicyManager.java | 15 +-
.../server/namenode/FSDirErasureCodingOp.java | 22 +-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 5 +-
.../hadoop/hdfs/server/namenode/FSImage.java | 5 +-
.../hdfs/server/namenode/FSNamesystem.java | 30 +-
.../server/namenode/FileJournalManager.java | 2 +-
.../hdfs/server/namenode/JournalManager.java | 2 +-
.../hadoop/hdfs/server/namenode/JournalSet.java | 2 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 4 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 8 +-
.../src/main/proto/QJournalProtocol.proto | 1 +
.../qjournal/client/TestEpochsAreUnique.java | 2 +-
.../hdfs/qjournal/client/TestQJMWithFaults.java | 6 +-
.../client/TestQuorumJournalManager.java | 4 +-
.../client/TestQuorumJournalManagerUnit.java | 4 +-
.../hdfs/qjournal/server/TestJournal.java | 17 +-
.../hdfs/qjournal/server/TestJournalNode.java | 4 +-
.../qjournal/server/TestJournalNodeMXBean.java | 2 +-
.../qjournal/server/TestJournalNodeSync.java | 2 +-
.../server/namenode/TestGenericJournalConf.java | 2 +-
.../server/namenode/TestNamenodeRetryCache.java | 2 +-
.../namenode/ha/TestRetryCacheWithHA.java | 2 +-
.../ozone/TestStorageContainerManager.java | 19 +-
.../common/TestBlockDeletingService.java | 8 +-
.../commandhandler/TestBlockDeletion.java | 19 +-
.../hadoop/ozone/web/client/TestKeys.java | 10 +-
.../contract/AbstractContractDistCpTest.java | 2 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 8 +-
.../yarn/conf/TestYarnConfigurationFields.java | 2 +
.../yarn/service/client/ApiServiceClient.java | 20 +
.../hadoop/yarn/service/webapp/ApiServer.java | 12 +-
.../hadoop/yarn/service/ClientAMService.java | 2 +-
.../hadoop/yarn/service/ServiceEvent.java | 25 +
.../hadoop/yarn/service/ServiceManager.java | 127 +++-
.../hadoop/yarn/service/ServiceScheduler.java | 15 +-
.../yarn/service/api/records/ServiceState.java | 2 +-
.../yarn/service/client/ServiceClient.java | 100 ++-
.../yarn/service/component/Component.java | 16 +-
.../yarn/service/component/ComponentEvent.java | 10 +
.../component/instance/ComponentInstance.java | 5 +
.../yarn/service/utils/ServiceApiUtil.java | 44 ++
.../src/main/proto/ClientAMProtocol.proto | 1 +
.../hadoop/yarn/service/TestServiceApiUtil.java | 653 ----------------
.../hadoop/yarn/service/TestServiceManager.java | 299 +++++---
.../yarn/service/TestYarnNativeServices.java | 35 +
.../yarn/service/utils/TestServiceApiUtil.java | 743 +++++++++++++++++++
.../hadoop/yarn/client/cli/ApplicationCLI.java | 20 +-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 4 +
.../hadoop/yarn/client/AMRMClientUtils.java | 47 ++
.../hadoop/yarn/client/api/AppAdminClient.java | 12 +
.../hadoop/yarn/server/AMRMClientRelayer.java | 130 ++--
.../LocalityMulticastAMRMProxyPolicy.java | 64 +-
.../utils/FederationStateStoreFacade.java | 9 +
.../server/uam/UnmanagedApplicationManager.java | 2 +-
.../yarn/server/TestAMRMClientRelayer.java | 53 +-
.../TestLocalityMulticastAMRMProxyPolicy.java | 91 ++-
.../utils/FederationPoliciesTestUtil.java | 7 +-
.../server/nodemanager/DeletionService.java | 25 +-
.../amrmproxy/FederationInterceptor.java | 6 +-
.../containermanager/ContainerManagerImpl.java | 26 +-
.../localizer/LocalResourcesTrackerImpl.java | 5 +
.../localizer/ResourceLocalizationService.java | 68 +-
.../recovery/NMLeveldbStateStoreService.java | 412 ++++++----
.../recovery/NMNullStateStoreService.java | 2 +-
.../recovery/NMStateStoreService.java | 55 +-
.../nodemanager/recovery/RecoveryIterator.java | 41 +
.../security/NMContainerTokenSecretManager.java | 27 +-
.../security/NMTokenSecretManagerInNM.java | 15 +-
.../TestResourceLocalizationService.java | 12 +-
.../recovery/NMMemoryStateStoreService.java | 82 +-
.../TestNMLeveldbStateStoreService.java | 216 ++++--
.../ApplicationMasterService.java | 9 +-
.../resourcemanager/RMActiveServiceContext.java | 16 +
.../yarn/server/resourcemanager/RMContext.java | 8 +-
.../server/resourcemanager/RMContextImpl.java | 14 +-
.../server/resourcemanager/ResourceManager.java | 12 +
.../scheduler/AbstractYarnScheduler.java | 10 +
.../scheduler/AppSchedulingInfo.java | 28 +-
.../scheduler/ClusterNodeTracker.java | 61 ++
.../scheduler/YarnScheduler.java | 10 +
.../scheduler/activities/ActivitiesLogger.java | 32 +-
.../scheduler/activities/ActivitiesManager.java | 8 +-
.../scheduler/capacity/AbstractCSQueue.java | 16 +-
.../scheduler/capacity/CSQueue.java | 6 +
.../scheduler/capacity/CapacityScheduler.java | 77 +-
.../CapacitySchedulerConfiguration.java | 116 +++
.../scheduler/capacity/LeafQueue.java | 49 +-
.../scheduler/capacity/ParentQueue.java | 4 +-
.../allocator/RegularContainerAllocator.java | 35 +-
.../common/ApplicationSchedulingConfig.java | 4 +
.../scheduler/common/fica/FiCaSchedulerApp.java | 23 +
.../placement/AppPlacementAllocator.java | 6 +
.../LocalityAppPlacementAllocator.java | 39 +-
.../placement/MultiNodeLookupPolicy.java | 67 ++
.../placement/MultiNodePolicySpec.java | 56 ++
.../scheduler/placement/MultiNodeSorter.java | 167 +++++
.../placement/MultiNodeSortingManager.java | 139 ++++
.../ResourceUsageMultiNodeLookupPolicy.java | 79 ++
.../SingleConstraintAppPlacementAllocator.java | 179 ++---
.../webapp/RMAppAttemptBlock.java | 24 +-
.../resourcemanager/webapp/dao/AppInfo.java | 10 +
.../webapp/dao/ResourceRequestInfo.java | 52 +-
.../yarn/server/resourcemanager/MockRM.java | 35 +-
.../reservation/ReservationSystemTestUtil.java | 3 +
.../scheduler/TestAppSchedulingInfo.java | 3 +-
.../capacity/CapacitySchedulerTestBase.java | 13 +
.../capacity/TestCapacityScheduler.java | 15 -
.../TestCapacitySchedulerMultiNodes.java | 166 +++++
.../TestCapacitySchedulerNodeLabelUpdate.java | 70 ++
...estSchedulingRequestContainerAllocation.java | 438 ++++++++++-
...stSingleConstraintAppPlacementAllocator.java | 78 --
.../reader/TimelineReaderWebServicesUtils.java | 6 +-
177 files changed, 7363 insertions(+), 2006 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManagerUnit.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManagerUnit.java
index b38a527,9e1e3bb..dd55af2
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManagerUnit.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManagerUnit.java
@@@ -17,13 -17,11 +17,14 @@@
*/
package org.apache.hadoop.hdfs.qjournal.client;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.anyLong;
+ import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.eq;
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
import java.io.IOException;
import java.net.URI;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournal.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournal.java
index d419f76,b8d2652..2f51275
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournal.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournal.java
@@@ -78,11 -71,9 +78,11 @@@ public class TestJournal
public void setup() throws Exception {
FileUtil.fullyDelete(TEST_LOG_DIR);
conf = new Configuration();
+ // Enable fetching edits via RPC
+ conf.setBoolean(DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY, true);
journal = new Journal(conf, TEST_LOG_DIR, JID, StartupOption.REGULAR,
mockErrorReporter);
- journal.format(FAKE_NSINFO);
+ journal.format(FAKE_NSINFO, false);
}
@After
@@@ -444,41 -435,14 +444,52 @@@
}
@Test
+ public void testReadFromCache() throws Exception {
+ journal.newEpoch(FAKE_NSINFO, 1);
+ journal.startLogSegment(makeRI(1), 1,
+ NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
+ journal.journal(makeRI(2), 1, 1, 5, QJMTestUtil.createTxnData(1, 5));
+ journal.journal(makeRI(3), 1, 6, 5, QJMTestUtil.createTxnData(6, 5));
+ journal.journal(makeRI(4), 1, 11, 5, QJMTestUtil.createTxnData(11, 5));
+ assertJournaledEditsTxnCountAndContents(1, 7, 7,
+ NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
+ assertJournaledEditsTxnCountAndContents(1, 30, 15,
+ NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
+
+ journal.finalizeLogSegment(makeRI(5), 1, 15);
+ int newLayoutVersion = NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION - 1;
+ journal.startLogSegment(makeRI(6), 16, newLayoutVersion);
+ journal.journal(makeRI(7), 16, 16, 5, QJMTestUtil.createTxnData(16, 5));
+
+ assertJournaledEditsTxnCountAndContents(16, 10, 20, newLayoutVersion);
+ }
+
+ private void assertJournaledEditsTxnCountAndContents(int startTxn,
+ int requestedMaxTxns, int expectedEndTxn, int layoutVersion)
+ throws Exception {
+ GetJournaledEditsResponseProto result =
+ journal.getJournaledEdits(startTxn, requestedMaxTxns);
+ int expectedTxnCount = expectedEndTxn - startTxn + 1;
+ ByteArrayOutputStream headerBytes = new ByteArrayOutputStream();
+ EditLogFileOutputStream.writeHeader(layoutVersion,
+ new DataOutputStream(headerBytes));
+ assertEquals(expectedTxnCount, result.getTxnCount());
+ assertArrayEquals(
+ Bytes.concat(
+ headerBytes.toByteArray(),
+ QJMTestUtil.createTxnData(startTxn, expectedTxnCount)),
+ result.getEditLog().toByteArray());
+ }
+
++ @Test
+ public void testFormatNonEmptyStorageDirectoriesWhenforceOptionIsTrue()
+ throws Exception {
+ try {
+ // Format again here and to format the non-empty directories in
+ // journal node.
+ journal.format(FAKE_NSINFO, true);
+ } catch (IOException ioe) {
+ fail("Format should be success with force option.");
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org