You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by el...@apache.org on 2018/08/24 20:58:49 UTC
hadoop git commit: HDDS-364. Update open container replica
information in SCM during DN register. Contributed by Ajay Kumar.
Repository: hadoop
Updated Branches:
refs/heads/trunk 8563fd67b -> a5eba2550
HDDS-364. Update open container replica information in SCM during DN register. Contributed by Ajay Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a5eba255
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a5eba255
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a5eba255
Branch: refs/heads/trunk
Commit: a5eba25506a4ca7ac9efa9b60b204c8cf1aa4160
Parents: 8563fd6
Author: Márton Elek <el...@apache.org>
Authored: Fri Aug 24 22:27:43 2018 +0200
Committer: Márton Elek <el...@apache.org>
Committed: Fri Aug 24 22:42:51 2018 +0200
----------------------------------------------------------------------
.../hdds/scm/container/ContainerMapping.java | 25 ++++++++++++++------
.../scm/container/ContainerReportHandler.java | 3 ++-
.../hadoop/hdds/scm/container/Mapping.java | 2 +-
.../scm/server/SCMDatanodeProtocolServer.java | 2 +-
.../scm/container/TestContainerMapping.java | 19 ++++++++++++---
.../container/closer/TestContainerCloser.java | 2 +-
.../commandhandler/TestBlockDeletion.java | 3 ++-
7 files changed, 41 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5eba255/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
index ba95764..b000bfd 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerMapping.java
@@ -505,15 +505,26 @@ public class ContainerMapping implements Mapping {
*/
@Override
public void processContainerReports(DatanodeDetails datanodeDetails,
- ContainerReportsProto reports)
+ ContainerReportsProto reports, boolean isRegisterCall)
throws IOException {
List<StorageContainerDatanodeProtocolProtos.ContainerInfo>
containerInfos = reports.getReportsList();
PendingDeleteStatusList pendingDeleteStatusList =
new PendingDeleteStatusList(datanodeDetails);
- for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
+ for (StorageContainerDatanodeProtocolProtos.ContainerInfo contInfo :
containerInfos) {
- byte[] dbKey = Longs.toByteArray(datanodeState.getContainerID());
+ // Update replica info during registration process.
+ if (isRegisterCall) {
+ try {
+ getStateManager().addContainerReplica(ContainerID.
+ valueof(contInfo.getContainerID()), datanodeDetails);
+ } catch (Exception ex) {
+ // Continue to next one after logging the error.
+ LOG.error("Error while adding replica for containerId {}.",
+ contInfo.getContainerID(), ex);
+ }
+ }
+ byte[] dbKey = Longs.toByteArray(contInfo.getContainerID());
lock.lock();
try {
byte[] containerBytes = containerStore.get(dbKey);
@@ -522,12 +533,12 @@ public class ContainerMapping implements Mapping {
HddsProtos.SCMContainerInfo.PARSER.parseFrom(containerBytes);
HddsProtos.SCMContainerInfo newState =
- reconcileState(datanodeState, knownState, datanodeDetails);
+ reconcileState(contInfo, knownState, datanodeDetails);
- if (knownState.getDeleteTransactionId() > datanodeState
+ if (knownState.getDeleteTransactionId() > contInfo
.getDeleteTransactionId()) {
pendingDeleteStatusList
- .addPendingDeleteStatus(datanodeState.getDeleteTransactionId(),
+ .addPendingDeleteStatus(contInfo.getDeleteTransactionId(),
knownState.getDeleteTransactionId(),
knownState.getContainerID());
}
@@ -558,7 +569,7 @@ public class ContainerMapping implements Mapping {
LOG.error("Error while processing container report from datanode :" +
" {}, for container: {}, reason: container doesn't exist in" +
"container database.", datanodeDetails,
- datanodeState.getContainerID());
+ contInfo.getContainerID());
}
} finally {
lock.unlock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5eba255/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
index b26eed2..5a9e726 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerReportHandler.java
@@ -84,7 +84,8 @@ public class ContainerReportHandler implements
try {
//update state in container db and trigger close container events
- containerMapping.processContainerReports(datanodeOrigin, containerReport);
+ containerMapping
+ .processContainerReports(datanodeOrigin, containerReport, false);
Set<ContainerID> containerIds = containerReport.getReportsList().stream()
.map(containerProto -> containerProto.getContainerID())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5eba255/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java
index ac84be4..f4b5bb2 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/Mapping.java
@@ -115,7 +115,7 @@ public interface Mapping extends Closeable {
* @param reports Container report
*/
void processContainerReports(DatanodeDetails datanodeDetails,
- ContainerReportsProto reports)
+ ContainerReportsProto reports, boolean isRegisterCall)
throws IOException;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5eba255/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
index 0d34787..9215803 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMDatanodeProtocolServer.java
@@ -196,7 +196,7 @@ public class SCMDatanodeProtocolServer implements
if (registeredCommand.getError()
== SCMRegisteredResponseProto.ErrorCode.success) {
scm.getScmContainerManager().processContainerReports(datanodeDetails,
- containerReportsProto);
+ containerReportsProto, true);
}
return getRegisteredResponse(registeredCommand);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5eba255/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
index 1e9c35b..b0b39f1 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerMapping.java
@@ -242,7 +242,7 @@ public class TestContainerMapping {
}
@Test
- public void testFullContainerReport() throws IOException {
+ public void testFullContainerReport() throws Exception {
ContainerInfo info = createContainer();
DatanodeDetails datanodeDetails = TestUtils.randomDatanodeDetails();
List<StorageContainerDatanodeProtocolProtos.ContainerInfo> reports =
@@ -266,13 +266,26 @@ public class TestContainerMapping {
.newBuilder();
crBuilder.addAllReports(reports);
- mapping.processContainerReports(datanodeDetails, crBuilder.build());
+ mapping.processContainerReports(datanodeDetails, crBuilder.build(), false);
ContainerInfo updatedContainer =
mapping.getContainer(info.getContainerID());
Assert.assertEquals(100000000L,
updatedContainer.getNumberOfKeys());
Assert.assertEquals(2000000000L, updatedContainer.getUsedBytes());
+
+ for (StorageContainerDatanodeProtocolProtos.ContainerInfo c : reports) {
+ LambdaTestUtils.intercept(SCMException.class, "No entry "
+ + "exist for containerId:", () -> mapping.getStateManager()
+ .getContainerReplicas(ContainerID.valueof(c.getContainerID())));
+ }
+
+ mapping.processContainerReports(TestUtils.randomDatanodeDetails(),
+ crBuilder.build(), true);
+ for (StorageContainerDatanodeProtocolProtos.ContainerInfo c : reports) {
+ Assert.assertTrue(mapping.getStateManager().getContainerReplicas(
+ ContainerID.valueof(c.getContainerID())).size() > 0);
+ }
}
@Test
@@ -301,7 +314,7 @@ public class TestContainerMapping {
ContainerReportsProto.newBuilder();
crBuilder.addAllReports(reports);
- mapping.processContainerReports(datanodeDetails, crBuilder.build());
+ mapping.processContainerReports(datanodeDetails, crBuilder.build(), false);
ContainerInfo updatedContainer =
mapping.getContainer(info.getContainerID());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5eba255/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java
index 608bb92..210df08 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/closer/TestContainerCloser.java
@@ -223,6 +223,6 @@ public class TestContainerCloser {
.setDeleteTransactionId(0);
reports.addReports(ciBuilder);
mapping.processContainerReports(TestUtils.randomDatanodeDetails(),
- reports.build());
+ reports.build(), false);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5eba255/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
index 94cdf61..5794151 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
@@ -187,7 +187,8 @@ public class TestBlockDeletion {
logCapturer.clearOutput();
scm.getScmContainerManager().processContainerReports(
- cluster.getHddsDatanodes().get(0).getDatanodeDetails(), dummyReport);
+ cluster.getHddsDatanodes().get(0).getDatanodeDetails(), dummyReport,
+ false);
// wait for event to be handled by event handler
Thread.sleep(1000);
String output = logCapturer.getOutput();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org