You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by el...@apache.org on 2019/11/11 18:14:06 UTC
[hadoop-ozone] branch master updated: HDDS-2325.
BenchMarkDatanodeDispatcher genesis test is failing with NPE
This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new a6f80c0 HDDS-2325. BenchMarkDatanodeDispatcher genesis test is failing with NPE
a6f80c0 is described below
commit a6f80c096b5320f50b6e9e9b4ba5f7c7e3544385
Author: Márton Elek <el...@apache.org>
AuthorDate: Mon Nov 11 19:09:55 2019 +0100
HDDS-2325. BenchMarkDatanodeDispatcher genesis test is failing with NPE
Closes #60
---
.../container/common/impl/HddsDispatcher.java | 5 ++
.../ozone/container/common/interfaces/Handler.java | 60 ++++++++--------
.../ozone/container/keyvalue/KeyValueHandler.java | 12 ++--
.../ozone/container/ozoneimpl/OzoneContainer.java | 59 ++++++++++------
.../container/common/impl/TestHddsDispatcher.java | 16 +++--
.../container/common/interfaces/TestHandler.java | 8 ++-
.../container/keyvalue/TestKeyValueHandler.java | 10 ++-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 7 +-
.../container/metrics/TestContainerMetrics.java | 9 ++-
.../container/server/TestContainerServer.java | 7 +-
.../server/TestSecureContainerServer.java | 7 +-
.../ozone/genesis/BenchMarkDatanodeDispatcher.java | 82 ++++++++++++----------
.../org/apache/hadoop/ozone/genesis/Genesis.java | 3 +-
13 files changed, 169 insertions(+), 116 deletions(-)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
index 7f46707..1feb500 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java
@@ -321,6 +321,11 @@ public class HddsDispatcher implements ContainerDispatcher, Auditor {
// Once container is marked unhealthy, all the subsequent write
// transactions will fail with UNHEALTHY_CONTAINER exception.
+ if (container == null) {
+ throw new NullPointerException(
+ "Error on creating containers " + result + " " + responseProto
+ .getMessage());
+ }
// For container to be moved to unhealthy state here, the container can
// only be in open or closing state.
State containerState = container.getContainerData().getState();
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
index 8c3b981..c623e14 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java
@@ -18,25 +18,19 @@
package org.apache.hadoop.ozone.container.common.interfaces;
-
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.util.function.Consumer;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerCommandRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerCommandResponseProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerType;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
@@ -54,41 +48,46 @@ public abstract class Handler {
protected final VolumeSet volumeSet;
protected String scmID;
protected final ContainerMetrics metrics;
+ protected String datanodeId;
+ private Consumer<ContainerReplicaProto> icrSender;
- private final StateContext context;
- private final DatanodeDetails datanodeDetails;
-
- protected Handler(Configuration config, StateContext context,
+ protected Handler(Configuration config, String datanodeId,
ContainerSet contSet, VolumeSet volumeSet,
- ContainerMetrics containerMetrics) {
+ ContainerMetrics containerMetrics,
+ Consumer<ContainerReplicaProto> icrSender) {
this.conf = config;
- this.context = context;
this.containerSet = contSet;
this.volumeSet = volumeSet;
this.metrics = containerMetrics;
- this.datanodeDetails = context.getParent().getDatanodeDetails();
+ this.datanodeId = datanodeId;
+ this.icrSender = icrSender;
}
public static Handler getHandlerForContainerType(
final ContainerType containerType, final Configuration config,
- final StateContext context, final ContainerSet contSet,
- final VolumeSet volumeSet, final ContainerMetrics metrics) {
+ final String datanodeId, final ContainerSet contSet,
+ final VolumeSet volumeSet, final ContainerMetrics metrics,
+ Consumer<ContainerReplicaProto> icrSender) {
switch (containerType) {
case KeyValueContainer:
- return new KeyValueHandler(config, context, contSet, volumeSet, metrics);
+ return new KeyValueHandler(config,
+ datanodeId, contSet, volumeSet, metrics,
+ icrSender);
default:
throw new IllegalArgumentException("Handler for ContainerType: " +
- containerType + "doesn't exist.");
+ containerType + "doesn't exist.");
}
}
/**
* Returns the Id of this datanode.
+ *
* @return datanode Id
*/
- protected DatanodeDetails getDatanodeDetails() {
- return datanodeDetails;
+ protected String getDatanodeId() {
+ return datanodeId;
}
+
/**
* This should be called whenever there is state change. It will trigger
* an ICR to SCM.
@@ -97,12 +96,8 @@ public abstract class Handler {
*/
protected void sendICR(final Container container)
throws StorageContainerException {
- IncrementalContainerReportProto icr = IncrementalContainerReportProto
- .newBuilder()
- .addReport(container.getContainerReport())
- .build();
- context.addReport(icr);
- context.getParent().triggerHeartbeat();
+ ContainerReplicaProto containerReport = container.getContainerReport();
+ icrSender.accept(containerReport);
}
public abstract ContainerCommandResponseProto handle(
@@ -175,8 +170,9 @@ public abstract class Handler {
* Deletes the given container.
*
* @param container container to be deleted
- * @param force if this is set to true, we delete container without checking
- * state of the container.
+ * @param force if this is set to true, we delete container without
+ * checking
+ * state of the container.
* @throws IOException
*/
public abstract void deleteContainer(Container container, boolean force)
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index 995b913..5da0b8f 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -27,6 +27,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Consumer;
import java.util.function.Function;
import org.apache.hadoop.conf.Configuration;
@@ -47,6 +48,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyValue;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.PutSmallFileRequestProto;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Type;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.hdds.scm.ByteStringConversion;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.common.helpers
@@ -60,7 +62,6 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.Handler;
import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.transport.server.ratis
.DispatcherContext;
import org.apache.hadoop.ozone.container.common.transport.server.ratis
@@ -109,9 +110,10 @@ public class KeyValueHandler extends Handler {
private final AutoCloseableLock containerCreationLock;
private final boolean doSyncWrite;
- public KeyValueHandler(Configuration config, StateContext context,
- ContainerSet contSet, VolumeSet volSet, ContainerMetrics metrics) {
- super(config, context, contSet, volSet, metrics);
+ public KeyValueHandler(Configuration config, String datanodeId,
+ ContainerSet contSet, VolumeSet volSet, ContainerMetrics metrics,
+ Consumer<ContainerReplicaProto> icrSender) {
+ super(config, datanodeId, contSet, volSet, metrics, icrSender);
containerType = ContainerType.KeyValueContainer;
blockManager = new BlockManagerImpl(config);
doSyncWrite =
@@ -220,7 +222,7 @@ public class KeyValueHandler extends Handler {
KeyValueContainerData newContainerData = new KeyValueContainerData(
containerID, maxContainerSize, request.getPipelineID(),
- getDatanodeDetails().getUuidString());
+ getDatanodeId());
// TODO: Add support to add metadataList to ContainerData. Add metadata
// to container during creation.
KeyValueContainer newContainer = new KeyValueContainer(
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
index 6bce22c..328cd91 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
@@ -18,16 +18,21 @@
package org.apache.hadoop.ozone.container.ozoneimpl;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Maps;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdds.protocol.datanode.proto
- .ContainerProtos.ContainerType;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos;
-import org.apache.hadoop.hdds.protocol.proto
- .StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerType;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.IncrementalContainerReportProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReportsProto;
import org.apache.hadoop.hdds.security.token.BlockTokenVerifier;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
@@ -42,24 +47,20 @@ import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerSp
import org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis;
import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
-
import org.apache.hadoop.ozone.container.keyvalue.statemachine.background.BlockDeletingService;
import org.apache.hadoop.ozone.container.replication.GrpcReplicationService;
-import org.apache.hadoop.ozone.container.replication
- .OnDemandContainerReplicationSource;
+import org.apache.hadoop.ozone.container.replication.OnDemandContainerReplicationSource;
import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Maps;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.*;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys.*;
-
/**
* Ozone main class sets up the network servers and initializes the container
* layer.
@@ -100,10 +101,22 @@ public class OzoneContainer {
buildContainerSet();
final ContainerMetrics metrics = ContainerMetrics.create(conf);
this.handlers = Maps.newHashMap();
+
+ Consumer<ContainerReplicaProto> icrSender = containerReplicaProto -> {
+ IncrementalContainerReportProto icr = IncrementalContainerReportProto
+ .newBuilder()
+ .addReport(containerReplicaProto)
+ .build();
+ context.addReport(icr);
+ context.getParent().triggerHeartbeat();
+ };
+
for (ContainerType containerType : ContainerType.values()) {
handlers.put(containerType,
Handler.getHandlerForContainerType(
- containerType, conf, context, containerSet, volumeSet, metrics));
+ containerType, conf,
+ context.getParent().getDatanodeDetails().getUuidString(),
+ containerSet, volumeSet, metrics, icrSender));
}
SecurityConfig secConf = new SecurityConfig(conf);
@@ -169,7 +182,6 @@ public class OzoneContainer {
}
-
/**
* Start background daemon thread for performing container integrity checks.
*/
@@ -240,13 +252,14 @@ public class OzoneContainer {
ContainerMetrics.remove();
}
-
@VisibleForTesting
public ContainerSet getContainerSet() {
return containerSet;
}
+
/**
* Returns container report.
+ *
* @return - container report.
*/
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
index e9e8da8..9db11d0 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestHddsDispatcher.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.WriteChunkRequestProto;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.ContainerAction;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto;
import org.apache.hadoop.ozone.common.Checksum;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
@@ -48,6 +49,7 @@ import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
import org.apache.hadoop.test.GenericTestUtils;
+
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.junit.Assert;
import org.junit.Test;
@@ -57,6 +59,7 @@ import java.io.File;
import java.io.IOException;
import java.util.Map;
import java.util.UUID;
+import java.util.function.Consumer;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
@@ -69,6 +72,9 @@ import static org.mockito.Mockito.verify;
*/
public class TestHddsDispatcher {
+ public static final Consumer<ContainerReplicaProto> NO_OP_ICR_SENDER =
+ c -> {};
+
@Test
public void testContainerCloseActionWhenFull() throws IOException {
String testDir = GenericTestUtils.getTempPath(
@@ -98,8 +104,9 @@ public class TestHddsDispatcher {
Map<ContainerType, Handler> handlers = Maps.newHashMap();
for (ContainerType containerType : ContainerType.values()) {
handlers.put(containerType,
- Handler.getHandlerForContainerType(containerType, conf, context,
- containerSet, volumeSet, metrics));
+ Handler.getHandlerForContainerType(containerType, conf,
+ context.getParent().getDatanodeDetails().getUuidString(),
+ containerSet, volumeSet, metrics, NO_OP_ICR_SENDER));
}
HddsDispatcher hddsDispatcher = new HddsDispatcher(
conf, containerSet, volumeSet, handlers, context, metrics, null);
@@ -214,8 +221,9 @@ public class TestHddsDispatcher {
Map<ContainerType, Handler> handlers = Maps.newHashMap();
for (ContainerType containerType : ContainerType.values()) {
handlers.put(containerType,
- Handler.getHandlerForContainerType(containerType, conf, context,
- containerSet, volumeSet, metrics));
+ Handler.getHandlerForContainerType(containerType, conf,
+ context.getParent().getDatanodeDetails().getUuidString(),
+ containerSet, volumeSet, metrics, NO_OP_ICR_SENDER));
}
HddsDispatcher hddsDispatcher = new HddsDispatcher(
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
index cb40f95..086b0fe 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/interfaces/TestHandler.java
@@ -19,16 +19,19 @@
package org.apache.hadoop.ozone.container.common.interfaces;
import com.google.common.collect.Maps;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
+import org.apache.hadoop.ozone.container.common.impl.TestHddsDispatcher;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
+
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
@@ -69,7 +72,10 @@ public class TestHandler {
ContainerProtos.ContainerType.values()) {
handlers.put(containerType,
Handler.getHandlerForContainerType(
- containerType, conf, context, containerSet, volumeSet, metrics));
+ containerType, conf,
+ context.getParent().getDatanodeDetails().getUuidString(),
+ containerSet, volumeSet, metrics,
+ TestHddsDispatcher.NO_OP_ICR_SENDER));
}
this.dispatcher = new HddsDispatcher(
conf, containerSet, volumeSet, handlers, null, metrics, null);
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
index 2c71fef..05b92cc 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java
@@ -244,8 +244,10 @@ public class TestKeyValueHandler {
Mockito.when(stateMachine.getDatanodeDetails())
.thenReturn(datanodeDetails);
Mockito.when(context.getParent()).thenReturn(stateMachine);
- KeyValueHandler keyValueHandler = new KeyValueHandler(conf, context, cset,
- volumeSet, metrics);
+ KeyValueHandler keyValueHandler = new KeyValueHandler(conf,
+ context.getParent().getDatanodeDetails().getUuidString(), cset,
+ volumeSet, metrics, c -> {
+ });
assertEquals("org.apache.hadoop.ozone.container.common" +
".volume.RoundRobinVolumeChoosingPolicy",
keyValueHandler.getVolumeChoosingPolicyForTesting()
@@ -255,7 +257,9 @@ public class TestKeyValueHandler {
conf.set(HDDS_DATANODE_VOLUME_CHOOSING_POLICY,
"org.apache.hadoop.ozone.container.common.impl.HddsDispatcher");
try {
- new KeyValueHandler(conf, context, cset, volumeSet, metrics);
+ new KeyValueHandler(conf,
+ context.getParent().getDatanodeDetails().getUuidString(),
+ cset, volumeSet, metrics, c->{});
} catch (RuntimeException ex) {
GenericTestUtils.assertExceptionContains("class org.apache.hadoop" +
".ozone.container.common.impl.HddsDispatcher not org.apache" +
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java
index e3ae56a..26603b6 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java
@@ -24,9 +24,11 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
+import org.apache.hadoop.ozone.container.common.impl.TestHddsDispatcher;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
+
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
@@ -41,7 +43,6 @@ import static org.junit.Assert.assertThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-
/**
* Test that KeyValueHandler fails certain operations when the
* container is unhealthy.
@@ -147,10 +148,10 @@ public class TestKeyValueHandlerWithUnhealthyContainer {
return new KeyValueHandler(
new OzoneConfiguration(),
- context,
+ stateMachine.getDatanodeDetails().getUuidString(),
mock(ContainerSet.class),
mock(VolumeSet.class),
- mock(ContainerMetrics.class));
+ mock(ContainerMetrics.class), TestHddsDispatcher.NO_OP_ICR_SENDER);
}
private KeyValueContainer getMockUnhealthyContainer() {
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
index 1d30653..0598f15 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java
@@ -22,6 +22,7 @@ import static org.apache.hadoop.test.MetricsAsserts.assertQuantileGauges;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
import com.google.common.collect.Maps;
+
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@@ -40,6 +41,7 @@ import org.apache.hadoop.ozone.container.ContainerTestHelper;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
+import org.apache.hadoop.ozone.container.common.impl.TestHddsDispatcher;
import org.apache.hadoop.ozone.container.common.interfaces.Handler;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
@@ -51,6 +53,7 @@ import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController;
import org.apache.hadoop.ozone.container.replication.GrpcReplicationService;
import org.apache.hadoop.ozone.container.replication.OnDemandContainerReplicationSource;
import org.apache.hadoop.test.GenericTestUtils;
+
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
@@ -104,8 +107,10 @@ public class TestContainerMetrics {
for (ContainerProtos.ContainerType containerType :
ContainerProtos.ContainerType.values()) {
handlers.put(containerType,
- Handler.getHandlerForContainerType(containerType, conf, context,
- containerSet, volumeSet, metrics));
+ Handler.getHandlerForContainerType(containerType, conf,
+ context.getParent().getDatanodeDetails().getUuidString(),
+ containerSet, volumeSet, metrics,
+ TestHddsDispatcher.NO_OP_ICR_SENDER));
}
HddsDispatcher dispatcher = new HddsDispatcher(conf, containerSet,
volumeSet, handlers, context, metrics, null);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
index d71c0e7..e382734 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.security.x509.certificate.client.DNCertificateClie
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
+import org.apache.hadoop.ozone.container.common.impl.TestHddsDispatcher;
import org.apache.hadoop.ozone.container.common.interfaces.Handler;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
@@ -224,8 +225,10 @@ public class TestContainerServer {
for (ContainerProtos.ContainerType containerType :
ContainerProtos.ContainerType.values()) {
handlers.put(containerType,
- Handler.getHandlerForContainerType(containerType, conf, context,
- containerSet, volumeSet, metrics));
+ Handler.getHandlerForContainerType(containerType, conf,
+ context.getParent().getDatanodeDetails().getUuidString(),
+ containerSet, volumeSet, metrics,
+ TestHddsDispatcher.NO_OP_ICR_SENDER));
}
HddsDispatcher dispatcher = new HddsDispatcher(
conf, containerSet, volumeSet, handlers, context, metrics, null);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
index 2253fe2..33f93e5 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.ozone.container.ContainerTestHelper;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
+import org.apache.hadoop.ozone.container.common.impl.TestHddsDispatcher;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
import org.apache.hadoop.ozone.container.common.interfaces.Handler;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
@@ -155,8 +156,10 @@ public class TestSecureContainerServer {
for (ContainerProtos.ContainerType containerType :
ContainerProtos.ContainerType.values()) {
handlers.put(containerType,
- Handler.getHandlerForContainerType(containerType, conf, context,
- containerSet, volumeSet, metrics));
+ Handler.getHandlerForContainerType(containerType, conf,
+ dd.getUuid().toString(),
+ containerSet, volumeSet, metrics,
+ TestHddsDispatcher.NO_OP_ICR_SENDER));
}
HddsDispatcher hddsDispatcher = new HddsDispatcher(
conf, containerSet, volumeSet, handlers, context, metrics,
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
index 28da134..1d087f9 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/BenchMarkDatanodeDispatcher.java
@@ -17,25 +17,42 @@
*/
package org.apache.hadoop.ozone.genesis;
-import com.google.common.collect.Maps;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.HddsUtils;
+import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumData;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.GetBlockRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.PutBlockRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ReadChunkRequestProto;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.WriteChunkRequestProto;
import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher;
import org.apache.hadoop.ozone.container.common.interfaces.Handler;
-import org.apache.hadoop.ozone.container.common.statemachine
- .DatanodeStateMachine.DatanodeStates;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine.DatanodeStates;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.common.volume.VolumeSet;
-import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+
+import com.google.common.collect.Maps;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.client.BlockID;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Scope;
@@ -43,28 +60,6 @@ import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.TearDown;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .PutBlockRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .GetBlockRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ContainerCommandRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .ReadChunkRequestProto;
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
- .WriteChunkRequestProto;
-
-import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-
/**
* Benchmarks DatanodeDispatcher class.
*/
@@ -83,6 +78,7 @@ public class BenchMarkDatanodeDispatcher {
private static final int INIT_CONTAINERS = 100;
private static final int INIT_KEYS = 50;
private static final int INIT_CHUNKS = 100;
+ public static final int CHUNK_SIZE = 1048576;
private List<Long> containers;
private List<Long> keys;
@@ -94,8 +90,8 @@ public class BenchMarkDatanodeDispatcher {
datanodeUuid = UUID.randomUUID().toString();
// 1 MB of data
- data = ByteString.copyFromUtf8(RandomStringUtils.randomAscii(1048576));
- random = new Random();
+ data = ByteString.copyFromUtf8(RandomStringUtils.randomAscii(CHUNK_SIZE));
+ random = new Random();
Configuration conf = new OzoneConfiguration();
baseDir = System.getProperty("java.io.tmpdir") + File.separator +
datanodeUuid;
@@ -103,6 +99,9 @@ public class BenchMarkDatanodeDispatcher {
// data directory
conf.set("dfs.datanode.data.dir", baseDir + File.separator + "data");
+ //We need 100 * container size minimum space
+ conf.set("ozone.scm.container.size", "10MB");
+
ContainerSet containerSet = new ContainerSet();
volumeSet = new VolumeSet(datanodeUuid, conf);
StateContext context = new StateContext(
@@ -111,9 +110,12 @@ public class BenchMarkDatanodeDispatcher {
Map<ContainerProtos.ContainerType, Handler> handlers = Maps.newHashMap();
for (ContainerProtos.ContainerType containerType :
ContainerProtos.ContainerType.values()) {
- handlers.put(containerType,
- Handler.getHandlerForContainerType(
- containerType, conf, context, containerSet, volumeSet, metrics));
+ Handler handler = Handler.getHandlerForContainerType(
+ containerType, conf, "datanodeid",
+ containerSet, volumeSet, metrics,
+ c -> {});
+ handler.setScmID("scm");
+ handlers.put(containerType, handler);
}
dispatcher = new HddsDispatcher(conf, containerSet, volumeSet, handlers,
context, metrics, null);
@@ -217,11 +219,16 @@ public class BenchMarkDatanodeDispatcher {
private ContainerProtos.ChunkInfo getChunkInfo(
BlockID blockID, String chunkName) {
ContainerProtos.ChunkInfo.Builder builder =
- ContainerProtos.ChunkInfo.newBuilder()
+ ChunkInfo.newBuilder()
.setChunkName(
DigestUtils.md5Hex(chunkName)
+ "_stream_" + blockID.getContainerID() + "_block_"
+ blockID.getLocalID())
+ .setChecksumData(
+ ChecksumData.newBuilder()
+ .setBytesPerChecksum(4)
+ .setType(ChecksumType.CRC32)
+ .build())
.setOffset(0).setLen(data.size());
return builder.build();
}
@@ -245,7 +252,7 @@ public class BenchMarkDatanodeDispatcher {
private ContainerCommandRequestProto getGetBlockCommand(BlockID blockID) {
GetBlockRequestProto.Builder readBlockRequest =
GetBlockRequestProto.newBuilder()
- .setBlockID(blockID.getDatanodeBlockIDProtobuf());
+ .setBlockID(blockID.getDatanodeBlockIDProtobuf());
ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto
.newBuilder()
.setCmdType(ContainerProtos.Type.GetBlock)
@@ -274,7 +281,6 @@ public class BenchMarkDatanodeDispatcher {
bmdd.containerCount.getAndIncrement();
}
-
@Benchmark
public void writeChunk(BenchMarkDatanodeDispatcher bmdd) {
bmdd.dispatcher.dispatch(getWriteChunkCommand(
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/Genesis.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/Genesis.java
index 2de9b0f..4361666 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/Genesis.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genesis/Genesis.java
@@ -41,7 +41,8 @@ public final class Genesis {
// After adding benchmark in genesis package add the benchmark name in the
// description for this option.
- @Option(names = "-benchmark", split = ",", description =
+ @Option(names = {"-b", "-benchmark", "--benchmark"},
+ split = ",", description =
"Option used for specifying benchmarks to run.\n"
+ "Ex. ozone genesis -benchmark BenchMarkContainerStateMap,"
+ "BenchMarkOMKeyAllocation.\n"
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org