You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by su...@apache.org on 2026/04/08 05:27:10 UTC
(ozone) branch master updated: HDDS-14103. Create an option to suppress/unsuppress containers from report (#9719)
This is an automated email from the ASF dual-hosted git repository.
sumitagrawal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 49d6c0b6cbd HDDS-14103. Create an option to suppress/unsuppress containers from report (#9719)
49d6c0b6cbd is described below
commit 49d6c0b6cbd3b1ceb4fb49924f72e2495e77a1af
Author: Sarveksha Yeshavantha Raju <79...@users.noreply.github.com>
AuthorDate: Wed Apr 8 10:57:04 2026 +0530
HDDS-14103. Create an option to suppress/unsuppress containers from report (#9719)
---
.../org/apache/hadoop/hdds/cli/ItemsFromStdin.java | 20 +-
.../hadoop/hdds/scm/container/ContainerInfo.java | 37 +++
.../apache/hadoop/hdds/scm/client/ScmClient.java | 29 ++
.../protocol/StorageContainerLocationProtocol.java | 33 +++
...inerLocationProtocolClientSideTranslatorPB.java | 28 ++
.../src/main/proto/ScmAdminProtocol.proto | 14 +
.../interface-client/src/main/proto/hdds.proto | 1 +
.../hdds/scm/container/ContainerManager.java | 11 +
.../hdds/scm/container/ContainerManagerImpl.java | 15 +
.../hdds/scm/container/ContainerStateManager.java | 10 +
.../scm/container/ContainerStateManagerImpl.java | 18 ++
.../container/replication/ReplicationManager.java | 6 +
...inerLocationProtocolServerSideTranslatorPB.java | 20 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 79 +++++-
.../org/apache/hadoop/ozone/audit/SCMAction.java | 4 +-
.../hdds/scm/cli/ContainerOperationClient.java | 21 ++
.../scm/cli/container/ContainerIDParameters.java | 14 +-
.../hdds/scm/cli/container/ListSubcommand.java | 9 +-
.../hdds/scm/cli/container/ReportSubcommand.java | 57 ++++
.../TestContainerReportSuppressOptions.java | 310 +++++++++++++++++++++
20 files changed, 717 insertions(+), 19 deletions(-)
diff --git a/hadoop-hdds/cli-common/src/main/java/org/apache/hadoop/hdds/cli/ItemsFromStdin.java b/hadoop-hdds/cli-common/src/main/java/org/apache/hadoop/hdds/cli/ItemsFromStdin.java
index 79bad7ed732..8d096ab3573 100644
--- a/hadoop-hdds/cli-common/src/main/java/org/apache/hadoop/hdds/cli/ItemsFromStdin.java
+++ b/hadoop-hdds/cli-common/src/main/java/org/apache/hadoop/hdds/cli/ItemsFromStdin.java
@@ -33,9 +33,21 @@ public abstract class ItemsFromStdin implements Iterable<String> {
": one or more, separated by spaces. To read from stdin, specify '-' and supply one item per line.";
private List<String> items = new ArrayList<>();
+ private boolean readFromStdin = false;
protected void setItems(List<String> arguments) {
- items = readItemsFromStdinIfNeeded(arguments);
+ readFromStdin = arguments != null && !arguments.isEmpty() &&
+ "-".equals(arguments.iterator().next());
+
+ if (readFromStdin) {
+ items = readItemsFromStdin();
+ } else {
+ items = arguments == null ? new ArrayList<>() : arguments;
+ }
+ }
+
+ public boolean isReadFromStdin() {
+ return readFromStdin;
}
public List<String> getItems() {
@@ -52,11 +64,7 @@ public int size() {
return items.size();
}
- private static List<String> readItemsFromStdinIfNeeded(List<String> parameters) {
- if (parameters.isEmpty() || !"-".equals(parameters.iterator().next())) {
- return parameters;
- }
-
+ private static List<String> readItemsFromStdin() {
List<String> items = new ArrayList<>();
Scanner scanner = new Scanner(System.in, StandardCharsets.UTF_8.name());
while (scanner.hasNextLine()) {
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java
index c09ba8a2f1d..acefb5f0b38 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerInfo.java
@@ -20,6 +20,7 @@
import static java.lang.Math.max;
import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.time.Clock;
import java.time.Instant;
@@ -87,6 +88,7 @@ public final class ContainerInfo implements Comparable<ContainerInfo> {
private long sequenceId;
// Health state of the container (determined by ReplicationManager)
private ContainerHealthState healthState;
+ private boolean suppressed;
private ContainerInfo(Builder b) {
containerID = ContainerID.valueOf(b.containerID);
@@ -102,6 +104,7 @@ private ContainerInfo(Builder b) {
replicationConfig = b.replicationConfig;
clock = b.clock;
healthState = b.healthState != null ? b.healthState : ContainerHealthState.HEALTHY;
+ suppressed = b.suppressed;
}
public static Codec<ContainerInfo> getCodec() {
@@ -123,6 +126,10 @@ public static ContainerInfo fromProtobuf(HddsProtos.ContainerInfoProto info) {
.setReplicationConfig(config)
.setSequenceId(info.getSequenceId());
+ if (info.hasSuppressed()) {
+ builder.setSuppressed(info.getSuppressed());
+ }
+
if (info.hasPipelineID()) {
builder.setPipelineID(PipelineID.getFromProtobuf(info.getPipelineID()));
}
@@ -263,6 +270,26 @@ public void setHealthState(ContainerHealthState newHealthState) {
this.healthState = newHealthState;
}
+ /**
+ * Check if container is suppressed.
+ * Only included in JSON output when true.
+ *
+ * @return boolean
+ */
+ @JsonInclude(JsonInclude.Include.NON_DEFAULT)
+ public boolean isSuppressed() {
+ return suppressed;
+ }
+
+ /**
+ * Set the boolean for suppressed.
+ *
+ * @param suppressed checks if container is suppressed or not
+ */
+ public void setSuppressed(boolean suppressed) {
+ this.suppressed = suppressed;
+ }
+
@JsonIgnore
public HddsProtos.ContainerInfoProto getProtobuf() {
HddsProtos.ContainerInfoProto.Builder builder =
@@ -288,6 +315,10 @@ public HddsProtos.ContainerInfoProto getProtobuf() {
builder.setPipelineID(getPipelineID().getProtobuf());
}
+ if (suppressed) {
+ builder.setSuppressed(true);
+ }
+
return builder.build();
}
@@ -390,6 +421,7 @@ public static class Builder {
private PipelineID pipelineID;
private ReplicationConfig replicationConfig;
private ContainerHealthState healthState;
+ private boolean suppressed;
public Builder setPipelineID(PipelineID pipelineId) {
this.pipelineID = pipelineId;
@@ -447,6 +479,11 @@ public Builder setHealthState(ContainerHealthState healthState) {
return this;
}
+ public Builder setSuppressed(boolean suppressed) {
+ this.suppressed = suppressed;
+ return this;
+ }
+
/**
* Also resets {@code stateEnterTime}, so make sure to set clock first.
*/
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
index fac1467dd8f..cb4b8471a00 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java
@@ -145,6 +145,25 @@ ContainerListResult listContainer(long startContainerID, int count,
ReplicationConfig replicationConfig)
throws IOException;
+ /**
+ * Lists a range of containers and get their info.
+ *
+ * @param startContainerID start containerID.
+ * @param count count must be {@literal >} 0.
+ * @param state Container of this state will be returned.
+ * @param replicationConfig container replication Config.
+ * @param suppressed container to be suppressed/unsuppressed from report
+ * @return a list of containers capped by max count allowed
+ * in "ozone.scm.container.list.max.count" and total number of containers.
+ * @throws IOException
+ */
+ ContainerListResult listContainer(long startContainerID, int count,
+ HddsProtos.LifeCycleState state,
+ HddsProtos.ReplicationType replicationType,
+ ReplicationConfig replicationConfig,
+ Boolean suppressed)
+ throws IOException;
+
/**
* Read meta data from an existing container.
* @param containerID - ID of the container.
@@ -465,4 +484,14 @@ DecommissionScmResponseProto decommissionScm(
*/
void reconcileContainer(long containerID) throws IOException;
+ /**
+ * Suppress or unsuppress containers from reports.
+ * Suppressed containers are excluded from replication manager reports
+ * regardless of their health state.
+ *
+ * @param containerIds container IDs to suppress or unsuppress
+ * @param suppress true to suppress, false to unsuppress
+ * @throws IOException
+ */
+ List<Long> suppressContainers(List<Long> containerIds, boolean suppress) throws IOException;
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
index 32da41f8f17..98f8efa9ae3 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java
@@ -228,6 +228,30 @@ ContainerListResult listContainer(long startContainerID,
HddsProtos.ReplicationType replicationType,
ReplicationConfig replicationConfig) throws IOException;
+ /**
+ * Ask SCM for a list of containers with a range of container ID, state
+ * and replication config, and the limit of count.
+ * The containers are returned from startID (exclusive), and
+ * filtered by state and replication config. The returned list is limited to
+ * count entries.
+ *
+ * @param startContainerID start container ID.
+ * @param count count, if count {@literal <} 0, the max size is unlimited.(
+ * Usually the count will be replace with a very big
+ * value instead of being unlimited in case the db is very big)
+ * @param state Container with this state will be returned.
+ * @param replicationConfig Replication config for the containers
+ * @param suppressed container to be suppressed/unsuppressed from report
+ * @return a list of containers capped by max count allowed
+ * in "ozone.scm.container.list.max.count" and total number of containers.
+ * @throws IOException
+ */
+ ContainerListResult listContainer(long startContainerID,
+ int count, HddsProtos.LifeCycleState state,
+ HddsProtos.ReplicationType replicationType,
+ ReplicationConfig replicationConfig,
+ Boolean suppressed) throws IOException;
+
/**
* Deletes a container in SCM.
*
@@ -521,4 +545,13 @@ DecommissionScmResponseProto decommissionScm(
* @throws IOException On error
*/
void reconcileContainer(long containerID) throws IOException;
+
+ /**
+ * Suppress or unsuppress containers from reports.
+ *
+ * @param containerIds container IDs to suppress or unsuppress
+ * @param suppress true to suppress, false to unsuppress
+ * @throws IOException
+ */
+ List<Long> suppressContainers(List<Long> containerIds, boolean suppress) throws IOException;
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
index ca942465770..56e2ef6408f 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
@@ -127,6 +127,8 @@
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartReplicationManagerRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StopContainerBalancerRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StopReplicationManagerRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SuppressContainerRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SuppressContainerResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.Type;
import org.apache.hadoop.hdds.scm.DatanodeAdminError;
import org.apache.hadoop.hdds.scm.ScmInfo;
@@ -445,6 +447,16 @@ public ContainerListResult listContainer(long startContainerID, int count,
HddsProtos.ReplicationType replicationType,
ReplicationConfig replicationConfig)
throws IOException {
+ return listContainer(startContainerID, count, state, replicationType, replicationConfig, null);
+ }
+
+ @Override
+ public ContainerListResult listContainer(long startContainerID, int count,
+ HddsProtos.LifeCycleState state,
+ HddsProtos.ReplicationType replicationType,
+ ReplicationConfig replicationConfig,
+ Boolean suppressed)
+ throws IOException {
Preconditions.checkState(startContainerID >= 0,
"Container ID cannot be negative.");
Preconditions.checkState(count > 0,
@@ -454,6 +466,9 @@ public ContainerListResult listContainer(long startContainerID, int count,
builder.setStartContainerID(startContainerID);
builder.setCount(count);
builder.setTraceID(TracingUtil.exportCurrentSpan());
+ if (suppressed != null) {
+ builder.setSuppressed(suppressed);
+ }
if (state != null) {
builder.setState(state);
}
@@ -1312,6 +1327,19 @@ public void reconcileContainer(long containerID) throws IOException {
submitRequest(Type.ReconcileContainer, builder -> builder.setReconcileContainerRequest(request));
}
+ @Override
+ public List<Long> suppressContainers(List<Long> containerIds, boolean suppress)
+ throws IOException {
+ SuppressContainerRequestProto request = SuppressContainerRequestProto.newBuilder()
+ .addAllContainerIDs(containerIds)
+ .setSuppress(suppress)
+ .build();
+ SuppressContainerResponseProto response =
+ submitRequest(Type.SuppressContainer, builder -> builder.setSuppressContainerRequest(request))
+ .getSuppressContainerResponse();
+ return response.getFailedContainerIDsList();
+ }
+
/**
* Holder class to store the target SCM node ID for routing requests.
* This allows requests to be directed to specific SCM nodes in an HA cluster.
diff --git a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
index b6508ca9688..933bb4a0087 100644
--- a/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
+++ b/hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto
@@ -88,6 +88,7 @@ message ScmContainerLocationRequest {
optional ReconcileContainerRequestProto reconcileContainerRequest = 49;
optional GetDeletedBlocksTxnSummaryRequestProto getDeletedBlocksTxnSummaryRequest = 50;
optional SCMListContainerIDsRequestProto scmListContainerIDsRequest = 51;
+ optional SuppressContainerRequestProto suppressContainerRequest = 52;
}
message ScmContainerLocationResponse {
@@ -147,6 +148,7 @@ message ScmContainerLocationResponse {
optional ReconcileContainerResponseProto reconcileContainerResponse = 49;
optional GetDeletedBlocksTxnSummaryResponseProto getDeletedBlocksTxnSummaryResponse = 50;
optional SCMListContainerIDsResponseProto scmListContainerIDsResponse = 51;
+ optional SuppressContainerResponseProto suppressContainerResponse = 52;
enum Status {
OK = 1;
@@ -205,6 +207,7 @@ enum Type {
ReconcileContainer = 45;
GetDeletedBlocksTransactionSummary = 46;
ListContainerIDs = 47;
+ SuppressContainer = 48;
}
/**
@@ -313,6 +316,7 @@ message SCMListContainerRequestProto {
optional ReplicationFactor factor = 5;
optional ReplicationType type = 6;
optional ECReplicationConfig ecReplicationConfig = 7;
+ optional bool suppressed = 8;
}
message SCMListContainerResponseProto {
@@ -711,6 +715,16 @@ message ReconcileContainerRequestProto {
message ReconcileContainerResponseProto {
}
+message SuppressContainerRequestProto {
+ repeated int64 containerIDs = 1;
+ optional bool suppress = 2;
+}
+
+message SuppressContainerResponseProto {
+ // Container IDs that failed to suppress or unsuppress. Empty if all succeeded.
+ repeated int64 failedContainerIDs = 1;
+}
+
/**
* Protocol used from an HDFS node to StorageContainerManager. See the request
* and response messages for details of the RPC calls.
diff --git a/hadoop-hdds/interface-client/src/main/proto/hdds.proto b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
index 77aa0ff51d9..0904802b9ae 100644
--- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto
+++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto
@@ -271,6 +271,7 @@ message ContainerInfoProto {
optional ReplicationFactor replicationFactor = 10;
required ReplicationType replicationType = 11;
optional ECReplicationConfig ecReplicationConfig = 12;
+ optional bool suppressed = 13;
}
message ContainerWithPipeline {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java
index 3c5706cc0fb..691a1965ab8 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java
@@ -24,6 +24,7 @@
import java.util.Map;
import java.util.Set;
import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ContainerInfoProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleEvent;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
@@ -238,4 +239,14 @@ void deleteContainer(ContainerID containerID)
* @return containerStateManger
*/
ContainerStateManager getContainerStateManager();
+
+ /**
+ * Update container info in the container manager.
+ * This is used for updating container metadata like ackMissing flag.
+ *
+ * @param containerInfo Updated container info proto
+ * @throws IOException
+ */
+ void updateContainerInfo(ContainerID containerID, ContainerInfoProto containerInfo)
+ throws IOException;
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java
index 432c9890e98..7780c083946 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java
@@ -302,6 +302,21 @@ public void updateContainerState(final ContainerID cid,
}
}
+ @Override
+ public void updateContainerInfo(final ContainerID cid, ContainerInfoProto containerInfo)
+ throws IOException {
+ lock.lock();
+ try {
+ if (containerExist(cid)) {
+ containerStateManager.updateContainerInfo(containerInfo);
+ } else {
+ throw new ContainerNotFoundException(cid);
+ }
+ } finally {
+ lock.unlock();
+ }
+ }
+
@Override
public void transitionDeletingOrDeletedToTargetState(ContainerID containerID, LifeCycleState targetState)
throws IOException {
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
index a853429faf2..9e31e21822b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManager.java
@@ -226,4 +226,14 @@ void reinitialize(Table<ContainerID, ContainerInfo> containerStore)
default RequestType getType() {
return RequestType.CONTAINER;
}
+
+ /**
+ * Update container info.
+ *
+ * @param containerInfo Updated container info proto
+ * @throws IOException
+ */
+ @Replicate
+ void updateContainerInfo(HddsProtos.ContainerInfoProto containerInfo)
+ throws IOException;
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManagerImpl.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManagerImpl.java
index 7a959f25902..7b50dfceb7b 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerStateManagerImpl.java
@@ -563,6 +563,24 @@ public void reinitialize(
}
}
+ @Override
+ public void updateContainerInfo(HddsProtos.ContainerInfoProto updatedInfoProto)
+ throws IOException {
+ ContainerInfo updatedInfo = ContainerInfo.fromProtobuf(updatedInfoProto);
+ ContainerID containerID = updatedInfo.containerID();
+
+ try (AutoCloseableLock ignored = writeLock(containerID)) {
+ final ContainerInfo currentInfo = containers.getContainerInfo(containerID);
+ if (currentInfo == null) {
+ throw new ContainerNotFoundException(containerID);
+ }
+ // Update suppressed flag
+ currentInfo.setSuppressed(updatedInfo.isSuppressed());
+ transactionBuffer.addToBuffer(containerStore, containerID, currentInfo);
+ LOG.debug("Updated container info for container: {}, suppressed={}", containerID, currentInfo.isSuppressed());
+ }
+ }
+
private AutoCloseableLock readLock() {
return AutoCloseableLock.acquire(lock.readLock());
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java
index 7096c18e6c5..8cd8444d1d2 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManager.java
@@ -867,6 +867,12 @@ protected boolean processContainer(ContainerInfo containerInfo,
ReplicationQueue repQueue, ReplicationManagerReport report,
boolean readOnly) throws ContainerNotFoundException {
synchronized (containerInfo) {
+ // Filter out suppressed containers early
+ if (containerInfo.isSuppressed()) {
+ LOG.debug("Skipping suppressed container: {}", containerInfo.getContainerID());
+ return false;
+ }
+
// Reset health state to HEALTHY before processing this container
report.resetContainerHealthState();
final boolean isEC = isEC(containerInfo.getReplicationConfig());
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
index dd18ad68d13..73bf92e9cd5 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java
@@ -137,6 +137,8 @@
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StopContainerBalancerResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StopReplicationManagerRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StopReplicationManagerResponseProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SuppressContainerRequestProto;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SuppressContainerResponseProto;
import org.apache.hadoop.hdds.scm.DatanodeAdminError;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.container.ContainerID;
@@ -760,6 +762,12 @@ public ScmContainerLocationResponse processRequest(
.setStatus(Status.OK)
.setScmListContainerIDsResponse(listContainerIDs(request.getScmListContainerIDsRequest()))
.build();
+ case SuppressContainer:
+ return ScmContainerLocationResponse.newBuilder()
+ .setCmdType(request.getCmdType())
+ .setStatus(Status.OK)
+ .setSuppressContainerResponse(suppressContainer(request.getSuppressContainerRequest()))
+ .build();
default:
throw new IllegalArgumentException(
"Unknown command type: " + request.getCmdType());
@@ -889,6 +897,9 @@ public SCMListContainerResponseProto listContainer(
} else if (request.hasFactor()) {
factor = request.getFactor();
}
+ // Filter by suppressed: true (suppressed only), false (unsuppressed only) or null (display all).
+ Boolean suppressed = request.hasSuppressed() ? request.getSuppressed() : null;
+
ContainerListResult containerListAndTotalCount;
if (factor != null) {
// Call from a legacy client
@@ -896,7 +907,7 @@ public SCMListContainerResponseProto listContainer(
impl.listContainer(startContainerID, count, state, factor);
} else {
containerListAndTotalCount =
- impl.listContainer(startContainerID, count, state, replicationType, repConfig);
+ impl.listContainer(startContainerID, count, state, replicationType, repConfig, suppressed);
}
SCMListContainerResponseProto.Builder builder =
SCMListContainerResponseProto.newBuilder();
@@ -1434,4 +1445,11 @@ public SCMListContainerIDsResponseProto listContainerIDs(
return builder.build();
}
+
+ public SuppressContainerResponseProto suppressContainer(SuppressContainerRequestProto request) throws IOException {
+ List<Long> failedContainerIDs = impl.suppressContainers(request.getContainerIDsList(), request.getSuppress());
+ return SuppressContainerResponseProto.newBuilder()
+ .addAllFailedContainerIDs(failedContainerIDs)
+ .build();
+ }
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
index 0725cb7b71b..8f0c79728ed 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java
@@ -451,7 +451,7 @@ private boolean hasRequiredReplicas(ContainerInfo contInfo) {
@Override
public ContainerListResult listContainer(long startContainerID,
int count) throws IOException {
- return listContainer(startContainerID, count, null, null, null);
+ return listContainer(startContainerID, count, null, null, null, null);
}
/**
@@ -468,7 +468,7 @@ public ContainerListResult listContainer(long startContainerID,
@Override
public ContainerListResult listContainer(long startContainerID,
int count, HddsProtos.LifeCycleState state) throws IOException {
- return listContainer(startContainerID, count, state, null, null);
+ return listContainer(startContainerID, count, state, null, null, null);
}
/**
@@ -487,20 +487,28 @@ public ContainerListResult listContainer(long startContainerID,
public ContainerListResult listContainer(long startContainerID,
int count, HddsProtos.LifeCycleState state,
HddsProtos.ReplicationFactor factor) throws IOException {
- return listContainerInternal(startContainerID, count, state, factor, null, null);
+ return listContainerInternal(startContainerID, count, state, factor, null, null, null);
}
private ContainerListResult listContainerInternal(long startContainerID, int count,
HddsProtos.LifeCycleState state,
HddsProtos.ReplicationFactor factor,
HddsProtos.ReplicationType replicationType,
- ReplicationConfig repConfig) throws IOException {
+ ReplicationConfig repConfig,
+ Boolean suppressed) throws IOException {
boolean auditSuccess = true;
- Map<String, String> auditMap = buildAuditMap(startContainerID, count, state, factor, replicationType, repConfig);
+ Map<String, String> auditMap = buildAuditMap(startContainerID, count, state, factor,
+ replicationType, repConfig, suppressed);
try {
Stream<ContainerInfo> containerStream =
buildContainerStream(factor, replicationType, repConfig, getBaseContainerStream(state));
+
+ // Filter by suppressed flag only if explicitly specified
+ if (suppressed != null) {
+ containerStream = containerStream.filter(info -> info.isSuppressed() == suppressed);
+ }
+
List<ContainerInfo> containerInfos =
containerStream.filter(info -> info.containerID().getId() >= startContainerID)
.sorted().collect(Collectors.toList());
@@ -552,7 +560,8 @@ private Map<String, String> buildAuditMap(long startContainerID, int count,
HddsProtos.LifeCycleState state,
HddsProtos.ReplicationFactor factor,
HddsProtos.ReplicationType replicationType,
- ReplicationConfig repConfig) {
+ ReplicationConfig repConfig,
+ Boolean suppressed) {
Map<String, String> auditMap = new HashMap<>();
auditMap.put("startContainerID", String.valueOf(startContainerID));
auditMap.put("count", String.valueOf(count));
@@ -568,6 +577,9 @@ private Map<String, String> buildAuditMap(long startContainerID, int count,
if (repConfig != null) {
auditMap.put("replicationConfig", repConfig.toString());
}
+ if (suppressed != null) {
+ auditMap.put("suppressed", suppressed.toString());
+ }
return auditMap;
}
@@ -588,7 +600,28 @@ public ContainerListResult listContainer(long startContainerID,
int count, HddsProtos.LifeCycleState state,
HddsProtos.ReplicationType replicationType,
ReplicationConfig repConfig) throws IOException {
- return listContainerInternal(startContainerID, count, state, null, replicationType, repConfig);
+ return listContainerInternal(startContainerID, count, state, null, replicationType, repConfig, null);
+ }
+
+ /**
+ * Lists a range of containers and get their info.
+ *
+ * @param startContainerID start containerID.
+ * @param count count must be {@literal >} 0.
+ * @param state Container with this state will be returned.
+ * @param repConfig Replication Config for the container.
+ * @param suppressed container to be suppressed/unsuppressed from report
+ * @return a list of containers capped by max count allowed
+ * in "ozone.scm.container.list.max.count" and total number of containers.
+ * @throws IOException
+ */
+ @Override
+ public ContainerListResult listContainer(long startContainerID,
+ int count, HddsProtos.LifeCycleState state,
+ HddsProtos.ReplicationType replicationType,
+ ReplicationConfig repConfig,
+ Boolean suppressed) throws IOException {
+ return listContainerInternal(startContainerID, count, state, null, replicationType, repConfig, suppressed);
}
@Override
@@ -1695,4 +1728,36 @@ public void reconcileContainer(long longContainerID) throws IOException {
throw ex;
}
}
+
+ @Override
+ public List<Long> suppressContainers(List<Long> containerIds, boolean suppress) throws IOException {
+ getScm().checkAdminAccess(getRemoteUser(), false);
+ SCMAction action = suppress ? SCMAction.SUPPRESS_CONTAINER : SCMAction.UNSUPPRESS_CONTAINER;
+ List<Long> failedContainerIDs = new ArrayList<>();
+ for (long containerId : containerIds) {
+ try {
+ persistContainerSuppression(containerId, suppress, action);
+ } catch (IOException ex) {
+ failedContainerIDs.add(containerId);
+ }
+ }
+ return failedContainerIDs;
+ }
+
+ private void persistContainerSuppression(long longContainerID, boolean suppress, SCMAction action)
+ throws IOException {
+ ContainerID containerID = ContainerID.valueOf(longContainerID);
+ final Map<String, String> auditMap = new HashMap<>();
+ auditMap.put("containerID", containerID.toString());
+ auditMap.put("suppress", String.valueOf(suppress));
+ try {
+ ContainerInfo containerInfo = scm.getContainerManager().getContainer(containerID);
+ containerInfo.setSuppressed(suppress);
+ scm.getContainerManager().updateContainerInfo(containerID, containerInfo.getProtobuf());
+ AUDIT.logWriteSuccess(buildAuditMessageForSuccess(action, auditMap));
+ } catch (IOException ex) {
+ AUDIT.logWriteFailure(buildAuditMessageForFailure(action, auditMap, ex));
+ throw ex;
+ }
+ }
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
index b7acb40d7ac..61d536e599f 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/ozone/audit/SCMAction.java
@@ -69,7 +69,9 @@ public enum SCMAction implements AuditAction {
GET_PIPELINE,
RECONCILE_CONTAINER,
GET_DELETED_BLOCK_SUMMARY,
- LIST_CONTAINER_IDS;
+ LIST_CONTAINER_IDS,
+ SUPPRESS_CONTAINER,
+ UNSUPPRESS_CONTAINER;
@Override
public String getAction() {
diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
index 18950053e44..c1973891d0a 100644
--- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
+++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerOperationClient.java
@@ -389,6 +389,22 @@ public ContainerListResult listContainer(long startContainerID,
startContainerID, count, state, repType, replicationConfig);
}
+ @Override
+ public ContainerListResult listContainer(long startContainerID,
+ int count, HddsProtos.LifeCycleState state,
+ HddsProtos.ReplicationType repType,
+ ReplicationConfig replicationConfig,
+ Boolean suppressed) throws IOException {
+ if (count > maxCountOfContainerList) {
+ LOG.warn("Attempting to list {} containers. However, this exceeds" +
+ " the cluster's current limit of {}. The results will be capped at the" +
+ " maximum allowed count.", count, maxCountOfContainerList);
+ count = maxCountOfContainerList;
+ }
+ return storageContainerLocationClient.listContainer(
+ startContainerID, count, state, repType, replicationConfig, suppressed);
+ }
+
@Override
public ContainerDataProto readContainer(long containerID,
Pipeline pipeline) throws IOException {
@@ -614,4 +630,9 @@ public String getMetrics(String query) throws IOException {
public void reconcileContainer(long id) throws IOException {
storageContainerLocationClient.reconcileContainer(id);
}
+
+ @Override
+ public List<Long> suppressContainers(List<Long> containerIds, boolean suppress) throws IOException {
+ return storageContainerLocationClient.suppressContainers(containerIds, suppress);
+ }
}
diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerIDParameters.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerIDParameters.java
index 36e615a5829..a201cc2ab9f 100644
--- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerIDParameters.java
+++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerIDParameters.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdds.scm.cli.container;
import java.util.ArrayList;
+import java.util.LinkedHashSet;
import java.util.List;
import org.apache.hadoop.hdds.cli.ItemsFromStdin;
import picocli.CommandLine;
@@ -30,13 +31,22 @@ public class ContainerIDParameters extends ItemsFromStdin {
private CommandLine.Model.CommandSpec spec;
@CommandLine.Parameters(description = "Container IDs" + FORMAT_DESCRIPTION,
- arity = "1..*",
+ arity = "0..*",
paramLabel = "<container ID>")
public void setContainerIDs(List<String> arguments) {
setItems(arguments);
}
public List<Long> getValidatedIDs() {
+ return getValidatedIDs(true);
+ }
+
+ public List<Long> getValidatedIDs(boolean required) {
+ if (required && size() == 0 && !isReadFromStdin()) {
+ throw new CommandLine.MissingParameterException(spec.commandLine(),
+ spec.commandLine().getCommandSpec().args(),
+ "Missing required parameter: '<container ID>'");
+ }
List<Long> containerIDs = new ArrayList<>(size());
List<String> invalidIDs = new ArrayList<>();
@@ -62,6 +72,6 @@ public List<Long> getValidatedIDs() {
throw new CommandLine.ParameterException(spec.commandLine(),
"Container IDs must be positive integers. Invalid container IDs: " + String.join(" ", invalidIDs));
}
- return containerIDs;
+ return new ArrayList<>(new LinkedHashSet<>(containerIDs));
}
}
diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java
index 4bc9af843a0..0cc4b8bdee1 100644
--- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java
+++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java
@@ -86,6 +86,11 @@ public class ListSubcommand extends ScmSubcommand {
"rs-6-3-1024k for EC)")
private String replication;
+ @Option(names = {"--suppressed"},
+ description = "Filter by suppression status. Set to 'true' to list only suppressed containers " +
+ "or 'false' to list only those that are not suppressed.")
+ private Boolean suppressed;
+
private static final ObjectWriter WRITER;
static {
@@ -129,7 +134,7 @@ public void execute(ScmClient scmClient) throws IOException {
}
ContainerListResult containerListResult =
- scmClient.listContainer(startId, count, state, type, repConfig);
+ scmClient.listContainer(startId, count, state, type, repConfig, suppressed);
writeContainers(sequenceWriter, containerListResult.getContainerInfoList());
@@ -169,7 +174,7 @@ private void listAllContainers(ScmClient scmClient, SequenceWriter writer,
do {
ContainerListResult result =
- scmClient.listContainer(currentStartId, batchSize, state, type, repConfig);
+ scmClient.listContainer(currentStartId, batchSize, state, type, repConfig, suppressed);
fetchedCount = result.getContainerInfoList().size();
writeContainers(writer, result.getContainerInfoList());
diff --git a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ReportSubcommand.java b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ReportSubcommand.java
index b5dc960d8c7..c3e809910f1 100644
--- a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ReportSubcommand.java
+++ b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ReportSubcommand.java
@@ -44,13 +44,70 @@ public class ReportSubcommand extends ScmSubcommand {
@CommandLine.Spec
private CommandLine.Model.CommandSpec spec;
+ @CommandLine.ArgGroup(exclusive = true, multiplicity = "0..1")
+ private SuppressOptions suppressOptions;
+
+ @CommandLine.Mixin
+ private ContainerIDParameters containerList;
+
@CommandLine.Option(names = { "--json" },
defaultValue = "false",
description = "Format output as JSON")
private boolean json;
+ static class SuppressOptions {
+ @CommandLine.Option(names = {"--suppress"},
+ description = "Suppress container(s) from future reports")
+ private boolean suppress;
+
+ @CommandLine.Option(names = {"--unsuppress"},
+ description = "Unsuppress container(s) to include in future reports")
+ private boolean unsuppress;
+ }
+
@Override
public void execute(ScmClient scmClient) throws IOException {
+ if (suppressOptions != null) {
+ handleSuppressUnsuppress(scmClient);
+ return;
+ }
+
+ if (containerList != null && containerList.size() > 0) {
+ throw new CommandLine.ParameterException(spec.commandLine(),
+ "Container IDs are only valid with --suppress or --unsuppress. " +
+ "To print the summary report, do not pass any container ID.");
+ }
+
+ printReport(scmClient);
+ }
+
+ private void handleSuppressUnsuppress(ScmClient scmClient) throws IOException {
+ boolean suppress = suppressOptions.suppress;
+ List<Long> containerIDs = containerList.getValidatedIDs();
+ List<Long> failedContainerIDs = scmClient.suppressContainers(containerIDs, suppress);
+
+ int failures = 0;
+ for (long id : containerIDs) {
+ if (failedContainerIDs.contains(id)) {
+ err().println("Failed to " + (suppress ? "suppress" : "unsuppress") + " container " + id + ".");
+ failures++;
+ } else {
+ out().println((suppress ? "Suppressed" : "Unsuppressed") + " container: " + id);
+ }
+ }
+
+ int numOfSuccess = containerIDs.size() - failures;
+ if (numOfSuccess > 0) {
+ out().println("\n" + (suppress ? "Suppressed " : "Unsuppressed ") + numOfSuccess + " container(s) successfully.");
+ out().println("Container report will be updated after the next Replication Manager cycle.");
+ }
+
+ if (failures > 0) {
+ throw new IOException("Failed to " + (suppress ? "suppress " : "unsuppress ") + failures + " container(s).");
+ }
+ }
+
+ private void printReport(ScmClient scmClient) throws IOException {
ReplicationManagerReport report = scmClient.getReplicationManagerReport();
if (report.getReportTimeStamp() == 0) {
System.err.println("The Container Report is not available until Replication Manager completes" +
diff --git a/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestContainerReportSuppressOptions.java b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestContainerReportSuppressOptions.java
new file mode 100644
index 00000000000..2e4ef55b8ec
--- /dev/null
+++ b/hadoop-ozone/cli-admin/src/test/java/org/apache/hadoop/hdds/scm/cli/container/TestContainerReportSuppressOptions.java
@@ -0,0 +1,310 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm.cli.container;
+
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState.CLOSED;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.hadoop.hdds.client.RatisReplicationConfig;
+import org.apache.hadoop.hdds.protocol.DatanodeID;
+import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
+import org.apache.hadoop.hdds.scm.client.ScmClient;
+import org.apache.hadoop.hdds.scm.container.ContainerHealthState;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerListResult;
+import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+import picocli.CommandLine;
+
+/**
+ * Tests the suppress/unsuppress options in container report.
+ */
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+public class TestContainerReportSuppressOptions {
+
+ private ScmClient scmClient;
+ private List<ContainerInfo> containers;
+ private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+ private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
+ private final PrintStream originalOut = System.out;
+ private final PrintStream originalErr = System.err;
+ private static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name();
+
+ @BeforeEach
+ public void setup() throws IOException {
+ scmClient = mock(ScmClient.class);
+ MockDatanodeDetails.createDatanodeDetails(DatanodeID.randomID());
+ containers = new ArrayList<>();
+
+ containers.add(createEmptyContainer(1L)); // EMPTY container (0 keys, no replicas)
+ containers.add(createMissingContainer(2L)); // MISSING container (has keys, no replicas)
+
+ // Mock RM report
+ when(scmClient.getReplicationManagerReport()).thenAnswer(inv -> createMockReport());
+
+ // Mock listContainer
+ when(scmClient.listContainer(anyLong(), anyInt(), eq(null), eq(null), eq(null), eq(true)))
+ .thenAnswer(inv -> listSuppressedContainers());
+ when(scmClient.listContainer(anyLong(), anyInt(), eq(null), eq(null), eq(null), eq(false)))
+ .thenAnswer(inv -> listNonSuppressedContainers());
+
+ // Mock suppress/unsuppress
+ doReturn(Collections.emptyList()).when(scmClient).suppressContainers(anyList(), eq(true));
+ doReturn(Collections.emptyList()).when(scmClient).suppressContainers(anyList(), eq(false));
+
+ System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING));
+ System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING));
+ }
+
+ @AfterEach
+ public void tearDown() {
+ System.setOut(originalOut);
+ System.setErr(originalErr);
+ }
+
+ /**
+ * Test container report shows empty and missing containers.
+ */
+ @Test
+ @Order(1)
+ public void testReportShowsEmptyAndMissingContainers() throws IOException {
+ ReportSubcommand reportCmd = new ReportSubcommand();
+ CommandLine c = new CommandLine(reportCmd);
+ c.parseArgs();
+ reportCmd.execute(scmClient);
+
+ String output = outContent.toString(DEFAULT_ENCODING);
+ assertTrue(output.contains("EMPTY: 1"));
+ assertTrue(output.contains("MISSING: 1"));
+ }
+
+ /**
+ * Test suppress missing container and check report.
+ */
+ @Test
+ @Order(2)
+ public void testSuppressMissingContainer() throws IOException {
+ outContent.reset();
+ ReportSubcommand reportCmd = new ReportSubcommand();
+ CommandLine c = new CommandLine(reportCmd);
+ c.parseArgs("--suppress", "2");
+ reportCmd.execute(scmClient);
+
+ String output = outContent.toString(DEFAULT_ENCODING);
+ assertTrue(output.contains("Suppressed container: 2"));
+
+ containers.get(1).setSuppressed(true);
+
+ outContent.reset();
+ reportCmd = new ReportSubcommand();
+ c = new CommandLine(reportCmd);
+ c.parseArgs();
+ reportCmd.execute(scmClient);
+
+ output = outContent.toString(DEFAULT_ENCODING);
+ assertTrue(output.contains("EMPTY: 1"));
+ assertTrue(output.contains("MISSING: 0"));
+ }
+
+ /**
+ * Test list suppressed containers from container list command.
+ */
+ @Test
+ @Order(3)
+ public void testListSuppressedContainers() throws IOException {
+ containers.get(1).setSuppressed(true);
+
+ ListSubcommand listCmd = new ListSubcommand();
+ CommandLine c = new CommandLine(listCmd);
+ c.parseArgs("--suppressed=true");
+
+ outContent.reset();
+ listCmd.execute(scmClient);
+
+ String output = outContent.toString(DEFAULT_ENCODING);
+ assertTrue(output.contains("\"containerID\" : 2"));
+ assertFalse(output.contains("\"containerID\" : 1"));
+ }
+
+ /**
+ * Test unsuppress missing container and check report.
+ */
+ @Test
+ @Order(4)
+ public void testUnsuppressContainer() throws IOException {
+ containers.get(1).setSuppressed(true);
+
+ outContent.reset();
+ ReportSubcommand reportCmd = new ReportSubcommand();
+ CommandLine c = new CommandLine(reportCmd);
+ c.parseArgs("--unsuppress", "2");
+ reportCmd.execute(scmClient);
+
+ String output = outContent.toString(DEFAULT_ENCODING);
+ assertTrue(output.contains("Unsuppressed container: 2"));
+
+ containers.get(1).setSuppressed(false);
+
+ outContent.reset();
+ reportCmd = new ReportSubcommand();
+ c = new CommandLine(reportCmd);
+ c.parseArgs();
+ reportCmd.execute(scmClient);
+
+ output = outContent.toString(DEFAULT_ENCODING);
+ assertTrue(output.contains("MISSING: 1"));
+ }
+
+ /**
+ * Test list unsuppressed containers from container list command.
+ */
+ @Test
+ @Order(5)
+ public void testListNonSuppressedContainers() throws IOException {
+ containers.get(1).setSuppressed(false);
+
+ // List only non-suppressed containers
+ ListSubcommand listCmd = new ListSubcommand();
+ CommandLine c = new CommandLine(listCmd);
+ c.parseArgs("--suppressed=false");
+
+ outContent.reset();
+ listCmd.execute(scmClient);
+
+ String output = outContent.toString(DEFAULT_ENCODING);
+ assertTrue(output.contains("\"containerID\" : 1"));
+ assertTrue(output.contains("\"containerID\" : 2"));
+ assertFalse(output.contains("\"suppressed\" : true"));
+ }
+
+ /**
+ * Test container report throws exception when container IDs are passed without --suppress/--unsuppress optionss.
+ */
+ @Test
+ @Order(6)
+ public void testContainerIDWithoutSuppressOptions() {
+ ReportSubcommand reportCmd = new ReportSubcommand();
+ CommandLine c = new CommandLine(reportCmd);
+ c.parseArgs("1", "2");
+ CommandLine.ParameterException ex =
+ assertThrows(CommandLine.ParameterException.class, () -> reportCmd.execute(scmClient));
+ assertThat(ex.getMessage()).contains("Container IDs are only valid with --suppress or --unsuppress");
+ }
+
+ /**
+ * Create mock RM report based on current container states.
+ */
+ private ReplicationManagerReport createMockReport() {
+ ReplicationManagerReport report = new ReplicationManagerReport(100);
+
+ for (ContainerInfo container : containers) {
+ if (container.isSuppressed()) {
+ // Suppressed containers are filtered from the report
+ continue;
+ }
+
+ // Determine health state based on container properties
+ ContainerHealthState healthState;
+ if (container.getNumberOfKeys() == 0) {
+ healthState = ContainerHealthState.EMPTY;
+ report.incrementAndSample(healthState, container);
+ } else {
+ healthState = ContainerHealthState.MISSING;
+ report.incrementAndSample(healthState, container);
+ }
+
+ // Also increment lifecycle state
+ report.increment(container.getState());
+ }
+
+ report.setComplete();
+ return report;
+ }
+
+ /**
+ * Create an EMPTY container (0 keys, no replicas).
+ */
+ private ContainerInfo createEmptyContainer(long containerID) {
+ return new ContainerInfo.Builder()
+ .setContainerID(containerID)
+ .setReplicationConfig(RatisReplicationConfig.getInstance(ONE))
+ .setState(CLOSED)
+ .setOwner("TestOwner")
+ .setNumberOfKeys(0) // Empty - 0 keys
+ .setPipelineID(PipelineID.randomId())
+ .build();
+ }
+
+ /**
+ * Create a MISSING container (has keys but no replicas).
+ */
+ private ContainerInfo createMissingContainer(long containerID) {
+ return new ContainerInfo.Builder()
+ .setContainerID(containerID)
+ .setReplicationConfig(RatisReplicationConfig.getInstance(ONE))
+ .setState(CLOSED)
+ .setOwner("TestOwner")
+ .setNumberOfKeys(100) // Has keys
+ .setPipelineID(PipelineID.randomId())
+ .build();
+ }
+
+ private ContainerListResult listSuppressedContainers() {
+ List<ContainerInfo> suppressed = new ArrayList<>();
+ for (ContainerInfo container : containers) {
+ if (container.isSuppressed()) {
+ suppressed.add(container);
+ }
+ }
+ return new ContainerListResult(suppressed, suppressed.size());
+ }
+
+ private ContainerListResult listNonSuppressedContainers() {
+ List<ContainerInfo> nonSuppressed = new ArrayList<>();
+ for (ContainerInfo container : containers) {
+ if (!container.isSuppressed()) {
+ nonSuppressed.add(container);
+ }
+ }
+ return new ContainerListResult(nonSuppressed, nonSuppressed.size());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org