You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ozone.apache.org by GitBox <gi...@apache.org> on 2022/06/23 18:33:57 UTC
[GitHub] [ozone] umamaheswararao opened a new pull request, #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
umamaheswararao opened a new pull request, #3542:
URL: https://github.com/apache/ozone/pull/3542
## What changes were proposed in this pull request?
Added a class to handle the under replication processing and forming the reconstruction command.
## What is the link to the Apache JIRA
https://issues.apache.org/jira/browse/HDDS-6894
## How was this patch tested?
Added the tests.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] JacksonYao287 commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
JacksonYao287 commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r907106862
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationCommandCreator.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationCommandCreator
+ implements ReplicationCommandCreator {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationCommandCreator.class);
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationCommandCreator(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas,
+ pendingOps.getPendingOps(container.containerID()),
+ remainingMaintenanceRedundancy);
+ ContainerHealthResult containerHealthResult =
+ new ECContainerHealthCheck().checkHealth(container, replicaCount);
Review Comment:
NIT: can we not create a ECContainerHealthCheck instance here? just make ECContainerHealthCheck as a member variable of ECUnderReplicationCommandCreator
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECContainerHealthCheck.java:
##########
@@ -89,19 +96,33 @@ public ContainerHealthResult checkHealth(ContainerInfo container,
return new ContainerHealthResult.HealthyResult(container);
}
- private ECContainerReplicaCount getReplicaCountWithPending(
+ private static ECContainerReplicaCount getReplicaCountWithPending(
ContainerInfo container, Set<ContainerReplica> replicas,
List<Pair<Integer, DatanodeDetails>> replicasPendingAdd,
List<Pair<Integer, DatanodeDetails>> replicasPendingDelete,
int remainingRedundancyForMaintenance) {
- List<Integer> indexesPendingAdd = replicasPendingAdd.stream()
- .map(i -> i.getLeft()).collect(Collectors.toList());
- List<Integer> indexesPendingDelete = replicasPendingDelete.stream()
- .map(i -> i.getLeft()).collect(Collectors.toList());
+ ContainerReplicaPendingOps pendingOps =
Review Comment:
i think we can collect all the `pendingAdd` and `pendinDelete` ops to a list , and then pass this list to the creator of `ECContainerReplicaCount`. it seems unnecessary to build a `ContainerReplicaPendingOps` and putting all the pending ops into it , and then extract all the pending ops from it.
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationCommandCreator.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationCommandCreator
+ implements ReplicationCommandCreator {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationCommandCreator.class);
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationCommandCreator(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas,
+ pendingOps.getPendingOps(container.containerID()),
+ remainingMaintenanceRedundancy);
+ ContainerHealthResult containerHealthResult =
+ new ECContainerHealthCheck().checkHealth(container, replicaCount);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+
+ if (containerHealthResult.getHealthState()
+ != ContainerHealthResult.HealthState.UNDER_REPLICATED) {
+ LOG.info(
+ "The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(id);
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
Review Comment:
if an EC container is under-replicated, it may
1 miss some indexes of replica
2 all the indexes of replica is available, but they does not satisfy the placement policy
we should take the seconde scenario into account later
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationCommandCreator.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationCommandCreator
Review Comment:
NIT, can we rename this to ECReconstructionCommandCreator
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationCommandCreator.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationCommandCreator
+ implements ReplicationCommandCreator {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationCommandCreator.class);
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationCommandCreator(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas,
+ pendingOps.getPendingOps(container.containerID()),
+ remainingMaintenanceRedundancy);
+ ContainerHealthResult containerHealthResult =
+ new ECContainerHealthCheck().checkHealth(container, replicaCount);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+
+ if (containerHealthResult.getHealthState()
Review Comment:
in fact , the the health state of an EC container may be both under and over replicated. for example, EC-3-2, the health stat is 1,2,3,4,5, but we might have 1,1,2,3,4. so , think we should have more enum items for `HealthState`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908371189
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ (ContainerHealthResult.UnderReplicatedHealthResult)
+ ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (containerHealthResult
+ .isSufficientlyReplicatedAfterPending() || containerHealthResult
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ } else if (result instanceof ContainerHealthResult
Review Comment:
What do you mean by single pass? For replication we iterate and for reconstruction all sources we need to pass to create command. Could you eloborate?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r907891579
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationCommandCreator.java:
##########
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * This interface to create respective commands after processing the replicas
+ * with pending ops and health check results.
+ */
+public interface ReplicationCommandCreator {
Review Comment:
I was trying to be more expressive here. however the classes are doing more than command creation. So, I changed them to Handler classes. ( To be honest I was little hesitant to keep handler in name as we have many all around :-) )
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908304436
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
Review Comment:
```
if (containerHealthResult
.isSufficientlyReplicatedAfterPending() || containerHealthResult
.getHealthState() != ContainerHealthResult.HealthState
.UNDER_REPLICATED) {
LOG.info("The container {} with replicas {} is sufficiently replicated",
container.getContainerID(), replicaCount.getReplicas());
```
Thanks for thinking that. Isn't this already doing?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r907884050
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationCommandCreator.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationCommandCreator
+ implements ReplicationCommandCreator {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationCommandCreator.class);
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationCommandCreator(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas,
+ pendingOps.getPendingOps(container.containerID()),
+ remainingMaintenanceRedundancy);
+ ContainerHealthResult containerHealthResult =
+ new ECContainerHealthCheck().checkHealth(container, replicaCount);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+
+ if (containerHealthResult.getHealthState()
+ != ContainerHealthResult.HealthState.UNDER_REPLICATED) {
+ LOG.info(
+ "The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(id);
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
Review Comment:
We are already passing the flag includePendingAdd as true. Which will remove them right.
```
// Now we have a list of missing. Remove any pending add as they should
// eventually recover.
if (includePendingAdd) {
for (Integer i : pendingAdd) {
missing.remove(i);
}
}
```
@JacksonYao287 We will consider the placement policy after the basic reconstruction flows done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r910116321
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - Set of available container replicas.
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand<?>> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return emptyMap();
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ final ContainerID id = container.containerID();
+ final Map<DatanodeDetails, SCMCommand<?>> commands = new HashMap<>();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOps) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
Review Comment:
so, looking at the statement:
int healthyDatanodes =
nodeManager.getNodeCount(NodeStatus.inServiceHealthy()) +
nodeManager.getNodeCount(NodeStatus.inServiceHealthyReadOnly());
Healthy state actually representing both of them combined. So, isHealthy changed to cover both now, instead of having separate API.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r909471557
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - Set of available container replicas.
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand<?>> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return emptyMap();
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ final ContainerID id = container.containerID();
+ final Map<DatanodeDetails, SCMCommand<?>> commands = new HashMap<>();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOps) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
Review Comment:
Seems it was added as part of upgrade. READ_ONLY should probably have been an operational state, not a health state. Sigh.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908891725
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - Set of available container replicas.
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand<?>> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return emptyMap();
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOps) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ }
+
+ if (containerHealthResult.underReplicatedDueToDecommission()) {
Review Comment:
This isn't correct, from the java doc:
```
/**
* Indicates whether the under-replication is caused only by replicas
* being decommissioned or entering maintenance. Ie, there are not replicas
* unavailable.
* @return True is the under-replication is caused by decommission.
*/
public boolean underReplicatedDueToDecommission() {
return dueToDecommission;
}
```
This only returns true when the under-replication is only due to decommission. If there are also unavailable indexes, it returns false. You probably need to get the count of `decommissionOnlyReplicas().size() > 0`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908274583
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ (ContainerHealthResult.UnderReplicatedHealthResult)
+ ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (containerHealthResult
+ .isSufficientlyReplicatedAfterPending() || containerHealthResult
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ } else if (result instanceof ContainerHealthResult
+ .UnderReplicatedHealthResult && ((ContainerHealthResult
+ .UnderReplicatedHealthResult) result)
+ .underReplicatedDueToDecommission()) {
+ Map<DatanodeDetails, SCMCommand> commands = new HashMap<>();
+ Set<Integer> decomIndexes = replicaCount.decommissioningIndexes();
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, decomIndexes.size());
+ Iterator<DatanodeDetails> iterator = selectedDatanodes.iterator();
+ // In this case we need to do one to one copy.
+ for (ContainerReplica replica : replicas) {
+ if (decomIndexes.contains(replica.getReplicaIndex())) {
+ if (!iterator.hasNext()) {
+ LOG.warn("Couldn't find the enough targets. Source nodes:"
+ + " {} and the target nodes: {}",
+ sources, selectedDatanodes);
+ break;
+ }
+ DatanodeDetails decommissioningSrcNode
+ = replica.getDatanodeDetails();
+ final ReplicateContainerCommand replicateCommand =
+ new ReplicateContainerCommand(id.getProtobuf().getId(),
+ ImmutableList.of(decommissioningSrcNode));
+ DatanodeDetails target = iterator.next();
+ commands.put(target, replicateCommand);
+ }
+ }
+ return commands;
+ } else {
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable due to lack of"
+ + " available source replicas.", id);
+ } else {
+ LOG.info("There are no missing indexes or decommissioning indexes"
+ + " found for {}", id);
+ }
+ }
+
+ } catch (IOException | IllegalStateException ex) {
+ LOG.warn("Exception while processing for creating the EC reconstruction" +
+ " container commands for {}.",
+ id, ex);
+ }
+ return null;
+ }
+
+ private List<DatanodeDetails> getTargetDatanodes(
+ Set<ContainerReplica> replicas, ContainerInfo container,
+ int requiredNodes) throws IOException {
+ // We should ensure that the target datanode has enough space
+ // for a complete container to be created, but since the container
+ // size may be changed smaller than origin, we should be defensive.
+ final long dataSizeRequired =
+ Math.max(container.getUsedBytes(), currentContainerSize);
Review Comment:
This container size check I just borrowed from LegacyRM#HandleUnderreplication. I think here, some times we go beyond the container sizes. To make sure we cover that case, otherwise configured container size.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908336290
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ (ContainerHealthResult.UnderReplicatedHealthResult)
+ ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (containerHealthResult
+ .isSufficientlyReplicatedAfterPending() || containerHealthResult
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
Review Comment:
We need to subtract and inflight add replicas here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r907885029
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECContainerHealthCheck.java:
##########
@@ -89,19 +96,33 @@ public ContainerHealthResult checkHealth(ContainerInfo container,
return new ContainerHealthResult.HealthyResult(container);
}
- private ECContainerReplicaCount getReplicaCountWithPending(
+ private static ECContainerReplicaCount getReplicaCountWithPending(
Review Comment:
Removed it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r907889001
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationCommandCreator.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationCommandCreator
+ implements ReplicationCommandCreator {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationCommandCreator.class);
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationCommandCreator(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas,
+ pendingOps.getPendingOps(container.containerID()),
+ remainingMaintenanceRedundancy);
+ ContainerHealthResult containerHealthResult =
+ new ECContainerHealthCheck().checkHealth(container, replicaCount);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+
+ if (containerHealthResult.getHealthState()
+ != ContainerHealthResult.HealthState.UNDER_REPLICATED) {
+ LOG.info(
+ "The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(id);
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() > 0) {
Review Comment:
Yeah, we should add enough nodes check here to void sending in sufficient sources.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r907433651
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationCommandCreator.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationCommandCreator
+ implements ReplicationCommandCreator {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationCommandCreator.class);
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationCommandCreator(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas,
+ pendingOps.getPendingOps(container.containerID()),
+ remainingMaintenanceRedundancy);
+ ContainerHealthResult containerHealthResult =
+ new ECContainerHealthCheck().checkHealth(container, replicaCount);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+
+ if (containerHealthResult.getHealthState()
Review Comment:
For now, to keep things simple, we allow a container to be in only 1 state. Under-replicated is more important that over-replicated, so we handle under-replication first, and then when it has recovered, we will handle the over replication.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r907430801
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECContainerHealthCheck.java:
##########
@@ -89,19 +96,33 @@ public ContainerHealthResult checkHealth(ContainerInfo container,
return new ContainerHealthResult.HealthyResult(container);
}
- private ECContainerReplicaCount getReplicaCountWithPending(
+ private static ECContainerReplicaCount getReplicaCountWithPending(
Review Comment:
I guess whole method is not needed now, after the change in HDDS-6699 goes in - we are just passing the List<ContainerReplicaOp> through all the methods directly.
As an aside, we would not have needed to create a new instance of `ContainerReplicaPendingOps` here to form the list - we could have just created a list of ops directly. However I don't think we need any of this after we commit the above Jira, so its not important now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r907471633
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationCommandCreator.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationCommandCreator
+ implements ReplicationCommandCreator {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationCommandCreator.class);
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationCommandCreator(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas,
+ pendingOps.getPendingOps(container.containerID()),
+ remainingMaintenanceRedundancy);
+ ContainerHealthResult containerHealthResult =
+ new ECContainerHealthCheck().checkHealth(container, replicaCount);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+
+ if (containerHealthResult.getHealthState()
+ != ContainerHealthResult.HealthState.UNDER_REPLICATED) {
+ LOG.info(
+ "The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ List<ContainerReplicaOp> pendingOpsForContainer =
Review Comment:
We are already getting the pending ops earlier in this method, so we should re-use the same result here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908779287
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ (ContainerHealthResult.UnderReplicatedHealthResult)
+ ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (containerHealthResult
+ .isSufficientlyReplicatedAfterPending() || containerHealthResult
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ } else if (result instanceof ContainerHealthResult
Review Comment:
discussed offline. yeah agree on that. Made that if if
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908749527
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ (ContainerHealthResult.UnderReplicatedHealthResult)
+ ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (containerHealthResult
+ .isSufficientlyReplicatedAfterPending() || containerHealthResult
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ } else if (result instanceof ContainerHealthResult
+ .UnderReplicatedHealthResult && ((ContainerHealthResult
+ .UnderReplicatedHealthResult) result)
+ .underReplicatedDueToDecommission()) {
+ Map<DatanodeDetails, SCMCommand> commands = new HashMap<>();
+ Set<Integer> decomIndexes = replicaCount.decommissioningIndexes();
Review Comment:
As discussed offline, the decommissionIndexes may already have replicas created, so we need to create a new method on ECContainerReplicaCounts - getDecommissionOnlyIndexes() to get the indexes that only have no other copies.
Maintenance can be handled in a followup Jira.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908291393
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ (ContainerHealthResult.UnderReplicatedHealthResult)
+ ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (containerHealthResult
+ .isSufficientlyReplicatedAfterPending() || containerHealthResult
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ } else if (result instanceof ContainerHealthResult
+ .UnderReplicatedHealthResult && ((ContainerHealthResult
+ .UnderReplicatedHealthResult) result)
+ .underReplicatedDueToDecommission()) {
+ Map<DatanodeDetails, SCMCommand> commands = new HashMap<>();
+ Set<Integer> decomIndexes = replicaCount.decommissioningIndexes();
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, decomIndexes.size());
+ Iterator<DatanodeDetails> iterator = selectedDatanodes.iterator();
+ // In this case we need to do one to one copy.
+ for (ContainerReplica replica : replicas) {
+ if (decomIndexes.contains(replica.getReplicaIndex())) {
+ if (!iterator.hasNext()) {
+ LOG.warn("Couldn't find the enough targets. Source nodes:"
+ + " {} and the target nodes: {}",
+ sources, selectedDatanodes);
+ break;
+ }
+ DatanodeDetails decommissioningSrcNode
+ = replica.getDatanodeDetails();
+ final ReplicateContainerCommand replicateCommand =
+ new ReplicateContainerCommand(id.getProtobuf().getId(),
+ ImmutableList.of(decommissioningSrcNode));
+ DatanodeDetails target = iterator.next();
+ commands.put(target, replicateCommand);
+ }
+ }
+ return commands;
+ } else {
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable due to lack of"
+ + " available source replicas.", id);
+ } else {
+ LOG.info("There are no missing indexes or decommissioning indexes"
+ + " found for {}", id);
+ }
+ }
+
+ } catch (IOException | IllegalStateException ex) {
+ LOG.warn("Exception while processing for creating the EC reconstruction" +
Review Comment:
Most of the specific cases we logged with the details, this is one case we may get throw from chooseDatanodes for which we already included the exception in log which will mostly redundant information.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908361826
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ (ContainerHealthResult.UnderReplicatedHealthResult)
+ ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (containerHealthResult
+ .isSufficientlyReplicatedAfterPending() || containerHealthResult
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ } else if (result instanceof ContainerHealthResult
+ .UnderReplicatedHealthResult && ((ContainerHealthResult
+ .UnderReplicatedHealthResult) result)
+ .underReplicatedDueToDecommission()) {
+ Map<DatanodeDetails, SCMCommand> commands = new HashMap<>();
+ Set<Integer> decomIndexes = replicaCount.decommissioningIndexes();
Review Comment:
Isn't that can be handled in followups? I think this still works as one decom replica acts as src. ( at the same time having over replication may be rare case, but still we can handle if available). In worst case we may end up doing one replication. As it's not harm, we can fix in followup once we fix ReplicaCount for figure out the right counters.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] adoroszlai commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
adoroszlai commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908488858
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/UnderReplicationHandler.java:
##########
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * This interface to create respective commands after processing the replicas
+ * with pending ops and health check results.
+ */
+public interface UnderReplicationHandler {
+
+ /**
+ * Identify a new set of datanode(s) to replicate/reconstruct the container
+ * and form the SCM commands to send it to DN.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
Review Comment:
Nit: comment seems stale, `replicas` is a `Set`.
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,243 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return null;
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return null;
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ }
Review Comment:
Is it intentional that the method proceeds for unrecoverable case?
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,243 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return null;
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return null;
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOps) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ } else if (containerHealthResult.underReplicatedDueToDecommission()) {
+ Map<DatanodeDetails, SCMCommand> commands = new HashMap<>();
+ Set<Integer> decomIndexes = replicaCount.decommissioningIndexes();
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, decomIndexes.size());
+ Iterator<DatanodeDetails> iterator = selectedDatanodes.iterator();
+ // In this case we need to do one to one copy.
+ for (ContainerReplica replica : replicas) {
+ if (decomIndexes.contains(replica.getReplicaIndex())) {
+ if (!iterator.hasNext()) {
+ LOG.warn("Couldn't find the enough targets. Available source"
+ + " nodes: {}, the target nodes: {} and the decommission"
+ + " indexes: {}", replicas, selectedDatanodes, decomIndexes);
+ break;
+ }
+ DatanodeDetails decommissioningSrcNode
+ = replica.getDatanodeDetails();
+ final ReplicateContainerCommand replicateCommand =
+ new ReplicateContainerCommand(id.getProtobuf().getId(),
+ ImmutableList.of(decommissioningSrcNode));
+ DatanodeDetails target = iterator.next();
+ commands.put(target, replicateCommand);
+ }
+ }
+ return commands;
+ } else {
+ LOG.info("There are no missing indexes or decommissioning indexes"
+ + " found for {}", id);
+ }
+
+ } catch (IOException | IllegalStateException ex) {
+ LOG.warn("Exception while processing for creating the EC reconstruction" +
+ " container commands for {}.",
+ id, ex);
+ }
+ return null;
+ }
+
+ private List<DatanodeDetails> getTargetDatanodes(
+ Set<ContainerReplica> replicas, ContainerInfo container,
+ int requiredNodes) throws IOException {
+ // We should ensure that the target datanode has enough space
+ // for a complete container to be created, but since the container
+ // size may be changed smaller than origin, we should be defensive.
+ final long dataSizeRequired =
+ Math.max(container.getUsedBytes(), currentContainerSize);
+ final List<DatanodeDetails> excludeList =
+ replicas.stream().map(ContainerReplica::getDatanodeDetails)
+ .collect(Collectors.toList());
+
+ return containerPlacement
+ .chooseDatanodes(excludeList, null, requiredNodes, 0, dataSizeRequired);
+ }
+
+ private static byte[] int2byte(List<Integer> src) {
+ byte[] dst = new byte[src.size()];
+
+ for (int i = 0; i < src.size(); i++) {
+ dst[i] = (byte) ((int) src.get(i));
Review Comment:
Nit: could use `byteValue` directly?
```suggestion
dst[i] = src.get(i).byteValue();
```
##########
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestECUnderReplicationHandler.java:
##########
@@ -0,0 +1,255 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.states;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.MockNodeManager;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementMetrics;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRackScatter;
+import org.apache.hadoop.hdds.scm.container.replication.ContainerHealthResult;
+import org.apache.hadoop.hdds.scm.container.replication.ECUnderReplicationHandler;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.net.NetworkTopology;
+import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl;
+import org.apache.hadoop.hdds.scm.net.NodeSchema;
+import org.apache.hadoop.hdds.scm.net.NodeSchemaManager;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.NodeStatus;
+import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
+import org.apache.hadoop.ozone.container.common.SCMTestUtils;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.assertj.core.util.Lists;
+import org.junit.Assert;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONING;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE;
+import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.CLOSED;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.LEAF_SCHEMA;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.RACK_SCHEMA;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_SCHEMA;
+
+/**
+ * Tests the ECUnderReplicationHandling functionality.
+ */
+public class TestECUnderReplicationHandler {
+ private ECReplicationConfig repConfig;
+ private ContainerInfo container;
+ private NodeManager nodeManager;
+ private OzoneConfiguration conf;
+ private NetworkTopology cluster;
+ private PlacementPolicy policy;
+ private SCMContainerPlacementMetrics metrics;
+
+ @BeforeEach
+ public void setup() {
+ nodeManager = new MockNodeManager(true, 10) {
+ @Override
+ public NodeStatus getNodeStatus(DatanodeDetails dd)
+ throws NodeNotFoundException {
+ return NodeStatus.inServiceHealthy();
+ }
+ };
+ conf = SCMTestUtils.getConf();
+ repConfig = new ECReplicationConfig(3, 2);
+ container = createContainer(HddsProtos.LifeCycleState.CLOSED, repConfig);
+ // create placement policy instances
+ cluster = new NetworkTopologyImpl(NodeSchemaManager.getInstance());
+ policy =
+ new SCMContainerPlacementRackScatter(nodeManager, conf, cluster, true,
+ metrics) {
+ @Override
+ public List<DatanodeDetails> chooseDatanodes(
+ List<DatanodeDetails> excludedNodes,
+ List<DatanodeDetails> favoredNodes, int nodesRequiredToChoose,
+ long metadataSizeRequired, long dataSizeRequired)
+ throws SCMException {
+ List<DatanodeDetails> dns = new ArrayList<>();
+ for (int i = 0; i < nodesRequiredToChoose; i++) {
+ dns.add(MockDatanodeDetails.randomDatanodeDetails());
+ }
+ return dns;
+ }
+ };
Review Comment:
Maybe I'm missing something, but I think we could use `new SCMCommonPlacementPolicy` here, so we can keep `SCMContainerPlacementRackScatter` as `final`.
```suggestion
policy = new SCMCommonPlacementPolicy(nodeManager, conf) {
@Override
public DatanodeDetails chooseNode(List<DatanodeDetails> healthyNodes) {
return null;
}
};
```
##########
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestECUnderReplicationHandler.java:
##########
@@ -0,0 +1,255 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.states;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.MockNodeManager;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementMetrics;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRackScatter;
+import org.apache.hadoop.hdds.scm.container.replication.ContainerHealthResult;
+import org.apache.hadoop.hdds.scm.container.replication.ECUnderReplicationHandler;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.net.NetworkTopology;
+import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl;
+import org.apache.hadoop.hdds.scm.net.NodeSchema;
+import org.apache.hadoop.hdds.scm.net.NodeSchemaManager;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.NodeStatus;
+import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
+import org.apache.hadoop.ozone.container.common.SCMTestUtils;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.assertj.core.util.Lists;
+import org.junit.Assert;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONING;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE;
+import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.CLOSED;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.LEAF_SCHEMA;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.RACK_SCHEMA;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_SCHEMA;
+
+/**
+ * Tests the ECUnderReplicationHandling functionality.
+ */
+public class TestECUnderReplicationHandler {
+ private ECReplicationConfig repConfig;
+ private ContainerInfo container;
+ private NodeManager nodeManager;
+ private OzoneConfiguration conf;
+ private NetworkTopology cluster;
+ private PlacementPolicy policy;
+ private SCMContainerPlacementMetrics metrics;
+
+ @BeforeEach
+ public void setup() {
+ nodeManager = new MockNodeManager(true, 10) {
+ @Override
+ public NodeStatus getNodeStatus(DatanodeDetails dd)
+ throws NodeNotFoundException {
+ return NodeStatus.inServiceHealthy();
+ }
+ };
+ conf = SCMTestUtils.getConf();
+ repConfig = new ECReplicationConfig(3, 2);
+ container = createContainer(HddsProtos.LifeCycleState.CLOSED, repConfig);
+ // create placement policy instances
+ cluster = new NetworkTopologyImpl(NodeSchemaManager.getInstance());
+ policy =
+ new SCMContainerPlacementRackScatter(nodeManager, conf, cluster, true,
+ metrics) {
+ @Override
+ public List<DatanodeDetails> chooseDatanodes(
+ List<DatanodeDetails> excludedNodes,
+ List<DatanodeDetails> favoredNodes, int nodesRequiredToChoose,
+ long metadataSizeRequired, long dataSizeRequired)
+ throws SCMException {
+ List<DatanodeDetails> dns = new ArrayList<>();
+ for (int i = 0; i < nodesRequiredToChoose; i++) {
+ dns.add(MockDatanodeDetails.randomDatanodeDetails());
+ }
+ return dns;
+ }
+ };
+ NodeSchema[] schemas =
+ new NodeSchema[] {ROOT_SCHEMA, RACK_SCHEMA, LEAF_SCHEMA};
+ NodeSchemaManager.getInstance().init(schemas, true);
+
+ }
+
+ @BeforeEach
+ public void init() {
+ metrics = SCMContainerPlacementMetrics.create();
+ }
+
+ @Test
+ public void testUnderReplicationWithMissingParityIndex5() {
+ Set<ContainerReplica> availableReplicas =
+ registerNodes(Pair.of(IN_SERVICE, 1), Pair.of(IN_SERVICE, 2),
+ Pair.of(IN_SERVICE, 3), Pair.of(IN_SERVICE, 4));
+ testUnderReplicationWithMissingIndexes(ImmutableList.of(5),
+ availableReplicas, false);
+ }
+
+ @Test
+ public void testUnderReplicationWithMissingIndex34() {
+ Set<ContainerReplica> availableReplicas =
+ registerNodes(Pair.of(IN_SERVICE, 1), Pair.of(IN_SERVICE, 2),
+ Pair.of(IN_SERVICE, 5));
+ testUnderReplicationWithMissingIndexes(ImmutableList.of(3, 4),
+ availableReplicas, false);
+ }
+
+ @Test
+ public void testUnderReplicationWithMissingIndex2345() {
+ Set<ContainerReplica> availableReplicas =
+ registerNodes(Pair.of(IN_SERVICE, 1));
+ testUnderReplicationWithMissingIndexes(ImmutableList.of(2, 3, 4, 5),
+ availableReplicas, false);
+ }
+
+ @Test
+ public void testUnderReplicationWithMissingIndex12345() {
+ Set<ContainerReplica> availableReplicas = new HashSet<>();
+ testUnderReplicationWithMissingIndexes(ImmutableList.of(1, 2, 3, 4, 5),
+ availableReplicas, false);
+ }
+
+ @Test
+ public void testUnderReplicationWithDecomIndex1() {
+ Set<ContainerReplica> availableReplicas =
+ registerNodes(Pair.of(DECOMMISSIONING, 1), Pair.of(IN_SERVICE, 2),
+ Pair.of(IN_SERVICE, 3), Pair.of(IN_SERVICE, 4),
+ Pair.of(IN_SERVICE, 5));
+ testUnderReplicationWithMissingIndexes(Lists.emptyList(), availableReplicas,
+ true);
+ }
+
+ @Test
+ public void testUnderReplicationWithDecomIndex12() {
+ Set<ContainerReplica> availableReplicas =
+ registerNodes(Pair.of(DECOMMISSIONING, 1), Pair.of(DECOMMISSIONING, 2),
+ Pair.of(IN_SERVICE, 3), Pair.of(IN_SERVICE, 4),
+ Pair.of(IN_SERVICE, 5));
+ testUnderReplicationWithMissingIndexes(Lists.emptyList(), availableReplicas,
+ true);
+ }
+
+ @Test
+ public void testUnderReplicationWithMixedDecomAndMissingIndexes() {
+ Set<ContainerReplica> availableReplicas =
+ registerNodes(Pair.of(DECOMMISSIONING, 1), Pair.of(DECOMMISSIONING, 2),
+ Pair.of(IN_SERVICE, 3), Pair.of(IN_SERVICE, 4));
+ testUnderReplicationWithMissingIndexes(ImmutableList.of(5),
+ availableReplicas, false);
+ }
+
+ public void testUnderReplicationWithMissingIndexes(
+ List<Integer> missingIndexes, Set<ContainerReplica> availableReplicas,
+ boolean decom) {
+ ECUnderReplicationHandler ecURH =
+ new ECUnderReplicationHandler(policy, conf, nodeManager);
+ ContainerHealthResult.UnderReplicatedHealthResult result =
+ Mockito.mock(ContainerHealthResult.UnderReplicatedHealthResult.class);
+ Mockito.when(result.underReplicatedDueToDecommission()).thenReturn(decom);
+ Mockito.when(result.isUnrecoverable()).thenReturn(false);
+ Mockito.when(result.getContainerInfo()).thenReturn(container);
+
+ Map<DatanodeDetails, SCMCommand> datanodeDetailsSCMCommandMap = ecURH
+ .processAndCreateCommands(availableReplicas, ImmutableList.of(),
+ result, 1);
+ if (!decom) {
+ if (missingIndexes.size() <= repConfig.getParity()) {
+ Assertions.assertEquals(1, datanodeDetailsSCMCommandMap.size());
+ Map.Entry<DatanodeDetails, SCMCommand> dnVsCommand =
+ datanodeDetailsSCMCommandMap.entrySet().iterator().next();
+ Assert.assertTrue(
+ dnVsCommand.getValue() instanceof ReconstructECContainersCommand);
+ byte[] missingIndexesByteArr = new byte[missingIndexes.size()];
+ for (int i = 0; i < missingIndexes.size(); i++) {
+ missingIndexesByteArr[i] = (byte) (int) missingIndexes.get(i);
+ }
+ Assert.assertArrayEquals(missingIndexesByteArr,
+ ((ReconstructECContainersCommand) dnVsCommand.getValue())
+ .getMissingContainerIndexes());
+ } else {
+ Assertions.assertNull(datanodeDetailsSCMCommandMap);
+ }
+ } else {
+ int numDecomIndexes = 0;
+ for (ContainerReplica repl : availableReplicas) {
+ if (repl.getDatanodeDetails()
+ .getPersistedOpState() == DECOMMISSIONING) {
+ numDecomIndexes++;
+ }
+ }
+ Assertions
+ .assertEquals(numDecomIndexes, datanodeDetailsSCMCommandMap.size());
+ Set<Map.Entry<DatanodeDetails, SCMCommand>> entries =
+ datanodeDetailsSCMCommandMap.entrySet();
+ for (Map.Entry<DatanodeDetails, SCMCommand> dnCommand : entries) {
+ Assert.assertTrue(
+ dnCommand.getValue() instanceof ReplicateContainerCommand);
+ }
+ }
+ }
+
+ private Set<ContainerReplica> registerNodes(
+ Pair<HddsProtos.NodeOperationalState, Integer>... states) {
+ Set<ContainerReplica> replica = new HashSet<>();
+ for (Pair<HddsProtos.NodeOperationalState, Integer> s : states) {
+ DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails();
+ dn.setPersistedOpState(s.getLeft());
+ replica.add(new ContainerReplica.ContainerReplicaBuilder()
+ .setContainerID(ContainerID.valueOf(1)).setContainerState(CLOSED)
+ .setDatanodeDetails(dn).setOriginNodeId(dn.getUuid()).setSequenceId(1)
+ .setReplicaIndex(s.getRight()).build());
+ }
+ return replica;
+ }
+
+ private ContainerInfo createContainer(HddsProtos.LifeCycleState state,
+ ReplicationConfig replicationConfig) {
+
+ return new ContainerInfo.Builder()
+ .setContainerID(ContainerID.valueOf(1).getId()).setState(state)
+ .setReplicationConfig(replicationConfig).build();
+ }
Review Comment:
Seems like these are duplicated from `TestECContainerReplicaCount`. Can we reuse instead of copy?
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,243 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return null;
Review Comment:
Nit: I think it would be more safe to return empty map instead of null. (Note: (1) needs import, (2) also apply to other return statements, (3) needs tweak in test.)
```suggestion
return emptyMap();
```
https://github.com/apache/ozone/blob/2fd9fce6582b94faa1dd5a70535af22f179ec55a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestECUnderReplicationHandler.java#L213
##########
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestECUnderReplicationHandler.java:
##########
@@ -0,0 +1,255 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.states;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.MockNodeManager;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementMetrics;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRackScatter;
+import org.apache.hadoop.hdds.scm.container.replication.ContainerHealthResult;
+import org.apache.hadoop.hdds.scm.container.replication.ECUnderReplicationHandler;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.net.NetworkTopology;
+import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl;
+import org.apache.hadoop.hdds.scm.net.NodeSchema;
+import org.apache.hadoop.hdds.scm.net.NodeSchemaManager;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.NodeStatus;
+import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
+import org.apache.hadoop.ozone.container.common.SCMTestUtils;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.assertj.core.util.Lists;
+import org.junit.Assert;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONING;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE;
+import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.CLOSED;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.LEAF_SCHEMA;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.RACK_SCHEMA;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_SCHEMA;
+
+/**
+ * Tests the ECUnderReplicationHandling functionality.
+ */
+public class TestECUnderReplicationHandler {
+ private ECReplicationConfig repConfig;
+ private ContainerInfo container;
+ private NodeManager nodeManager;
+ private OzoneConfiguration conf;
+ private NetworkTopology cluster;
+ private PlacementPolicy policy;
+ private SCMContainerPlacementMetrics metrics;
+
+ @BeforeEach
+ public void setup() {
+ nodeManager = new MockNodeManager(true, 10) {
+ @Override
+ public NodeStatus getNodeStatus(DatanodeDetails dd)
+ throws NodeNotFoundException {
+ return NodeStatus.inServiceHealthy();
+ }
+ };
+ conf = SCMTestUtils.getConf();
+ repConfig = new ECReplicationConfig(3, 2);
+ container = createContainer(HddsProtos.LifeCycleState.CLOSED, repConfig);
+ // create placement policy instances
+ cluster = new NetworkTopologyImpl(NodeSchemaManager.getInstance());
+ policy =
+ new SCMContainerPlacementRackScatter(nodeManager, conf, cluster, true,
+ metrics) {
+ @Override
+ public List<DatanodeDetails> chooseDatanodes(
+ List<DatanodeDetails> excludedNodes,
+ List<DatanodeDetails> favoredNodes, int nodesRequiredToChoose,
+ long metadataSizeRequired, long dataSizeRequired)
+ throws SCMException {
+ List<DatanodeDetails> dns = new ArrayList<>();
+ for (int i = 0; i < nodesRequiredToChoose; i++) {
+ dns.add(MockDatanodeDetails.randomDatanodeDetails());
+ }
+ return dns;
+ }
+ };
+ NodeSchema[] schemas =
+ new NodeSchema[] {ROOT_SCHEMA, RACK_SCHEMA, LEAF_SCHEMA};
+ NodeSchemaManager.getInstance().init(schemas, true);
+
+ }
+
+ @BeforeEach
+ public void init() {
+ metrics = SCMContainerPlacementMetrics.create();
+ }
Review Comment:
Can be merged with `setup()`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908932511
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - Set of available container replicas.
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand<?>> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return emptyMap();
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOps) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ }
+
+ if (containerHealthResult.underReplicatedDueToDecommission()) {
+ Map<DatanodeDetails, SCMCommand<?>> commands = new HashMap<>();
+ Set<Integer> decomIndexes = replicaCount.decommissioningOnlyIndexes();
Review Comment:
Done
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - Set of available container replicas.
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand<?>> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return emptyMap();
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOps) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ }
+
+ if (containerHealthResult.underReplicatedDueToDecommission()) {
Review Comment:
used decomOnly indexes check.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r909182971
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - Set of available container replicas.
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand<?>> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return emptyMap();
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ if (replicaCount.unRecoverable()) {
Review Comment:
From where to delete? We just need to retry in next attempt.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r905959715
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ECContainerReplicaCount.java:
##########
@@ -68,16 +70,27 @@ public class ECContainerReplicaCount {
private final Map<Integer, Integer> healthyIndexes = new HashMap<>();
private final Map<Integer, Integer> decommissionIndexes = new HashMap<>();
private final Map<Integer, Integer> maintenanceIndexes = new HashMap<>();
+ private final Set<ContainerReplica> replicas;
public ECContainerReplicaCount(ContainerInfo containerInfo,
- Set<ContainerReplica> replicas, List<Integer> indexesPendingAdd,
- List<Integer> indexesPendingDelete, int remainingMaintenanceRedundancy) {
+ Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
Review Comment:
I don't think we should pass the instance of ContainerReplicaPending ops in here, as it makes this class dependent on it. Rather we should pass in the list of ops which were returned from the ContainerReplicaPendingOps instance in RM. Ie the parameter is List<ContainerReplicaOp>.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r907884925
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECContainerHealthCheck.java:
##########
@@ -89,19 +96,33 @@ public ContainerHealthResult checkHealth(ContainerInfo container,
return new ContainerHealthResult.HealthyResult(container);
}
- private ECContainerReplicaCount getReplicaCountWithPending(
+ private static ECContainerReplicaCount getReplicaCountWithPending(
ContainerInfo container, Set<ContainerReplica> replicas,
List<Pair<Integer, DatanodeDetails>> replicasPendingAdd,
List<Pair<Integer, DatanodeDetails>> replicasPendingDelete,
int remainingRedundancyForMaintenance) {
- List<Integer> indexesPendingAdd = replicasPendingAdd.stream()
- .map(i -> i.getLeft()).collect(Collectors.toList());
- List<Integer> indexesPendingDelete = replicasPendingDelete.stream()
- .map(i -> i.getLeft()).collect(Collectors.toList());
+ ContainerReplicaPendingOps pendingOps =
Review Comment:
Now this method no more needed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908274583
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ (ContainerHealthResult.UnderReplicatedHealthResult)
+ ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (containerHealthResult
+ .isSufficientlyReplicatedAfterPending() || containerHealthResult
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ } else if (result instanceof ContainerHealthResult
+ .UnderReplicatedHealthResult && ((ContainerHealthResult
+ .UnderReplicatedHealthResult) result)
+ .underReplicatedDueToDecommission()) {
+ Map<DatanodeDetails, SCMCommand> commands = new HashMap<>();
+ Set<Integer> decomIndexes = replicaCount.decommissioningIndexes();
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, decomIndexes.size());
+ Iterator<DatanodeDetails> iterator = selectedDatanodes.iterator();
+ // In this case we need to do one to one copy.
+ for (ContainerReplica replica : replicas) {
+ if (decomIndexes.contains(replica.getReplicaIndex())) {
+ if (!iterator.hasNext()) {
+ LOG.warn("Couldn't find the enough targets. Source nodes:"
+ + " {} and the target nodes: {}",
+ sources, selectedDatanodes);
+ break;
+ }
+ DatanodeDetails decommissioningSrcNode
+ = replica.getDatanodeDetails();
+ final ReplicateContainerCommand replicateCommand =
+ new ReplicateContainerCommand(id.getProtobuf().getId(),
+ ImmutableList.of(decommissioningSrcNode));
+ DatanodeDetails target = iterator.next();
+ commands.put(target, replicateCommand);
+ }
+ }
+ return commands;
+ } else {
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable due to lack of"
+ + " available source replicas.", id);
+ } else {
+ LOG.info("There are no missing indexes or decommissioning indexes"
+ + " found for {}", id);
+ }
+ }
+
+ } catch (IOException | IllegalStateException ex) {
+ LOG.warn("Exception while processing for creating the EC reconstruction" +
+ " container commands for {}.",
+ id, ex);
+ }
+ return null;
+ }
+
+ private List<DatanodeDetails> getTargetDatanodes(
+ Set<ContainerReplica> replicas, ContainerInfo container,
+ int requiredNodes) throws IOException {
+ // We should ensure that the target datanode has enough space
+ // for a complete container to be created, but since the container
+ // size may be changed smaller than origin, we should be defensive.
+ final long dataSizeRequired =
+ Math.max(container.getUsedBytes(), currentContainerSize);
Review Comment:
This container size check I just borrowed from LegacyRM#HandleUnderreplication. I think here, some times we go beyond the configured container sizes. To make sure we cover that case, otherwise configured container size.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908746980
##########
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestECUnderReplicationHandler.java:
##########
@@ -0,0 +1,255 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.states;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.MockNodeManager;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementMetrics;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRackScatter;
+import org.apache.hadoop.hdds.scm.container.replication.ContainerHealthResult;
+import org.apache.hadoop.hdds.scm.container.replication.ECUnderReplicationHandler;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.net.NetworkTopology;
+import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl;
+import org.apache.hadoop.hdds.scm.net.NodeSchema;
+import org.apache.hadoop.hdds.scm.net.NodeSchemaManager;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.NodeStatus;
+import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
+import org.apache.hadoop.ozone.container.common.SCMTestUtils;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.assertj.core.util.Lists;
+import org.junit.Assert;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONING;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE;
+import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.CLOSED;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.LEAF_SCHEMA;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.RACK_SCHEMA;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_SCHEMA;
+
+/**
+ * Tests the ECUnderReplicationHandling functionality.
+ */
+public class TestECUnderReplicationHandler {
+ private ECReplicationConfig repConfig;
+ private ContainerInfo container;
+ private NodeManager nodeManager;
+ private OzoneConfiguration conf;
+ private NetworkTopology cluster;
+ private PlacementPolicy policy;
+ private SCMContainerPlacementMetrics metrics;
+
+ @BeforeEach
+ public void setup() {
+ nodeManager = new MockNodeManager(true, 10) {
+ @Override
+ public NodeStatus getNodeStatus(DatanodeDetails dd)
+ throws NodeNotFoundException {
+ return NodeStatus.inServiceHealthy();
+ }
+ };
+ conf = SCMTestUtils.getConf();
+ repConfig = new ECReplicationConfig(3, 2);
+ container = createContainer(HddsProtos.LifeCycleState.CLOSED, repConfig);
+ // create placement policy instances
+ cluster = new NetworkTopologyImpl(NodeSchemaManager.getInstance());
+ policy =
+ new SCMContainerPlacementRackScatter(nodeManager, conf, cluster, true,
+ metrics) {
+ @Override
+ public List<DatanodeDetails> chooseDatanodes(
+ List<DatanodeDetails> excludedNodes,
+ List<DatanodeDetails> favoredNodes, int nodesRequiredToChoose,
+ long metadataSizeRequired, long dataSizeRequired)
+ throws SCMException {
+ List<DatanodeDetails> dns = new ArrayList<>();
+ for (int i = 0; i < nodesRequiredToChoose; i++) {
+ dns.add(MockDatanodeDetails.randomDatanodeDetails());
+ }
+ return dns;
+ }
+ };
+ NodeSchema[] schemas =
+ new NodeSchema[] {ROOT_SCHEMA, RACK_SCHEMA, LEAF_SCHEMA};
+ NodeSchemaManager.getInstance().init(schemas, true);
+
+ }
+
+ @BeforeEach
+ public void init() {
+ metrics = SCMContainerPlacementMetrics.create();
+ }
+
+ @Test
+ public void testUnderReplicationWithMissingParityIndex5() {
+ Set<ContainerReplica> availableReplicas =
+ registerNodes(Pair.of(IN_SERVICE, 1), Pair.of(IN_SERVICE, 2),
+ Pair.of(IN_SERVICE, 3), Pair.of(IN_SERVICE, 4));
+ testUnderReplicationWithMissingIndexes(ImmutableList.of(5),
+ availableReplicas, false);
+ }
+
+ @Test
+ public void testUnderReplicationWithMissingIndex34() {
+ Set<ContainerReplica> availableReplicas =
+ registerNodes(Pair.of(IN_SERVICE, 1), Pair.of(IN_SERVICE, 2),
+ Pair.of(IN_SERVICE, 5));
+ testUnderReplicationWithMissingIndexes(ImmutableList.of(3, 4),
+ availableReplicas, false);
+ }
+
+ @Test
+ public void testUnderReplicationWithMissingIndex2345() {
+ Set<ContainerReplica> availableReplicas =
+ registerNodes(Pair.of(IN_SERVICE, 1));
+ testUnderReplicationWithMissingIndexes(ImmutableList.of(2, 3, 4, 5),
+ availableReplicas, false);
+ }
+
+ @Test
+ public void testUnderReplicationWithMissingIndex12345() {
+ Set<ContainerReplica> availableReplicas = new HashSet<>();
+ testUnderReplicationWithMissingIndexes(ImmutableList.of(1, 2, 3, 4, 5),
+ availableReplicas, false);
+ }
+
+ @Test
+ public void testUnderReplicationWithDecomIndex1() {
+ Set<ContainerReplica> availableReplicas =
+ registerNodes(Pair.of(DECOMMISSIONING, 1), Pair.of(IN_SERVICE, 2),
+ Pair.of(IN_SERVICE, 3), Pair.of(IN_SERVICE, 4),
+ Pair.of(IN_SERVICE, 5));
+ testUnderReplicationWithMissingIndexes(Lists.emptyList(), availableReplicas,
+ true);
+ }
+
+ @Test
+ public void testUnderReplicationWithDecomIndex12() {
+ Set<ContainerReplica> availableReplicas =
+ registerNodes(Pair.of(DECOMMISSIONING, 1), Pair.of(DECOMMISSIONING, 2),
+ Pair.of(IN_SERVICE, 3), Pair.of(IN_SERVICE, 4),
+ Pair.of(IN_SERVICE, 5));
+ testUnderReplicationWithMissingIndexes(Lists.emptyList(), availableReplicas,
+ true);
+ }
+
+ @Test
+ public void testUnderReplicationWithMixedDecomAndMissingIndexes() {
+ Set<ContainerReplica> availableReplicas =
+ registerNodes(Pair.of(DECOMMISSIONING, 1), Pair.of(DECOMMISSIONING, 2),
+ Pair.of(IN_SERVICE, 3), Pair.of(IN_SERVICE, 4));
+ testUnderReplicationWithMissingIndexes(ImmutableList.of(5),
+ availableReplicas, false);
+ }
+
+ public void testUnderReplicationWithMissingIndexes(
+ List<Integer> missingIndexes, Set<ContainerReplica> availableReplicas,
+ boolean decom) {
+ ECUnderReplicationHandler ecURH =
+ new ECUnderReplicationHandler(policy, conf, nodeManager);
+ ContainerHealthResult.UnderReplicatedHealthResult result =
+ Mockito.mock(ContainerHealthResult.UnderReplicatedHealthResult.class);
+ Mockito.when(result.underReplicatedDueToDecommission()).thenReturn(decom);
+ Mockito.when(result.isUnrecoverable()).thenReturn(false);
+ Mockito.when(result.getContainerInfo()).thenReturn(container);
+
+ Map<DatanodeDetails, SCMCommand> datanodeDetailsSCMCommandMap = ecURH
+ .processAndCreateCommands(availableReplicas, ImmutableList.of(),
+ result, 1);
+ if (!decom) {
+ if (missingIndexes.size() <= repConfig.getParity()) {
+ Assertions.assertEquals(1, datanodeDetailsSCMCommandMap.size());
+ Map.Entry<DatanodeDetails, SCMCommand> dnVsCommand =
+ datanodeDetailsSCMCommandMap.entrySet().iterator().next();
+ Assert.assertTrue(
+ dnVsCommand.getValue() instanceof ReconstructECContainersCommand);
+ byte[] missingIndexesByteArr = new byte[missingIndexes.size()];
+ for (int i = 0; i < missingIndexes.size(); i++) {
+ missingIndexesByteArr[i] = (byte) (int) missingIndexes.get(i);
+ }
+ Assert.assertArrayEquals(missingIndexesByteArr,
+ ((ReconstructECContainersCommand) dnVsCommand.getValue())
+ .getMissingContainerIndexes());
+ } else {
+ Assertions.assertNull(datanodeDetailsSCMCommandMap);
+ }
+ } else {
+ int numDecomIndexes = 0;
+ for (ContainerReplica repl : availableReplicas) {
+ if (repl.getDatanodeDetails()
+ .getPersistedOpState() == DECOMMISSIONING) {
+ numDecomIndexes++;
+ }
+ }
+ Assertions
+ .assertEquals(numDecomIndexes, datanodeDetailsSCMCommandMap.size());
+ Set<Map.Entry<DatanodeDetails, SCMCommand>> entries =
+ datanodeDetailsSCMCommandMap.entrySet();
+ for (Map.Entry<DatanodeDetails, SCMCommand> dnCommand : entries) {
+ Assert.assertTrue(
+ dnCommand.getValue() instanceof ReplicateContainerCommand);
+ }
+ }
+ }
+
+ private Set<ContainerReplica> registerNodes(
+ Pair<HddsProtos.NodeOperationalState, Integer>... states) {
+ Set<ContainerReplica> replica = new HashSet<>();
+ for (Pair<HddsProtos.NodeOperationalState, Integer> s : states) {
+ DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails();
+ dn.setPersistedOpState(s.getLeft());
+ replica.add(new ContainerReplica.ContainerReplicaBuilder()
+ .setContainerID(ContainerID.valueOf(1)).setContainerState(CLOSED)
+ .setDatanodeDetails(dn).setOriginNodeId(dn.getUuid()).setSequenceId(1)
+ .setReplicaIndex(s.getRight()).build());
+ }
+ return replica;
+ }
+
+ private ContainerInfo createContainer(HddsProtos.LifeCycleState state,
+ ReplicationConfig replicationConfig) {
+
+ return new ContainerInfo.Builder()
+ .setContainerID(ContainerID.valueOf(1).getId()).setState(state)
+ .setReplicationConfig(replicationConfig).build();
+ }
Review Comment:
There is `ReplicationTestUtil.createReplicas(...)` that perhaps replaces registerNodes. We could do with making `TestECContainerReplicaCount` use the Util class too I guess.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908778165
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,243 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return null;
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return null;
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOps) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ } else if (containerHealthResult.underReplicatedDueToDecommission()) {
+ Map<DatanodeDetails, SCMCommand> commands = new HashMap<>();
+ Set<Integer> decomIndexes = replicaCount.decommissioningIndexes();
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, decomIndexes.size());
+ Iterator<DatanodeDetails> iterator = selectedDatanodes.iterator();
+ // In this case we need to do one to one copy.
+ for (ContainerReplica replica : replicas) {
+ if (decomIndexes.contains(replica.getReplicaIndex())) {
+ if (!iterator.hasNext()) {
+ LOG.warn("Couldn't find the enough targets. Available source"
+ + " nodes: {}, the target nodes: {} and the decommission"
+ + " indexes: {}", replicas, selectedDatanodes, decomIndexes);
+ break;
+ }
+ DatanodeDetails decommissioningSrcNode
+ = replica.getDatanodeDetails();
+ final ReplicateContainerCommand replicateCommand =
+ new ReplicateContainerCommand(id.getProtobuf().getId(),
+ ImmutableList.of(decommissioningSrcNode));
+ DatanodeDetails target = iterator.next();
+ commands.put(target, replicateCommand);
+ }
+ }
+ return commands;
+ } else {
+ LOG.info("There are no missing indexes or decommissioning indexes"
+ + " found for {}", id);
+ }
+
+ } catch (IOException | IllegalStateException ex) {
+ LOG.warn("Exception while processing for creating the EC reconstruction" +
+ " container commands for {}.",
+ id, ex);
+ }
+ return null;
+ }
+
+ private List<DatanodeDetails> getTargetDatanodes(
+ Set<ContainerReplica> replicas, ContainerInfo container,
+ int requiredNodes) throws IOException {
+ // We should ensure that the target datanode has enough space
+ // for a complete container to be created, but since the container
+ // size may be changed smaller than origin, we should be defensive.
+ final long dataSizeRequired =
+ Math.max(container.getUsedBytes(), currentContainerSize);
+ final List<DatanodeDetails> excludeList =
+ replicas.stream().map(ContainerReplica::getDatanodeDetails)
+ .collect(Collectors.toList());
+
+ return containerPlacement
+ .chooseDatanodes(excludeList, null, requiredNodes, 0, dataSizeRequired);
+ }
+
+ private static byte[] int2byte(List<Integer> src) {
+ byte[] dst = new byte[src.size()];
+
+ for (int i = 0; i < src.size(); i++) {
+ dst[i] = (byte) ((int) src.get(i));
Review Comment:
ah, done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r907451183
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationCommandCreator.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationCommandCreator
+ implements ReplicationCommandCreator {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationCommandCreator.class);
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationCommandCreator(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas,
+ pendingOps.getPendingOps(container.containerID()),
+ remainingMaintenanceRedundancy);
+ ContainerHealthResult containerHealthResult =
+ new ECContainerHealthCheck().checkHealth(container, replicaCount);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+
+ if (containerHealthResult.getHealthState()
+ != ContainerHealthResult.HealthState.UNDER_REPLICATED) {
+ LOG.info(
+ "The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(id);
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() > 0) {
Review Comment:
Should the sources check be `>= ec_data_num` - as if we don't have dataNum sources, we cannot recover?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908776710
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ (ContainerHealthResult.UnderReplicatedHealthResult)
+ ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (containerHealthResult
+ .isSufficientlyReplicatedAfterPending() || containerHealthResult
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ } else if (result instanceof ContainerHealthResult
+ .UnderReplicatedHealthResult && ((ContainerHealthResult
+ .UnderReplicatedHealthResult) result)
+ .underReplicatedDueToDecommission()) {
+ Map<DatanodeDetails, SCMCommand> commands = new HashMap<>();
+ Set<Integer> decomIndexes = replicaCount.decommissioningIndexes();
Review Comment:
HDDS-6967 for Maintenance replicas handling
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908930182
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ECContainerReplicaCount.java:
##########
@@ -127,6 +143,21 @@ public Set<Integer> decommissioningIndexes() {
return decommissionIndexes.keySet();
}
+ /**
+ * Get a set containing all decommissioning only indexes, or an empty set if
+ * none are decommissioning.
+ * @return Set of indexes in decommission only.
+ */
+ public Set<Integer> decommissioningOnlyIndexes() {
Review Comment:
Added tests
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] JacksonYao287 commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
JacksonYao287 commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908255692
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ (ContainerHealthResult.UnderReplicatedHealthResult)
+ ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (containerHealthResult
+ .isSufficientlyReplicatedAfterPending() || containerHealthResult
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ } else if (result instanceof ContainerHealthResult
+ .UnderReplicatedHealthResult && ((ContainerHealthResult
+ .UnderReplicatedHealthResult) result)
+ .underReplicatedDueToDecommission()) {
+ Map<DatanodeDetails, SCMCommand> commands = new HashMap<>();
+ Set<Integer> decomIndexes = replicaCount.decommissioningIndexes();
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, decomIndexes.size());
+ Iterator<DatanodeDetails> iterator = selectedDatanodes.iterator();
+ // In this case we need to do one to one copy.
+ for (ContainerReplica replica : replicas) {
+ if (decomIndexes.contains(replica.getReplicaIndex())) {
+ if (!iterator.hasNext()) {
+ LOG.warn("Couldn't find the enough targets. Source nodes:"
+ + " {} and the target nodes: {}",
+ sources, selectedDatanodes);
+ break;
+ }
+ DatanodeDetails decommissioningSrcNode
+ = replica.getDatanodeDetails();
+ final ReplicateContainerCommand replicateCommand =
+ new ReplicateContainerCommand(id.getProtobuf().getId(),
+ ImmutableList.of(decommissioningSrcNode));
+ DatanodeDetails target = iterator.next();
+ commands.put(target, replicateCommand);
+ }
+ }
+ return commands;
+ } else {
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable due to lack of"
+ + " available source replicas.", id);
+ } else {
+ LOG.info("There are no missing indexes or decommissioning indexes"
+ + " found for {}", id);
+ }
+ }
+
+ } catch (IOException | IllegalStateException ex) {
+ LOG.warn("Exception while processing for creating the EC reconstruction" +
+ " container commands for {}.",
+ id, ex);
+ }
+ return null;
+ }
+
+ private List<DatanodeDetails> getTargetDatanodes(
+ Set<ContainerReplica> replicas, ContainerInfo container,
+ int requiredNodes) throws IOException {
+ // We should ensure that the target datanode has enough space
+ // for a complete container to be created, but since the container
+ // size may be changed smaller than origin, we should be defensive.
+ final long dataSizeRequired =
+ Math.max(container.getUsedBytes(), currentContainerSize);
Review Comment:
according to the logic here, we always have to find a datanode that has at least 5G disk space, even if the container usedBytes is much smaller (for example , 100M). i am not sure if it is reasonable.
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ (ContainerHealthResult.UnderReplicatedHealthResult)
+ ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (containerHealthResult
+ .isSufficientlyReplicatedAfterPending() || containerHealthResult
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
Review Comment:
NIT: put these code inside line 141 if code block.
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ (ContainerHealthResult.UnderReplicatedHealthResult)
+ ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (containerHealthResult
+ .isSufficientlyReplicatedAfterPending() || containerHealthResult
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ } else if (result instanceof ContainerHealthResult
+ .UnderReplicatedHealthResult && ((ContainerHealthResult
+ .UnderReplicatedHealthResult) result)
+ .underReplicatedDueToDecommission()) {
+ Map<DatanodeDetails, SCMCommand> commands = new HashMap<>();
+ Set<Integer> decomIndexes = replicaCount.decommissioningIndexes();
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, decomIndexes.size());
+ Iterator<DatanodeDetails> iterator = selectedDatanodes.iterator();
+ // In this case we need to do one to one copy.
+ for (ContainerReplica replica : replicas) {
+ if (decomIndexes.contains(replica.getReplicaIndex())) {
+ if (!iterator.hasNext()) {
+ LOG.warn("Couldn't find the enough targets. Source nodes:"
+ + " {} and the target nodes: {}",
+ sources, selectedDatanodes);
+ break;
+ }
+ DatanodeDetails decommissioningSrcNode
+ = replica.getDatanodeDetails();
+ final ReplicateContainerCommand replicateCommand =
+ new ReplicateContainerCommand(id.getProtobuf().getId(),
+ ImmutableList.of(decommissioningSrcNode));
+ DatanodeDetails target = iterator.next();
+ commands.put(target, replicateCommand);
+ }
+ }
+ return commands;
+ } else {
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable due to lack of"
+ + " available source replicas.", id);
+ } else {
+ LOG.info("There are no missing indexes or decommissioning indexes"
+ + " found for {}", id);
+ }
+ }
+
+ } catch (IOException | IllegalStateException ex) {
+ LOG.warn("Exception while processing for creating the EC reconstruction" +
Review Comment:
NIT: can we log out in more detail why fail to create EC reconstruction. for example, not enough target datanode can be selected.
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
Review Comment:
we already an input parameter "final ContainerHealthResult result". if i do not missunderstand, checkHealth should be done before calling this function. seems no need to do the check here again, let us just use the input result.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908270080
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
Review Comment:
Here were doing intentionally to make sure if any situation changed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908338009
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ (ContainerHealthResult.UnderReplicatedHealthResult)
+ ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (containerHealthResult
+ .isSufficientlyReplicatedAfterPending() || containerHealthResult
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
Review Comment:
Sorry, cancel that - the "true" passed in here considers the pending add.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908332499
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
Review Comment:
Is it safe to cast this directly, without first checking if the result is underReplicated? I am not sure what will happen if, for example, the result is OverReplicated - I suspect it will give some sort of cast / runtime error.
I think we need to check the result:
```
ContainerHealthResult currentHealth = ecContainerHealthCheck
.checkHealth(container, replicas, pendingOpsForContainer,
remainingMaintenanceRedundancy);
if (currentHealth.getHealthState() != UNDER_REPLICATED) {
// Log and return null
}
ContainerHealthResult.UnderReplicatedHealthResult currentUnderRepRes = ((ContainerHealthResult.UnderReplicatedHealthResult) currentHealth);
```
Then we can simplify the condition at line 109 a little, as we know for sure the state is UNDER_REP so we don't need to check it there. Similar for the condition at 174 - we can just use the already cast variable from here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r907884816
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationCommandCreator.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationCommandCreator
+ implements ReplicationCommandCreator {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationCommandCreator.class);
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationCommandCreator(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas,
+ pendingOps.getPendingOps(container.containerID()),
+ remainingMaintenanceRedundancy);
+ ContainerHealthResult containerHealthResult =
+ new ECContainerHealthCheck().checkHealth(container, replicaCount);
Review Comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908893517
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - Set of available container replicas.
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand<?>> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return emptyMap();
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOps) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ }
+
+ if (containerHealthResult.underReplicatedDueToDecommission()) {
+ Map<DatanodeDetails, SCMCommand<?>> commands = new HashMap<>();
+ Set<Integer> decomIndexes = replicaCount.decommissioningOnlyIndexes();
Review Comment:
This does not consider pendingAdd replicas - there may be some already scheduled for the decommission only indexes. I think this new `decommissioningOnlyIndexes` and its twin `maintenanceOnlyIndexes` need to take the `includePendingAdd` flag like I changed most of the other methods to do.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r909212008
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - Set of available container replicas.
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand<?>> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return emptyMap();
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ final ContainerID id = container.containerID();
+ final Map<DatanodeDetails, SCMCommand<?>> commands = new HashMap<>();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOps) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
Review Comment:
I am not sure why we did not add HEALTHY_READONLY check. But I just added this check here. If that needs to be added there we can file a separate bug to fix Legacy one too.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r907890247
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationCommandCreator.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationCommandCreator
+ implements ReplicationCommandCreator {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationCommandCreator.class);
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationCommandCreator(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas,
+ pendingOps.getPendingOps(container.containerID()),
+ remainingMaintenanceRedundancy);
+ ContainerHealthResult containerHealthResult =
+ new ECContainerHealthCheck().checkHealth(container, replicaCount);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+
+ if (containerHealthResult.getHealthState()
+ != ContainerHealthResult.HealthState.UNDER_REPLICATED) {
Review Comment:
Ok. Added the check.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] JacksonYao287 commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
JacksonYao287 commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908295951
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
Review Comment:
if so , seems we should add sanity check and write the code like this :
```
ContainerHealthResult currentHealthResult = ecContainerHealthCheck
.checkHealth(container, replicas, pendingOpsForContainer, remainingMaintenanceRedundancy);
if(currentHealthResult.getHealthState() != HealthState.UNDER_REPLICATED) {
Log.warn("state changed, not under replicated any more");
return;
}
ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
(ContainerHealthResult.UnderReplicatedHealthResult) currentHealthResult;
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908778481
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,243 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return null;
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return null;
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ }
Review Comment:
Good catch. I did return empty map now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on PR #3542:
URL: https://github.com/apache/ozone/pull/3542#issuecomment-1170486526
Thanks @sodonnel @adoroszlai @JacksonYao287 for the great reviews.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] JacksonYao287 commented on pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
JacksonYao287 commented on PR #3542:
URL: https://github.com/apache/ozone/pull/3542#issuecomment-1169959793
LGTM +1! pending clean CI
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao merged pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao merged PR #3542:
URL: https://github.com/apache/ozone/pull/3542
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r907891863
##########
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestECContainerReplicaCount.java:
##########
@@ -88,10 +95,14 @@ public void testContainerMissingReplicaDueToPendingDelete() {
registerNodes(Pair.of(IN_SERVICE, 1), Pair.of(IN_SERVICE, 2),
Pair.of(IN_SERVICE, 3), Pair.of(IN_SERVICE, 4),
Pair.of(IN_SERVICE, 5));
- List<Integer> delete = new ArrayList<>();
- delete.add(1);
- ECContainerReplicaCount rcnt = new ECContainerReplicaCount(container,
- replica, new ArrayList<>(), delete, 1);
+ ContainerReplicaPendingOps pendingOps =
Review Comment:
I just used the create method and also extracted to a method to avoid some duplication here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908339040
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ (ContainerHealthResult.UnderReplicatedHealthResult)
+ ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (containerHealthResult
+ .isSufficientlyReplicatedAfterPending() || containerHealthResult
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
Review Comment:
When we pass flag true, isn't it subtracted already?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r907475200
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationCommandCreator.java:
##########
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * This interface to create respective commands after processing the replicas
+ * with pending ops and health check results.
+ */
+public interface ReplicationCommandCreator {
Review Comment:
I wonder if a better name would be `ContainerUnderReplicationHandler` or just `UnderReplicationHandler`. Then we could have `ECUnderReplicationHandler`, `RatisUnderReplicationHandler`, etc
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r907884670
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationCommandCreator.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationCommandCreator
Review Comment:
Changed it to ECUnderReplicationHandler
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] JacksonYao287 commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
JacksonYao287 commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r909147369
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - Set of available container replicas.
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand<?>> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return emptyMap();
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ final ContainerID id = container.containerID();
+ final Map<DatanodeDetails, SCMCommand<?>> commands = new HashMap<>();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOps) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
Review Comment:
the eligible source datanode should satisfy:
1 NodeState is HEALTHY or HEALTHY_READONLY
2 NodeOperationalState is IN_SERVICE
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - Set of available container replicas.
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand<?>> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return emptyMap();
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ if (replicaCount.unRecoverable()) {
Review Comment:
i am not sure whether we should take some other actions if unRecoverable?
delete it or some other actions. we can just keep it like this for now, and take any necessary actions if needed in the future
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - Set of available container replicas.
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand<?>> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return emptyMap();
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ final ContainerID id = container.containerID();
+ final Map<DatanodeDetails, SCMCommand<?>> commands = new HashMap<>();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOps) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
Review Comment:
we should not just check the size of sources here, we should check that the eligible source datanodes holds the different replica indexes , which should be larger than repConfig.getData()
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - Set of available container replicas.
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand<?>> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return emptyMap();
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ final ContainerID id = container.containerID();
+ final Map<DatanodeDetails, SCMCommand<?>> commands = new HashMap<>();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOps) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
Review Comment:
"!replicaCount.unRecoverable()" is unnecessary here, since we return at line 124 if it is unRecoverable
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r909213118
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - Set of available container replicas.
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand<?>> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return emptyMap();
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ final ContainerID id = container.containerID();
+ final Map<DatanodeDetails, SCMCommand<?>> commands = new HashMap<>();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOps) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
Review Comment:
Tried to map things into map with key ad replica index finally after filtering. So, the sources will be unique.
Probably this filtering can be done more based on DN load as well. Let's say if we have 2 src nodes with same replica index, that whichever node has less load that node can be chosen. That can be improved later.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r906276739
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ECContainerReplicaCount.java:
##########
@@ -68,16 +70,27 @@ public class ECContainerReplicaCount {
private final Map<Integer, Integer> healthyIndexes = new HashMap<>();
private final Map<Integer, Integer> decommissionIndexes = new HashMap<>();
private final Map<Integer, Integer> maintenanceIndexes = new HashMap<>();
+ private final Set<ContainerReplica> replicas;
public ECContainerReplicaCount(ContainerInfo containerInfo,
- Set<ContainerReplica> replicas, List<Integer> indexesPendingAdd,
- List<Integer> indexesPendingDelete, int remainingMaintenanceRedundancy) {
+ Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
Review Comment:
I updated it to take List<ContainerReplicaOP>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908334155
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ (ContainerHealthResult.UnderReplicatedHealthResult)
+ ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (containerHealthResult
+ .isSufficientlyReplicatedAfterPending() || containerHealthResult
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ } else if (result instanceof ContainerHealthResult
+ .UnderReplicatedHealthResult && ((ContainerHealthResult
+ .UnderReplicatedHealthResult) result)
+ .underReplicatedDueToDecommission()) {
+ Map<DatanodeDetails, SCMCommand> commands = new HashMap<>();
+ Set<Integer> decomIndexes = replicaCount.decommissioningIndexes();
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, decomIndexes.size());
+ Iterator<DatanodeDetails> iterator = selectedDatanodes.iterator();
+ // In this case we need to do one to one copy.
+ for (ContainerReplica replica : replicas) {
+ if (decomIndexes.contains(replica.getReplicaIndex())) {
+ if (!iterator.hasNext()) {
+ LOG.warn("Couldn't find the enough targets. Available source"
+ + " nodes: {}, the target nodes: {} and the decommission"
+ + " indexes: {}", replicas, selectedDatanodes, decomIndexes);
+ break;
+ }
+ DatanodeDetails decommissioningSrcNode
+ = replica.getDatanodeDetails();
+ final ReplicateContainerCommand replicateCommand =
+ new ReplicateContainerCommand(id.getProtobuf().getId(),
+ ImmutableList.of(decommissioningSrcNode));
+ DatanodeDetails target = iterator.next();
+ commands.put(target, replicateCommand);
+ }
+ }
+ return commands;
+ } else {
+ if (replicaCount.unRecoverable()) {
Review Comment:
Might be cleaner if we lifted these checks to the top of the method along with the others, as they they are all groups together.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908335263
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ (ContainerHealthResult.UnderReplicatedHealthResult)
+ ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (containerHealthResult
+ .isSufficientlyReplicatedAfterPending() || containerHealthResult
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ } else if (result instanceof ContainerHealthResult
Review Comment:
I think we can handle the reconstruction command and the replication commands in a single pass - we don't need it to be `if () ... else if ()` - we should be able to create both sets of commands to repair the container in a single pass.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908324533
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
Review Comment:
I remember we defined this API to pass containerPendingOps initially. I agree this instance is not needed here anymore as this class is not firing events, we need this at the place where we fire the events and should update scheduleAdd and scheduledDelete ops.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r909213118
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - Set of available container replicas.
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand<?>> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return emptyMap();
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ final ContainerID id = container.containerID();
+ final Map<DatanodeDetails, SCMCommand<?>> commands = new HashMap<>();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOps) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
Review Comment:
Tried to map things into map with key ad replica index finally after filtering. So, the sources will be unique.
Probably this filtering can be more based on DN load as well. Let's say if we have 2 src nodes with same replica index, that whichever node has less load that node can be chosen. That can be improved later.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r909470003
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - Set of available container replicas.
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand<?>> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return emptyMap();
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ final ContainerID id = container.containerID();
+ final Map<DatanodeDetails, SCMCommand<?>> commands = new HashMap<>();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOps) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
Review Comment:
@JacksonYao287 Do you know where this HEALTHY_READONLY state came from and why? Surely by definition any node which is HEALTHY and NOT IN_SERVICE is HEALTHY_READONLY - I don't see why we needed an extra health state for this?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r907480075
##########
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestECContainerReplicaCount.java:
##########
@@ -88,10 +95,14 @@ public void testContainerMissingReplicaDueToPendingDelete() {
registerNodes(Pair.of(IN_SERVICE, 1), Pair.of(IN_SERVICE, 2),
Pair.of(IN_SERVICE, 3), Pair.of(IN_SERVICE, 4),
Pair.of(IN_SERVICE, 5));
- List<Integer> delete = new ArrayList<>();
- delete.add(1);
- ECContainerReplicaCount rcnt = new ECContainerReplicaCount(container,
- replica, new ArrayList<>(), delete, 1);
+ ContainerReplicaPendingOps pendingOps =
Review Comment:
We don't need an instance of ContainerReplicaPendingOps for these tests.
Just:
```
List<ContainerReplicaOP> pending = new ArrayList<>();
pending.add(ContainerReplicaOp.create(...));
```
Note, I think the static `ContainerReplicaOp.create` method is only added in HDDS-6699 - ie not yet committed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908343648
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ (ContainerHealthResult.UnderReplicatedHealthResult)
+ ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (containerHealthResult
+ .isSufficientlyReplicatedAfterPending() || containerHealthResult
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ } else if (result instanceof ContainerHealthResult
+ .UnderReplicatedHealthResult && ((ContainerHealthResult
+ .UnderReplicatedHealthResult) result)
+ .underReplicatedDueToDecommission()) {
+ Map<DatanodeDetails, SCMCommand> commands = new HashMap<>();
+ Set<Integer> decomIndexes = replicaCount.decommissioningIndexes();
Review Comment:
The decommissionIndexes returned from the ReplicaCount class might already have an additional copy on the cluster. The is a method maintenanceOnlyIndexes, but it only considers the maintenance state. I think we may need to add something more to repliaCount to allow the indexes which need more copies to be found.
We also need to handle maintenance together with decommission.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908308681
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
Review Comment:
We should pass in the `List<ContainerReplicaOp>` here, rather than the `ContainerReplicaPendingOps` instance.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908361826
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ (ContainerHealthResult.UnderReplicatedHealthResult)
+ ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (containerHealthResult
+ .isSufficientlyReplicatedAfterPending() || containerHealthResult
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ } else if (result instanceof ContainerHealthResult
+ .UnderReplicatedHealthResult && ((ContainerHealthResult
+ .UnderReplicatedHealthResult) result)
+ .underReplicatedDueToDecommission()) {
+ Map<DatanodeDetails, SCMCommand> commands = new HashMap<>();
+ Set<Integer> decomIndexes = replicaCount.decommissioningIndexes();
Review Comment:
Isn't that can be handled in followups? I think this still works as one decom replica acts as src. ( at the same time having over replication may be rare case, but still we can handle if available)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908777974
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/UnderReplicationHandler.java:
##########
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * This interface to create respective commands after processing the replicas
+ * with pending ops and health check results.
+ */
+public interface UnderReplicationHandler {
+
+ /**
+ * Identify a new set of datanode(s) to replicate/reconstruct the container
+ * and form the SCM commands to send it to DN.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908895201
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ECContainerReplicaCount.java:
##########
@@ -127,6 +143,21 @@ public Set<Integer> decommissioningIndexes() {
return decommissionIndexes.keySet();
}
+ /**
+ * Get a set containing all decommissioning only indexes, or an empty set if
+ * none are decommissioning.
+ * @return Set of indexes in decommission only.
+ */
+ public Set<Integer> decommissioningOnlyIndexes() {
Review Comment:
Please try to add a simply test for this, and if we change it to have an `includePendingAdd` flag, then test both scenarios. It may fit in as part of another test with decommission making it just one or two additional lines.
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ECContainerReplicaCount.java:
##########
@@ -127,6 +143,21 @@ public Set<Integer> decommissioningIndexes() {
return decommissionIndexes.keySet();
}
+ /**
+ * Get a set containing all decommissioning only indexes, or an empty set if
+ * none are decommissioning.
+ * @return Set of indexes in decommission only.
+ */
+ public Set<Integer> decommissioningOnlyIndexes() {
Review Comment:
Please try to add a simple test for this, and if we change it to have an `includePendingAdd` flag, then test both scenarios. It may fit in as part of another test with decommission making it just one or two additional lines.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r909191560
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - Set of available container replicas.
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand<?>> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return emptyMap();
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ final ContainerID id = container.containerID();
+ final Map<DatanodeDetails, SCMCommand<?>> commands = new HashMap<>();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOps) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
Review Comment:
The API defined with replica set. However, ContainerReplica is not considered replica index in equals method. Even though it's set replicas can be duplicated if caller does not check.
It will fail at DN side later anyway as not enough sources since we are putting them in map with replicaIndex as key:
SortedMap<Integer, DatanodeDetails> sourceNodeMap =
reconstructionCommandInfo.getSources().stream().collect(Collectors
.toMap(DatanodeDetailsAndReplicaIndex::getReplicaIndex,
DatanodeDetailsAndReplicaIndex::getDnDetails, (v1, v2) -> v1,
TreeMap::new));
But we can handle here itself. Let me think.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r909209985
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - Set of available container replicas.
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand<?>> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return emptyMap();
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ final ContainerID id = container.containerID();
+ final Map<DatanodeDetails, SCMCommand<?>> commands = new HashMap<>();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOps) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
Review Comment:
Why we need to add IN_SERVICE? I believe Decom replica can be source too right?
BTW : Here is the LegacyReplicationManager#Underreplication filtering:
```
final List<DatanodeDetails> source = replicas.stream()
.filter(r ->
r.getState() == State.QUASI_CLOSED ||
r.getState() == State.CLOSED)
// Exclude stale and dead nodes. This is particularly important for
// maintenance nodes, as the replicas will remain present in the
// container manager, even when they go dead.
.filter(r ->
getNodeStatus(r.getDatanodeDetails()).isHealthy())
.filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
.sorted((r1, r2) -> r2.getSequenceId().compareTo(r1.getSequenceId()))
.map(ContainerReplica::getDatanodeDetails)
.collect(Collectors.toList());
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r909621904
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - Set of available container replicas.
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand<?>> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return emptyMap();
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ final ContainerID id = container.containerID();
+ final Map<DatanodeDetails, SCMCommand<?>> commands = new HashMap<>();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOps) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
Review Comment:
What's the conclusion here?
Looks like healthy nodes are considered with the combination of healthy + healthy_readonly.
```
/**
* Return a summary report on current cluster state.
* @return {@link Response}
*/
@GET
public Response getClusterState() {
List<DatanodeDetails> datanodeDetails = nodeManager.getAllNodes();
int containers = this.containerManager.getContainers().size();
int pipelines = this.pipelineManager.getPipelines().size();
int healthyDatanodes =
nodeManager.getNodeCount(NodeStatus.inServiceHealthy()) +
nodeManager.getNodeCount(NodeStatus.inServiceHealthyReadOnly());
```
Operational state would have been much better. If nodes is truely healthy, then node should be healthy. readonly, decom, maintenance are situations user made ones despite node is healthy.
But I am not sure we can do anything at upgrade side. Clearly looking at the above code, we may need to consider read_only also.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r907472537
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationCommandCreator.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationCommandCreator
+ implements ReplicationCommandCreator {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationCommandCreator.class);
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationCommandCreator(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas,
+ pendingOps.getPendingOps(container.containerID()),
+ remainingMaintenanceRedundancy);
+ ContainerHealthResult containerHealthResult =
+ new ECContainerHealthCheck().checkHealth(container, replicaCount);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+
+ if (containerHealthResult.getHealthState()
+ != ContainerHealthResult.HealthState.UNDER_REPLICATED) {
+ LOG.info(
+ "The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(id);
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
Review Comment:
We should subtract any indexsPendingAdd from the missingIndexes, as a recovery for some of the indexes may be inflight.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r907581869
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECContainerHealthCheck.java:
##########
@@ -89,19 +96,33 @@ public ContainerHealthResult checkHealth(ContainerInfo container,
return new ContainerHealthResult.HealthyResult(container);
}
- private ECContainerReplicaCount getReplicaCountWithPending(
+ private static ECContainerReplicaCount getReplicaCountWithPending(
Review Comment:
Yes, I agree, there is a overlap between, HDDS-6699. Let me wait on that.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r907890807
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationCommandCreator.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationCommandCreator
+ implements ReplicationCommandCreator {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationCommandCreator.class);
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationCommandCreator(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas,
+ pendingOps.getPendingOps(container.containerID()),
+ remainingMaintenanceRedundancy);
+ ContainerHealthResult containerHealthResult =
+ new ECContainerHealthCheck().checkHealth(container, replicaCount);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+
+ if (containerHealthResult.getHealthState()
+ != ContainerHealthResult.HealthState.UNDER_REPLICATED) {
+ LOG.info(
+ "The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ List<ContainerReplicaOp> pendingOpsForContainer =
Review Comment:
Done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r907906093
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationCommandCreator.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationCommandCreator
+ implements ReplicationCommandCreator {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationCommandCreator.class);
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationCommandCreator(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas,
+ pendingOps.getPendingOps(container.containerID()),
+ remainingMaintenanceRedundancy);
+ ContainerHealthResult containerHealthResult =
+ new ECContainerHealthCheck().checkHealth(container, replicaCount);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+
+ if (containerHealthResult.getHealthState()
Review Comment:
Yeah, it is ok to give underreplication preference over the overreplication. Once underreplication fixed, next interations can pick up overreplications. Later we can have hybrid state to check both.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] adoroszlai commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
adoroszlai commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908557422
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/UnderReplicationHandler.java:
##########
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * This interface to create respective commands after processing the replicas
+ * with pending ops and health check results.
+ */
+public interface UnderReplicationHandler {
+
+ /**
+ * Identify a new set of datanode(s) to replicate/reconstruct the container
+ * and form the SCM commands to send it to DN.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
Review Comment:
Nit: can get rid of warning due to raw type. (Needs tweaks where this method is referenced.)
```suggestion
Map<DatanodeDetails, SCMCommand<?> processAndCreateCommands(
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908794652
##########
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/states/TestECUnderReplicationHandler.java:
##########
@@ -0,0 +1,255 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.states;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.MockNodeManager;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementMetrics;
+import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementRackScatter;
+import org.apache.hadoop.hdds.scm.container.replication.ContainerHealthResult;
+import org.apache.hadoop.hdds.scm.container.replication.ECUnderReplicationHandler;
+import org.apache.hadoop.hdds.scm.exceptions.SCMException;
+import org.apache.hadoop.hdds.scm.net.NetworkTopology;
+import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl;
+import org.apache.hadoop.hdds.scm.net.NodeSchema;
+import org.apache.hadoop.hdds.scm.net.NodeSchemaManager;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.hdds.scm.node.NodeStatus;
+import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException;
+import org.apache.hadoop.ozone.container.common.SCMTestUtils;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.assertj.core.util.Lists;
+import org.junit.Assert;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.DECOMMISSIONING;
+import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeOperationalState.IN_SERVICE;
+import static org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReplicaProto.State.CLOSED;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.LEAF_SCHEMA;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.RACK_SCHEMA;
+import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_SCHEMA;
+
+/**
+ * Tests the ECUnderReplicationHandling functionality.
+ */
+public class TestECUnderReplicationHandler {
+ private ECReplicationConfig repConfig;
+ private ContainerInfo container;
+ private NodeManager nodeManager;
+ private OzoneConfiguration conf;
+ private NetworkTopology cluster;
+ private PlacementPolicy policy;
+ private SCMContainerPlacementMetrics metrics;
+
+ @BeforeEach
+ public void setup() {
+ nodeManager = new MockNodeManager(true, 10) {
+ @Override
+ public NodeStatus getNodeStatus(DatanodeDetails dd)
+ throws NodeNotFoundException {
+ return NodeStatus.inServiceHealthy();
+ }
+ };
+ conf = SCMTestUtils.getConf();
+ repConfig = new ECReplicationConfig(3, 2);
+ container = createContainer(HddsProtos.LifeCycleState.CLOSED, repConfig);
+ // create placement policy instances
+ cluster = new NetworkTopologyImpl(NodeSchemaManager.getInstance());
+ policy =
+ new SCMContainerPlacementRackScatter(nodeManager, conf, cluster, true,
+ metrics) {
+ @Override
+ public List<DatanodeDetails> chooseDatanodes(
+ List<DatanodeDetails> excludedNodes,
+ List<DatanodeDetails> favoredNodes, int nodesRequiredToChoose,
+ long metadataSizeRequired, long dataSizeRequired)
+ throws SCMException {
+ List<DatanodeDetails> dns = new ArrayList<>();
+ for (int i = 0; i < nodesRequiredToChoose; i++) {
+ dns.add(MockDatanodeDetails.randomDatanodeDetails());
+ }
+ return dns;
+ }
+ };
+ NodeSchema[] schemas =
+ new NodeSchema[] {ROOT_SCHEMA, RACK_SCHEMA, LEAF_SCHEMA};
+ NodeSchemaManager.getInstance().init(schemas, true);
+
+ }
+
+ @BeforeEach
+ public void init() {
+ metrics = SCMContainerPlacementMetrics.create();
+ }
+
+ @Test
+ public void testUnderReplicationWithMissingParityIndex5() {
+ Set<ContainerReplica> availableReplicas =
+ registerNodes(Pair.of(IN_SERVICE, 1), Pair.of(IN_SERVICE, 2),
+ Pair.of(IN_SERVICE, 3), Pair.of(IN_SERVICE, 4));
+ testUnderReplicationWithMissingIndexes(ImmutableList.of(5),
+ availableReplicas, false);
+ }
+
+ @Test
+ public void testUnderReplicationWithMissingIndex34() {
+ Set<ContainerReplica> availableReplicas =
+ registerNodes(Pair.of(IN_SERVICE, 1), Pair.of(IN_SERVICE, 2),
+ Pair.of(IN_SERVICE, 5));
+ testUnderReplicationWithMissingIndexes(ImmutableList.of(3, 4),
+ availableReplicas, false);
+ }
+
+ @Test
+ public void testUnderReplicationWithMissingIndex2345() {
+ Set<ContainerReplica> availableReplicas =
+ registerNodes(Pair.of(IN_SERVICE, 1));
+ testUnderReplicationWithMissingIndexes(ImmutableList.of(2, 3, 4, 5),
+ availableReplicas, false);
+ }
+
+ @Test
+ public void testUnderReplicationWithMissingIndex12345() {
+ Set<ContainerReplica> availableReplicas = new HashSet<>();
+ testUnderReplicationWithMissingIndexes(ImmutableList.of(1, 2, 3, 4, 5),
+ availableReplicas, false);
+ }
+
+ @Test
+ public void testUnderReplicationWithDecomIndex1() {
+ Set<ContainerReplica> availableReplicas =
+ registerNodes(Pair.of(DECOMMISSIONING, 1), Pair.of(IN_SERVICE, 2),
+ Pair.of(IN_SERVICE, 3), Pair.of(IN_SERVICE, 4),
+ Pair.of(IN_SERVICE, 5));
+ testUnderReplicationWithMissingIndexes(Lists.emptyList(), availableReplicas,
+ true);
+ }
+
+ @Test
+ public void testUnderReplicationWithDecomIndex12() {
+ Set<ContainerReplica> availableReplicas =
+ registerNodes(Pair.of(DECOMMISSIONING, 1), Pair.of(DECOMMISSIONING, 2),
+ Pair.of(IN_SERVICE, 3), Pair.of(IN_SERVICE, 4),
+ Pair.of(IN_SERVICE, 5));
+ testUnderReplicationWithMissingIndexes(Lists.emptyList(), availableReplicas,
+ true);
+ }
+
+ @Test
+ public void testUnderReplicationWithMixedDecomAndMissingIndexes() {
+ Set<ContainerReplica> availableReplicas =
+ registerNodes(Pair.of(DECOMMISSIONING, 1), Pair.of(DECOMMISSIONING, 2),
+ Pair.of(IN_SERVICE, 3), Pair.of(IN_SERVICE, 4));
+ testUnderReplicationWithMissingIndexes(ImmutableList.of(5),
+ availableReplicas, false);
+ }
+
+ public void testUnderReplicationWithMissingIndexes(
+ List<Integer> missingIndexes, Set<ContainerReplica> availableReplicas,
+ boolean decom) {
+ ECUnderReplicationHandler ecURH =
+ new ECUnderReplicationHandler(policy, conf, nodeManager);
+ ContainerHealthResult.UnderReplicatedHealthResult result =
+ Mockito.mock(ContainerHealthResult.UnderReplicatedHealthResult.class);
+ Mockito.when(result.underReplicatedDueToDecommission()).thenReturn(decom);
+ Mockito.when(result.isUnrecoverable()).thenReturn(false);
+ Mockito.when(result.getContainerInfo()).thenReturn(container);
+
+ Map<DatanodeDetails, SCMCommand> datanodeDetailsSCMCommandMap = ecURH
+ .processAndCreateCommands(availableReplicas, ImmutableList.of(),
+ result, 1);
+ if (!decom) {
+ if (missingIndexes.size() <= repConfig.getParity()) {
+ Assertions.assertEquals(1, datanodeDetailsSCMCommandMap.size());
+ Map.Entry<DatanodeDetails, SCMCommand> dnVsCommand =
+ datanodeDetailsSCMCommandMap.entrySet().iterator().next();
+ Assert.assertTrue(
+ dnVsCommand.getValue() instanceof ReconstructECContainersCommand);
+ byte[] missingIndexesByteArr = new byte[missingIndexes.size()];
+ for (int i = 0; i < missingIndexes.size(); i++) {
+ missingIndexesByteArr[i] = (byte) (int) missingIndexes.get(i);
+ }
+ Assert.assertArrayEquals(missingIndexesByteArr,
+ ((ReconstructECContainersCommand) dnVsCommand.getValue())
+ .getMissingContainerIndexes());
+ } else {
+ Assertions.assertNull(datanodeDetailsSCMCommandMap);
+ }
+ } else {
+ int numDecomIndexes = 0;
+ for (ContainerReplica repl : availableReplicas) {
+ if (repl.getDatanodeDetails()
+ .getPersistedOpState() == DECOMMISSIONING) {
+ numDecomIndexes++;
+ }
+ }
+ Assertions
+ .assertEquals(numDecomIndexes, datanodeDetailsSCMCommandMap.size());
+ Set<Map.Entry<DatanodeDetails, SCMCommand>> entries =
+ datanodeDetailsSCMCommandMap.entrySet();
+ for (Map.Entry<DatanodeDetails, SCMCommand> dnCommand : entries) {
+ Assert.assertTrue(
+ dnCommand.getValue() instanceof ReplicateContainerCommand);
+ }
+ }
+ }
+
+ private Set<ContainerReplica> registerNodes(
+ Pair<HddsProtos.NodeOperationalState, Integer>... states) {
+ Set<ContainerReplica> replica = new HashSet<>();
+ for (Pair<HddsProtos.NodeOperationalState, Integer> s : states) {
+ DatanodeDetails dn = MockDatanodeDetails.randomDatanodeDetails();
+ dn.setPersistedOpState(s.getLeft());
+ replica.add(new ContainerReplica.ContainerReplicaBuilder()
+ .setContainerID(ContainerID.valueOf(1)).setContainerState(CLOSED)
+ .setDatanodeDetails(dn).setOriginNodeId(dn.getUuid()).setSequenceId(1)
+ .setReplicaIndex(s.getRight()).build());
+ }
+ return replica;
+ }
+
+ private ContainerInfo createContainer(HddsProtos.LifeCycleState state,
+ ReplicationConfig replicationConfig) {
+
+ return new ContainerInfo.Builder()
+ .setContainerID(ContainerID.valueOf(1).getId()).setState(state)
+ .setReplicationConfig(replicationConfig).build();
+ }
Review Comment:
I did tried to reuse for both TestECContainerReplicaCount and TestECUnderReplicationHandler
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r907469097
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationCommandCreator.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationCommandCreator
+ implements ReplicationCommandCreator {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationCommandCreator.class);
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationCommandCreator(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas,
+ pendingOps.getPendingOps(container.containerID()),
+ remainingMaintenanceRedundancy);
+ ContainerHealthResult containerHealthResult =
+ new ECContainerHealthCheck().checkHealth(container, replicaCount);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+
+ if (containerHealthResult.getHealthState()
+ != ContainerHealthResult.HealthState.UNDER_REPLICATED) {
Review Comment:
I guess we also need to check if it is under_replicated considering pending `result.isSufficientlyReplicatedAfterPending()`, as there is a chance there have already been jobs assigned to correct this container.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] umamaheswararao commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
umamaheswararao commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908913432
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - Set of available container replicas.
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand<?>> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return emptyMap();
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOps) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ }
+
+ if (containerHealthResult.underReplicatedDueToDecommission()) {
Review Comment:
oh now I remember why I put if else.
Since we removed if else, every time we check decom, this may not meet when missingIndexes>0 already. There if else make sense.
Since we don't need to do this in if else, we need decom only counter here. I will fix this.
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - Set of available container replicas.
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand<?>> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return emptyMap();
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOps) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ }
+
+ if (containerHealthResult.underReplicatedDueToDecommission()) {
+ Map<DatanodeDetails, SCMCommand<?>> commands = new HashMap<>();
+ Set<Integer> decomIndexes = replicaCount.decommissioningOnlyIndexes();
Review Comment:
Yeah, it's good count class is evolving by the use. Let's fix them and I will try to add test as well.
includePendingAdd was completely ignored when we discuss offline in the morning. Thanks for catching. maintenanceOnlyIndexes also need that flag, but not in this patch as we have separate JIRA for that. I will add the comment there not to miss.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] sodonnel commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
sodonnel commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r908751515
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - An instance of ContainerReplicaCount, containing the
+ * current replica count and inflight adds and deletes
+ * @param pendingOps - Inflight replications and deletion metrics.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final ContainerReplicaPendingOps pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ List<ContainerReplicaOp> pendingOpsForContainer =
+ pendingOps.getPendingOps(container.containerID());
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ (ContainerHealthResult.UnderReplicatedHealthResult)
+ ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOpsForContainer,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (containerHealthResult
+ .isSufficientlyReplicatedAfterPending() || containerHealthResult
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ // Assert replicaCount also telling it's sufficiently replicated.
+ return null;
+ }
+ final ContainerID id = container.containerID();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOpsForContainer) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
+ final List<DatanodeDetails> selectedDatanodes =
+ getTargetDatanodes(replicas, container, missingIndexes.size());
+
+ List<ReconstructECContainersCommand.DatanodeDetailsAndReplicaIndex>
+ sourceDatanodesWithIndex = new ArrayList<>();
+ for (ContainerReplica srcReplica : sources) {
+ sourceDatanodesWithIndex.add(
+ new ReconstructECContainersCommand
+ .DatanodeDetailsAndReplicaIndex(
+ srcReplica.getDatanodeDetails(),
+ srcReplica.getReplicaIndex()));
+ }
+
+ final ReconstructECContainersCommand reconstructionCommand =
+ new ReconstructECContainersCommand(id.getProtobuf().getId(),
+ sourceDatanodesWithIndex, selectedDatanodes,
+ int2byte(missingIndexes),
+ repConfig);
+ // Keeping the first target node as coordinator.
+ return ImmutableMap.of(selectedDatanodes.get(0),
+ reconstructionCommand);
+ } else {
+ LOG.warn("Cannot proceed for EC container reconstruction for {}, due"
+ + "to insufficient source replicas found. Number of source "
+ + "replicas needed: {}. Number of available source replicas are:"
+ + " {}. Available sources are: {}", container.containerID(),
+ repConfig.getData(), sources.size(), sources);
+ }
+ } else if (result instanceof ContainerHealthResult
Review Comment:
We should handle both reconstruction and decommission copies in one call to this method - they are not exclusive so it is not `if...else if`, but `if missing_indexes ... end` and `if decommission_only_indexes()... end`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org
[GitHub] [ozone] JacksonYao287 commented on a diff in pull request #3542: HDDS-6894. EC: ReplicationManager - Add class to handle under-replication and form a command for a datanode
Posted by GitBox <gi...@apache.org>.
JacksonYao287 commented on code in PR #3542:
URL: https://github.com/apache/ozone/pull/3542#discussion_r909149246
##########
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ECUnderReplicationHandler.java:
##########
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm.container.replication;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
+import org.apache.hadoop.hdds.scm.PlacementPolicy;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerReplica;
+import org.apache.hadoop.hdds.scm.container.ECContainerReplicaCount;
+import org.apache.hadoop.hdds.scm.node.NodeManager;
+import org.apache.hadoop.ozone.protocol.commands.ReconstructECContainersCommand;
+import org.apache.hadoop.ozone.protocol.commands.ReplicateContainerCommand;
+import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * Handles the EC Under replication processing and forming the respective SCM
+ * commands.
+ */
+public class ECUnderReplicationHandler implements UnderReplicationHandler {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ECUnderReplicationHandler.class);
+ private final ECContainerHealthCheck ecContainerHealthCheck =
+ new ECContainerHealthCheck();
+ private final PlacementPolicy containerPlacement;
+ private final long currentContainerSize;
+ private final NodeManager nodeManager;
+
+ public ECUnderReplicationHandler(
+ final PlacementPolicy containerPlacement, final ConfigurationSource conf,
+ NodeManager nodeManager) {
+ this.containerPlacement = containerPlacement;
+ this.currentContainerSize = (long) conf
+ .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE,
+ ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES);
+ this.nodeManager = nodeManager;
+ }
+
+ /**
+ * Identify a new set of datanode(s) to reconstruct the container and form the
+ * SCM command to send it to DN. In the case of decommission, it will just
+ * generate the replicate commands instead of reconstruction commands.
+ *
+ * @param replicas - Set of available container replicas.
+ * @param pendingOps - Inflight replications and deletion ops.
+ * @param result - Health check result.
+ * @param remainingMaintenanceRedundancy - represents that how many nodes go
+ * into maintenance.
+ * @return Returns the key value pair of destination dn where the command gets
+ * executed and the command itself.
+ */
+ @Override
+ public Map<DatanodeDetails, SCMCommand<?>> processAndCreateCommands(
+ final Set<ContainerReplica> replicas,
+ final List<ContainerReplicaOp> pendingOps,
+ final ContainerHealthResult result,
+ final int remainingMaintenanceRedundancy) {
+ ContainerInfo container = result.getContainerInfo();
+ ECReplicationConfig repConfig =
+ (ECReplicationConfig) container.getReplicationConfig();
+ final ECContainerReplicaCount replicaCount =
+ new ECContainerReplicaCount(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ ContainerHealthResult currentUnderRepRes = ecContainerHealthCheck
+ .checkHealth(container, replicas, pendingOps,
+ remainingMaintenanceRedundancy);
+
+ LOG.debug("Handling under-replicated EC container: {}", container);
+ if (currentUnderRepRes
+ .getHealthState() != ContainerHealthResult.HealthState
+ .UNDER_REPLICATED) {
+ LOG.info("The container {} state changed and it's not in under"
+ + " replication any more. Current state is: {}",
+ container.getContainerID(), currentUnderRepRes);
+ return emptyMap();
+ }
+ ContainerHealthResult.UnderReplicatedHealthResult containerHealthResult =
+ ((ContainerHealthResult.UnderReplicatedHealthResult)
+ currentUnderRepRes);
+ if (containerHealthResult.isSufficientlyReplicatedAfterPending()) {
+ LOG.info("The container {} with replicas {} is sufficiently replicated",
+ container.getContainerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ if (replicaCount.unRecoverable()) {
+ LOG.warn("The container {} is unrecoverable. The available replicas" +
+ " are: {}.", container.containerID(), replicaCount.getReplicas());
+ return emptyMap();
+ }
+ final ContainerID id = container.containerID();
+ final Map<DatanodeDetails, SCMCommand<?>> commands = new HashMap<>();
+ try {
+ // State is UNDER_REPLICATED
+ final List<DatanodeDetails> deletionInFlight = new ArrayList<>();
+ for (ContainerReplicaOp op : pendingOps) {
+ if (op.getOpType() == ContainerReplicaOp.PendingOpType.DELETE) {
+ deletionInFlight.add(op.getTarget());
+ }
+ }
+ List<Integer> missingIndexes = replicaCount.unavailableIndexes(true);
+ // We got the missing indexes, this is excluded any decommissioning
+ // indexes. Find the good source nodes.
+ if (missingIndexes.size() > 0 && !replicaCount.unRecoverable()) {
+ List<ContainerReplica> sources = replicas.stream().filter(r -> r
+ .getState() == StorageContainerDatanodeProtocolProtos
+ .ContainerReplicaProto.State.CLOSED)
+ // Exclude stale and dead nodes. This is particularly important for
+ // maintenance nodes, as the replicas will remain present in the
+ // container manager, even when they go dead.
+ .filter(r -> ReplicationManager
+ .getNodeStatus(r.getDatanodeDetails(), nodeManager).isHealthy())
+ .filter(r -> !deletionInFlight.contains(r.getDatanodeDetails()))
+ .collect(Collectors.toList());
+ LOG.debug("Missing indexes detected for the container {}." +
+ " The missing indexes are {}", id, missingIndexes);
+ // We have source nodes.
+ if (sources.size() >= repConfig.getData()) {
Review Comment:
we should not just check the size of sources here, we should check that the eligible source datanodes holds the different replica indexes , which should be larger than or equal to repConfig.getData()
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@ozone.apache.org
For additional commands, e-mail: issues-help@ozone.apache.org